Porbability features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import StratifiedKFold, train_test_split\n",
    "import scikitplot as skplt\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import lightgbm as lgb\n",
    "import pickle\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = [1,1,1,1,0]\n",
    "b = [0.4,0.4,0.4,0.4,0.3]\n",
    "roc_auc_score(a, b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make probability Dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def augment_probs(mydf):\n",
    "    info = {}\n",
    "    for var in mydf.columns:\n",
    "        if var in ['ID_code', 'target']:\n",
    "            continue\n",
    "        IQR = mydf[var].quantile([0.75]).values - mydf[var].quantile([0.25]).values \n",
    "        n = mydf.shape[0] \n",
    "        bin_size = 2.5 * IQR / (n ** (1/3)) \n",
    "        bin_number = int((mydf[var].max() - mydf[var].min())/bin_size)\n",
    "        key = 'prob_' + var\n",
    "        mydf[key] = pd.cut(mydf[var], bins = bin_number, labels = range(bin_number)).astype('float')\n",
    "        # df_map1 = mydf[mydf['target'] == 0].groupby(key)['target'].count()\n",
    "        df_map2 = mydf[mydf['target'] == 1].groupby(key)['target'].count()\n",
    "        df_map3 = mydf.groupby(key)['target'].count()\n",
    "        df_map_prob = (df_map2 / df_map3).fillna(0)\n",
    "        mydf[key] = mydf[key].map(df_map_prob)\n",
    "        info[var] = {\n",
    "            'bin' : bin_number,\n",
    "            'map' : df_map_prob,\n",
    "        }\n",
    "    return mydf, info\n",
    "\n",
    "def test_augment_probs(mydf, info):\n",
    "    for var in mydf.columns:\n",
    "        if var in ['ID_code', 'target']:\n",
    "            continue\n",
    "        bin_number = info[var]['bin']\n",
    "        pmap = info[var]['map']\n",
    "        key = 'prob_' + var\n",
    "        mydf[key] = pd.cut(mydf[var], bins = bin_number, labels = range(bin_number)).astype('float')\n",
    "        mydf[key] = mydf[key].map(pmap)\n",
    "    return mydf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "aug_df, info = augment_probs(mydf)\n",
    "aug_df.to_csv('paugTrain.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "testmydf = pd.read_csv('data/test.csv')\n",
    "testdf = test_augment_probs(testmydf, info)\n",
    "testdf.to_csv('paugTest', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train on prob dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('paugTrain.csv')\n",
    "X_train = df_train.drop(['ID_code', 'target'], axis=1)\n",
    "y_train = df_train.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xtr, Xval, ytr, yval  = train_test_split(X_train, y_train, test_size = 0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(190000, 400)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xtr.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_model(models, filename):\n",
    "    with open(filename, 'wb') as handle:\n",
    "        pickle.dump(models, handle)\n",
    "        \n",
    "def augment(x,y,t=2):\n",
    "    xs,xn = [],[]\n",
    "    for i in range(t):\n",
    "        mask = y>0\n",
    "        x1 = x[mask].copy()\n",
    "        ids = np.arange(x1.shape[0])\n",
    "        for c in range(x1.shape[1]):\n",
    "            np.random.shuffle(ids)\n",
    "            x1[:,c] = x1[ids][:,c]\n",
    "        xs.append(x1)\n",
    "\n",
    "    for i in range(t//2):\n",
    "        mask = y==0\n",
    "        x1 = x[mask].copy()\n",
    "        ids = np.arange(x1.shape[0])\n",
    "        for c in range(x1.shape[1]):\n",
    "            np.random.shuffle(ids)\n",
    "            x1[:,c] = x1[ids][:,c]\n",
    "        xn.append(x1)\n",
    "    xs = np.vstack(xs)\n",
    "    xn = np.vstack(xn)\n",
    "    ys = np.ones(xs.shape[0])\n",
    "    yn = np.zeros(xn.shape[0])\n",
    "    x = np.vstack([x,xs,xn])\n",
    "    y = np.concatenate([y,ys,yn])\n",
    "    return x,y\n",
    "\n",
    "def lgb_trainer(X, y, params, n_folds):\n",
    "    skf = StratifiedKFold(n_splits=n_folds)\n",
    "    models = []\n",
    "    for train_idx, test_idx in skf.split(X.values, y.values):\n",
    "            gc.collect()\n",
    "            X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]\n",
    "            X_tr, y_tr = augment(X_train.values, y_train.values)\n",
    "            X_tr = pd.DataFrame(X_tr)\n",
    "            trn_data = lgb.Dataset(X_tr, label=y_tr)\n",
    "            test_data = lgb.Dataset(X.values[test_idx], label=y.values[test_idx])\n",
    "            model_lgb     = lgb.train(param, trn_data, 1000000, valid_sets = [trn_data, test_data], verbose_eval=5000, early_stopping_rounds = 4000)\n",
    "            models.append(model_lgb)\n",
    "            auc = roc_auc_score(y.values[test_idx], model_lgb.predict(X.values[test_idx]))\n",
    "    return models\n",
    "\n",
    "def lgb_trainer_no_aug(X, y, params, n_folds):\n",
    "    skf = StratifiedKFold(n_splits=n_folds)\n",
    "    models = []\n",
    "    for train_idx, test_idx in skf.split(X.values, y.values):\n",
    "            gc.collect()\n",
    "            X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]\n",
    "            X_tr, y_tr = X_train.values, y_train.values\n",
    "            X_tr = pd.DataFrame(X_tr)\n",
    "            trn_data = lgb.Dataset(X_tr, label=y_tr)\n",
    "            test_data = lgb.Dataset(X.values[test_idx], label=y.values[test_idx])\n",
    "            model_lgb     = lgb.train(param, trn_data, 1000000, valid_sets = [trn_data, test_data], verbose_eval=5000, early_stopping_rounds = 4000)\n",
    "            models.append(model_lgb)\n",
    "            auc = roc_auc_score(y.values[test_idx], model_lgb.predict(X.values[test_idx]))\n",
    "    return models\n",
    "\n",
    "def test(X, y, models):\n",
    "    preds = pd.DataFrame({})\n",
    "    for i, model in enumerate(models):\n",
    "        preds[str(i)] = model.predict(X)\n",
    "        print(f\"Fold: {i} \\t Score: {roc_auc_score(y, preds[str(i)].values)}\")\n",
    "    averaged_preds = preds.mean(axis=1)\n",
    "    print(f\"Score: {roc_auc_score(y, averaged_preds)}\")\n",
    "    return averaged_preds, preds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "param = {\n",
    "    'bagging_freq': 5,\n",
    "    'bagging_fraction': 0.335,\n",
    "    'boost_from_average':'false',\n",
    "    'boost': 'gbdt',\n",
    "    'feature_fraction': 0.041,\n",
    "    'learning_rate': 0.0083,\n",
    "    'max_depth': -1,\n",
    "    'metric':'auc',\n",
    "    'min_data_in_leaf': 80,\n",
    "    'min_sum_hessian_in_leaf': 10.0,\n",
    "    'num_leaves': 13,\n",
    "    'num_threads': 8,\n",
    "    'tree_learner': 'serial',\n",
    "    'objective': 'binary', \n",
    "    'verbosity': -1,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.953788\tvalid_1's auc: 0.903553\n",
      "[10000]\ttraining's auc: 0.965129\tvalid_1's auc: 0.91489\n",
      "[15000]\ttraining's auc: 0.970238\tvalid_1's auc: 0.917575\n",
      "[20000]\ttraining's auc: 0.974018\tvalid_1's auc: 0.918367\n",
      "[25000]\ttraining's auc: 0.977409\tvalid_1's auc: 0.918371\n",
      "Early stopping, best iteration is:\n",
      "[21669]\ttraining's auc: 0.97518\tvalid_1's auc: 0.91843\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.953962\tvalid_1's auc: 0.899077\n",
      "[10000]\ttraining's auc: 0.965241\tvalid_1's auc: 0.911142\n",
      "[15000]\ttraining's auc: 0.970298\tvalid_1's auc: 0.915007\n",
      "[20000]\ttraining's auc: 0.97407\tvalid_1's auc: 0.916235\n",
      "[25000]\ttraining's auc: 0.977441\tvalid_1's auc: 0.916481\n",
      "Early stopping, best iteration is:\n",
      "[24699]\ttraining's auc: 0.977248\tvalid_1's auc: 0.916558\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.953783\tvalid_1's auc: 0.905273\n",
      "[10000]\ttraining's auc: 0.965083\tvalid_1's auc: 0.916748\n",
      "[15000]\ttraining's auc: 0.970157\tvalid_1's auc: 0.920083\n",
      "[20000]\ttraining's auc: 0.973951\tvalid_1's auc: 0.921112\n",
      "[25000]\ttraining's auc: 0.97731\tvalid_1's auc: 0.921297\n",
      "Early stopping, best iteration is:\n",
      "[24807]\ttraining's auc: 0.977186\tvalid_1's auc: 0.921324\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.95336\tvalid_1's auc: 0.907031\n",
      "[10000]\ttraining's auc: 0.9649\tvalid_1's auc: 0.918649\n",
      "[15000]\ttraining's auc: 0.970045\tvalid_1's auc: 0.921647\n",
      "[20000]\ttraining's auc: 0.973856\tvalid_1's auc: 0.922398\n",
      "[25000]\ttraining's auc: 0.977255\tvalid_1's auc: 0.922461\n",
      "Early stopping, best iteration is:\n",
      "[23800]\ttraining's auc: 0.976462\tvalid_1's auc: 0.922523\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.954279\tvalid_1's auc: 0.897563\n",
      "[10000]\ttraining's auc: 0.965622\tvalid_1's auc: 0.910156\n",
      "[15000]\ttraining's auc: 0.970703\tvalid_1's auc: 0.913333\n",
      "[20000]\ttraining's auc: 0.974437\tvalid_1's auc: 0.914047\n",
      "[25000]\ttraining's auc: 0.977789\tvalid_1's auc: 0.914112\n",
      "Early stopping, best iteration is:\n",
      "[23200]\ttraining's auc: 0.976611\tvalid_1's auc: 0.914193\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.954031\tvalid_1's auc: 0.898767\n",
      "[10000]\ttraining's auc: 0.965474\tvalid_1's auc: 0.909738\n",
      "[15000]\ttraining's auc: 0.970545\tvalid_1's auc: 0.912915\n",
      "[20000]\ttraining's auc: 0.974334\tvalid_1's auc: 0.913556\n",
      "[25000]\ttraining's auc: 0.977723\tvalid_1's auc: 0.913476\n",
      "Early stopping, best iteration is:\n",
      "[22728]\ttraining's auc: 0.976205\tvalid_1's auc: 0.913632\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.953956\tvalid_1's auc: 0.90479\n",
      "[10000]\ttraining's auc: 0.965212\tvalid_1's auc: 0.915906\n",
      "[15000]\ttraining's auc: 0.970228\tvalid_1's auc: 0.919093\n",
      "[20000]\ttraining's auc: 0.974006\tvalid_1's auc: 0.920224\n",
      "[25000]\ttraining's auc: 0.977399\tvalid_1's auc: 0.920253\n",
      "Early stopping, best iteration is:\n",
      "[23625]\ttraining's auc: 0.976494\tvalid_1's auc: 0.920399\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.953855\tvalid_1's auc: 0.903587\n",
      "[10000]\ttraining's auc: 0.965182\tvalid_1's auc: 0.914808\n",
      "[15000]\ttraining's auc: 0.970309\tvalid_1's auc: 0.917838\n",
      "[20000]\ttraining's auc: 0.974116\tvalid_1's auc: 0.918424\n",
      "[25000]\ttraining's auc: 0.977521\tvalid_1's auc: 0.918413\n",
      "Early stopping, best iteration is:\n",
      "[22259]\ttraining's auc: 0.975699\tvalid_1's auc: 0.918491\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.953423\tvalid_1's auc: 0.910979\n",
      "[10000]\ttraining's auc: 0.964841\tvalid_1's auc: 0.921456\n",
      "[15000]\ttraining's auc: 0.96998\tvalid_1's auc: 0.924155\n",
      "[20000]\ttraining's auc: 0.973811\tvalid_1's auc: 0.924896\n",
      "[25000]\ttraining's auc: 0.977225\tvalid_1's auc: 0.924796\n",
      "Early stopping, best iteration is:\n",
      "[22270]\ttraining's auc: 0.975396\tvalid_1's auc: 0.925024\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.953818\tvalid_1's auc: 0.906483\n",
      "[10000]\ttraining's auc: 0.965125\tvalid_1's auc: 0.917268\n",
      "[15000]\ttraining's auc: 0.970156\tvalid_1's auc: 0.920138\n",
      "[20000]\ttraining's auc: 0.973919\tvalid_1's auc: 0.921084\n",
      "Early stopping, best iteration is:\n",
      "[20689]\ttraining's auc: 0.974403\tvalid_1's auc: 0.921106\n"
     ]
    }
   ],
   "source": [
    "first_models_v = lgb_trainer(Xtr, ytr, param, n_folds = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_model(first_models_v, \"probs_model_augmented_on_val.m\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.942318\tvalid_1's auc: 0.913375\n",
      "[10000]\ttraining's auc: 0.959361\tvalid_1's auc: 0.919047\n",
      "[15000]\ttraining's auc: 0.970405\tvalid_1's auc: 0.919307\n",
      "Early stopping, best iteration is:\n",
      "[12679]\ttraining's auc: 0.965561\tvalid_1's auc: 0.919548\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.942287\tvalid_1's auc: 0.909312\n",
      "[10000]\ttraining's auc: 0.959412\tvalid_1's auc: 0.917216\n",
      "[15000]\ttraining's auc: 0.970512\tvalid_1's auc: 0.91774\n",
      "Early stopping, best iteration is:\n",
      "[13026]\ttraining's auc: 0.966445\tvalid_1's auc: 0.917904\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.942102\tvalid_1's auc: 0.914823\n",
      "[10000]\ttraining's auc: 0.959238\tvalid_1's auc: 0.920966\n",
      "[15000]\ttraining's auc: 0.970352\tvalid_1's auc: 0.92161\n",
      "Early stopping, best iteration is:\n",
      "[13324]\ttraining's auc: 0.966863\tvalid_1's auc: 0.921742\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.94196\tvalid_1's auc: 0.916089\n",
      "[10000]\ttraining's auc: 0.959123\tvalid_1's auc: 0.922691\n",
      "[15000]\ttraining's auc: 0.970425\tvalid_1's auc: 0.923372\n",
      "Early stopping, best iteration is:\n",
      "[14322]\ttraining's auc: 0.96902\tvalid_1's auc: 0.923449\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.942584\tvalid_1's auc: 0.907933\n",
      "[10000]\ttraining's auc: 0.95957\tvalid_1's auc: 0.914609\n",
      "[15000]\ttraining's auc: 0.97065\tvalid_1's auc: 0.914979\n",
      "Early stopping, best iteration is:\n",
      "[15542]\ttraining's auc: 0.97171\tvalid_1's auc: 0.915118\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.94272\tvalid_1's auc: 0.908915\n",
      "[10000]\ttraining's auc: 0.959677\tvalid_1's auc: 0.914555\n",
      "[15000]\ttraining's auc: 0.970731\tvalid_1's auc: 0.915528\n",
      "Early stopping, best iteration is:\n",
      "[14442]\ttraining's auc: 0.969626\tvalid_1's auc: 0.915603\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.942293\tvalid_1's auc: 0.914252\n",
      "[10000]\ttraining's auc: 0.959321\tvalid_1's auc: 0.920055\n",
      "[15000]\ttraining's auc: 0.970459\tvalid_1's auc: 0.920704\n",
      "[20000]\ttraining's auc: 0.979333\tvalid_1's auc: 0.920539\n",
      "Early stopping, best iteration is:\n",
      "[16121]\ttraining's auc: 0.972573\tvalid_1's auc: 0.92079\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.942148\tvalid_1's auc: 0.913838\n",
      "[10000]\ttraining's auc: 0.959266\tvalid_1's auc: 0.919689\n",
      "[15000]\ttraining's auc: 0.970474\tvalid_1's auc: 0.919598\n",
      "Early stopping, best iteration is:\n",
      "[12007]\ttraining's auc: 0.964081\tvalid_1's auc: 0.919994\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.941731\tvalid_1's auc: 0.921474\n",
      "[10000]\ttraining's auc: 0.959042\tvalid_1's auc: 0.926583\n",
      "[15000]\ttraining's auc: 0.970298\tvalid_1's auc: 0.926451\n",
      "Early stopping, best iteration is:\n",
      "[12801]\ttraining's auc: 0.965623\tvalid_1's auc: 0.926859\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.94229\tvalid_1's auc: 0.915309\n",
      "[10000]\ttraining's auc: 0.959333\tvalid_1's auc: 0.921504\n",
      "[15000]\ttraining's auc: 0.970437\tvalid_1's auc: 0.922461\n",
      "Early stopping, best iteration is:\n",
      "[14180]\ttraining's auc: 0.96877\tvalid_1's auc: 0.922596\n"
     ]
    }
   ],
   "source": [
    "first_models_no_aug = lgb_trainer_no_aug(Xtr, ytr, param, n_folds = 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_model(first_models_no_aug, \"probs_model_no_augmented.m\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold: 0 \t Score: 0.9238241618755103\n",
      "Fold: 1 \t Score: 0.9234295391503152\n",
      "Fold: 2 \t Score: 0.9234469537189564\n",
      "Fold: 3 \t Score: 0.9233404238577011\n",
      "Fold: 4 \t Score: 0.9236695807044859\n",
      "Fold: 5 \t Score: 0.9242265244089904\n",
      "Fold: 6 \t Score: 0.9239748731423924\n",
      "Fold: 7 \t Score: 0.923447168713631\n",
      "Fold: 8 \t Score: 0.9239153196175331\n",
      "Fold: 9 \t Score: 0.9232223917813556\n",
      "Score: 0.9244281894137482\n"
     ]
    }
   ],
   "source": [
    "q, w = test(Xval, yval, first_models_v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold: 0 \t Score: 0.9241793330779197\n",
      "Fold: 1 \t Score: 0.9244944077735195\n",
      "Fold: 2 \t Score: 0.9246285644504585\n",
      "Fold: 3 \t Score: 0.9246700584226529\n",
      "Fold: 4 \t Score: 0.9243303668368136\n",
      "Fold: 5 \t Score: 0.9241090298193314\n",
      "Fold: 6 \t Score: 0.9236623783828875\n",
      "Fold: 7 \t Score: 0.9248365717981165\n",
      "Fold: 8 \t Score: 0.9244765632155292\n",
      "Fold: 9 \t Score: 0.9238271718009545\n",
      "Score: 0.9252236697097013\n"
     ]
    }
   ],
   "source": [
    "q, w = test(Xval, yval, first_models_no_aug)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_answer(models):\n",
    "    answer_qdf = pd.read_csv('data/test.csv')\n",
    "    test_labels = answer_qdf.ID_code\n",
    "    answer_qdf = answer_qdf.drop('ID_code', axis=1)\n",
    "    y_preds = {}\n",
    "    for i, model in enumerate(models):\n",
    "            print(f\"On fold: {i}\")\n",
    "            y_preds[str(i)] = model.predict(answer_qdf)\n",
    "    y_preds = pd.DataFrame(y_preds)\n",
    "    answer_df = pd.DataFrame({\n",
    "        'ID_code' : test_labels,\n",
    "        'target' : y_preds.mean(axis = 1),\n",
    "        })\n",
    "    return answer_df "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}