LB augment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LB Shuffle Augment answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pandas_profiling\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import train_test_split, ParameterGrid\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\n",
    "from parfit import bestFit\n",
    "import lightgbm as lgb\n",
    "from skopt.space import Real, Integer\n",
    "from skopt.utils import use_named_args\n",
    "import itertools\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from skopt import gp_minimize\n",
    "TRAIN = 'data/train.csv'\n",
    "TEST = 'data/test.csv'\n",
    "SAMPLE = 'data/sample_submission.csv'\n",
    "train = pd.read_csv(TRAIN)\n",
    "test = pd.read_csv(TEST)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = train.drop(['ID_code', 'target'], axis=1)\n",
    "y = train.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def augment(x,y,t=2):\n",
    "    xs,xn = [],[]\n",
    "    for i in range(t):\n",
    "        mask = y>0\n",
    "        x1 = x[mask].copy()\n",
    "        ids = np.arange(x1.shape[0])\n",
    "        for c in range(x1.shape[1]):\n",
    "            np.random.shuffle(ids)\n",
    "            x1[:,c] = x1[ids][:,c]\n",
    "        xs.append(x1)\n",
    "\n",
    "    for i in range(t//2):\n",
    "        mask = y==0\n",
    "        x1 = x[mask].copy()\n",
    "        ids = np.arange(x1.shape[0])\n",
    "        for c in range(x1.shape[1]):\n",
    "            np.random.shuffle(ids)\n",
    "            x1[:,c] = x1[ids][:,c]\n",
    "        xn.append(x1)\n",
    "\n",
    "    xs = np.vstack(xs)\n",
    "    xn = np.vstack(xn)\n",
    "    ys = np.ones(xs.shape[0])\n",
    "    yn = np.zeros(xn.shape[0])\n",
    "    x = np.vstack([x,xs,xn])\n",
    "    y = np.concatenate([y,ys,yn])\n",
    "    return x,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "param = {\n",
    "    'bagging_freq': 5,\n",
    "    'bagging_fraction': 0.335,\n",
    "    'boost_from_average':'false',\n",
    "    'boost': 'gbdt',\n",
    "    'feature_fraction': 0.041,\n",
    "    'learning_rate': 0.0083,\n",
    "    'max_depth': -1,\n",
    "    'metric':'auc',\n",
    "    'min_data_in_leaf': 80,\n",
    "    'min_sum_hessian_in_leaf': 10.0,\n",
    "    'num_leaves': 13,\n",
    "    'num_threads': 8,\n",
    "    'tree_learner': 'serial',\n",
    "    'objective': 'binary', \n",
    "    'verbosity': -1\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.911936\tvalid_1's auc: 0.896492\n",
      "[10000]\ttraining's auc: 0.9219\tvalid_1's auc: 0.900367\n",
      "[15000]\ttraining's auc: 0.929507\tvalid_1's auc: 0.900799\n",
      "Early stopping, best iteration is:\n",
      "[14063]\ttraining's auc: 0.928155\tvalid_1's auc: 0.900877\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.912908\tvalid_1's auc: 0.896762\n",
      "[10000]\ttraining's auc: 0.922734\tvalid_1's auc: 0.900175\n",
      "[15000]\ttraining's auc: 0.930334\tvalid_1's auc: 0.900483\n",
      "[20000]\ttraining's auc: 0.937161\tvalid_1's auc: 0.900366\n",
      "Early stopping, best iteration is:\n",
      "[16413]\ttraining's auc: 0.932313\tvalid_1's auc: 0.900573\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.912846\tvalid_1's auc: 0.891986\n",
      "[10000]\ttraining's auc: 0.922749\tvalid_1's auc: 0.895467\n",
      "[15000]\ttraining's auc: 0.930317\tvalid_1's auc: 0.895507\n",
      "Early stopping, best iteration is:\n",
      "[15350]\ttraining's auc: 0.930815\tvalid_1's auc: 0.895548\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.912979\tvalid_1's auc: 0.896975\n",
      "[10000]\ttraining's auc: 0.922909\tvalid_1's auc: 0.899572\n",
      "Early stopping, best iteration is:\n",
      "[9750]\ttraining's auc: 0.922512\tvalid_1's auc: 0.899598\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.911984\tvalid_1's auc: 0.898063\n",
      "[10000]\ttraining's auc: 0.922009\tvalid_1's auc: 0.90071\n",
      "[15000]\ttraining's auc: 0.929598\tvalid_1's auc: 0.900848\n",
      "Early stopping, best iteration is:\n",
      "[12738]\ttraining's auc: 0.926276\tvalid_1's auc: 0.900933\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.912237\tvalid_1's auc: 0.901119\n",
      "[10000]\ttraining's auc: 0.922077\tvalid_1's auc: 0.903685\n",
      "[15000]\ttraining's auc: 0.929597\tvalid_1's auc: 0.903838\n",
      "Early stopping, best iteration is:\n",
      "[14119]\ttraining's auc: 0.928328\tvalid_1's auc: 0.903992\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.912224\tvalid_1's auc: 0.899462\n",
      "[10000]\ttraining's auc: 0.922222\tvalid_1's auc: 0.902162\n",
      "[15000]\ttraining's auc: 0.929841\tvalid_1's auc: 0.90251\n",
      "Early stopping, best iteration is:\n",
      "[13743]\ttraining's auc: 0.928017\tvalid_1's auc: 0.902599\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.912294\tvalid_1's auc: 0.897444\n",
      "[10000]\ttraining's auc: 0.922227\tvalid_1's auc: 0.901022\n",
      "[15000]\ttraining's auc: 0.929918\tvalid_1's auc: 0.901234\n",
      "Early stopping, best iteration is:\n",
      "[15633]\ttraining's auc: 0.930837\tvalid_1's auc: 0.9013\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.911517\tvalid_1's auc: 0.903272\n",
      "[10000]\ttraining's auc: 0.921483\tvalid_1's auc: 0.905875\n",
      "[15000]\ttraining's auc: 0.92908\tvalid_1's auc: 0.905893\n",
      "Early stopping, best iteration is:\n",
      "[11123]\ttraining's auc: 0.923284\tvalid_1's auc: 0.906026\n",
      "Training until validation scores don't improve for 4000 rounds.\n",
      "[5000]\ttraining's auc: 0.912255\tvalid_1's auc: 0.897606\n",
      "[10000]\ttraining's auc: 0.922128\tvalid_1's auc: 0.900963\n",
      "[15000]\ttraining's auc: 0.929757\tvalid_1's auc: 0.901416\n",
      "Early stopping, best iteration is:\n",
      "[14366]\ttraining's auc: 0.928833\tvalid_1's auc: 0.901574\n"
     ]
    }
   ],
   "source": [
    "skf = StratifiedKFold(n_splits=10)\n",
    "models = []\n",
    "for train_idx, test_idx in skf.split(X.values, y.values):\n",
    "        X_train, y_train = X.iloc[train_idx], y.iloc[train_idx]\n",
    "        X_tr, y_tr = augment(X_train.values, y_train.values)\n",
    "        X_tr = pd.DataFrame(X_tr)\n",
    "        trn_data = lgb.Dataset(X_tr, label=y_tr)\n",
    "        test_data = lgb.Dataset(X.values[test_idx], label=y.values[test_idx])\n",
    "        model_lgb     = lgb.train(param, trn_data, 1000000, valid_sets = [trn_data, test_data], verbose_eval=5000, early_stopping_rounds = 4000)\n",
    "        models.append(model_lgb)\n",
    "        auc = roc_auc_score(y.values[test_idx], model_lgb.predict(X.values[test_idx]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Create answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "codes = test.ID_code\n",
    "df_test = test.drop('ID_code', axis=1)\n",
    "answer = [model.predict(df_test, num_iteration=model.best_iteration) for model in models]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "ans = np.array(answer)\n",
    "a = np.average(ans, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "answer_df = pd.DataFrame({\n",
    "    'ID_code' : codes,\n",
    "    'target' : a\n",
    "})\n",
    "answer_df.to_csv('answer_9_lb_aug.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}