Model 12-Feature Extraction from VGG.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 387,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "from subprocess import check_output\n",
    "#print(check_output([\"ls\", \"../input/\"]))\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import xgboost as xgb\n",
    "from sklearn.pipeline import Pipeline,make_pipeline\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier,GradientBoostingClassifier\n",
    "from sklearn.naive_bayes import GaussianNB,MultinomialNB,BernoulliNB\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.preprocessing import MinMaxScaler,StandardScaler\n",
    "from mlens.visualization import corrmat\n",
    "from os.path import join as opj\n",
    "from matplotlib import pyplot as plt\n",
    "import tensorflow as tf\n",
    "import os\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from matplotlib import pyplot\n",
    "from keras.applications import VGG19\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Input, Flatten, Activation,Add, ZeroPadding2D ,AveragePooling2D\n",
    "from keras.layers import GlobalMaxPooling2D,concatenate\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.layers.merge import Concatenate\n",
    "from keras.models import Model\n",
    "from keras import initializers,regularizers\n",
    "from keras.optimizers import Adam,SGD\n",
    "from keras.initializers import glorot_uniform\n",
    "from keras.callbacks import ModelCheckpoint, Callback, EarlyStopping\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "import keras.backend as K\n",
    "K.set_image_data_format('channels_last')\n",
    "#K.set_learning_phase(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_json(\"input/train.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def collect_test():\n",
    "    test_data = pd.DataFrame()\n",
    "    for i in range(4):\n",
    "        test_piece = pd.read_csv('input/test_subparts/test'+str(i+1)+'.csv')\n",
    "        test_data = pd.concat([test_data,test_piece])\n",
    "        test_data = test_data.reset_index().drop(['index'],axis = 1)\n",
    "    return test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data = collect_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def stats_and_fill_na(data,training = True):\n",
    "    if training ==True:\n",
    "        data['b1_median'] = [np.median(i) for i in data['band_1']]\n",
    "        data['b2_median'] = [np.median(i) for i in data['band_2']]\n",
    "        data['b1_mean'] = [np.mean(i) for i in data['band_1']]\n",
    "        data['b2_mean'] = [np.mean(i) for i in data['band_2']]\n",
    "        data['b1_max'] = [np.max(i) for i in data['band_1']]\n",
    "        data['b2_max'] = [np.max(i) for i in data['band_2']]\n",
    "        data['b1_min'] = [np.min(i) for i in data['band_1']]\n",
    "        data['b2_min'] = [np.min(i) for i in data['band_2']]\n",
    "        data['b1_std'] = [np.std(i) for i in data['band_1']]\n",
    "        data['b2_std'] = [np.std(i) for i in data['band_2']]\n",
    "        data['b1_var'] = [np.var(i) for i in data['band_1']]\n",
    "        data['b2_var'] = [np.var(i) for i in data['band_2']]\n",
    "        global stats\n",
    "        stats = data.columns[-12:]\n",
    "        X_train = data[data['inc_angle']!='na'][stats]\n",
    "        y_train = data[data['inc_angle']!='na']['inc_angle']\n",
    "        lr_model = LinearRegression().fit(X_train,y_train)\n",
    "        data['inc_angle'][data['inc_angle']=='na'] = lr_model.predict(data[stats])\n",
    "    else:\n",
    "        only_band1 = ['band1_'+str(i) for i in range(75*75)]\n",
    "        only_band2 = ['band2_'+str(i) for i in range(75*75)]\n",
    "        both_bands = only_band1+only_band2\n",
    "        data['b1_median'] = np.median(data[only_band1],axis = 1)\n",
    "        data['b2_median'] = np.median(data[only_band2],axis = 1)\n",
    "        data['b1_mean'] = np.mean(data[only_band1],axis = 1)\n",
    "        data['b2_mean'] = np.mean(data[only_band2],axis = 1)\n",
    "        data['b1_max'] = np.max(data[only_band1],axis = 1)\n",
    "        data['b2_max'] = np.max(data[only_band2],axis = 1)\n",
    "        data['b1_min'] = np.min(data[only_band1],axis = 1)\n",
    "        data['b2_min'] = np.min(data[only_band2],axis = 1)\n",
    "        data['b1_std'] = np.std(data[only_band1],axis = 1)\n",
    "        data['b2_std'] = np.std(data[only_band2],axis = 1)\n",
    "        data['b1_var'] = np.var(data[only_band1],axis = 1)\n",
    "        data['b2_var'] = np.var(data[only_band2],axis = 1)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\yltbe\\Anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\ipykernel_launcher.py:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "train = stats_and_fill_na(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data = stats_and_fill_na(test_data,training = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prepare_band12(data,training = True):\n",
    "    if training==True:\n",
    "        X_band_1=np.array([np.array(band).astype(np.float32).reshape(75, 75) for band in data[\"band_1\"]])\n",
    "        X_band_2=np.array([np.array(band).astype(np.float32).reshape(75, 75) for band in data[\"band_2\"]])\n",
    "        target_train = data['is_iceberg']\n",
    "        X_other = X_other = data.drop(['band_1','band_2','id','is_iceberg'],axis = 1)\n",
    "        return X_band_1,X_band_2,target_train,X_other\n",
    "    else :\n",
    "        only_band1 = ['band1_'+str(i) for i in range(75*75)]\n",
    "        only_band2 = ['band2_'+str(i) for i in range(75*75)]\n",
    "        both_bands = only_band1+only_band2\n",
    "        X_band_1=np.array(data[only_band1]).reshape(8424,75,75)\n",
    "        X_band_2=np.array(data[only_band2]).reshape(8424,75,75)\n",
    "        X_other = data.iloc[:,-13:]\n",
    "        return X_band_1,X_band_2,X_other"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_band_1,X_band_2,target,X_others = prepare_band12(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_band_test_1,X_band_test_2,X_test_others = prepare_band12(test_data,training=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prepare_RGB(band1,band2,method = 'third mean'):\n",
    "    \n",
    "    if method=='third mean':\n",
    "        third = (band1+band2)/2\n",
    "        image = np.concatenate([band1[:, :, :, np.newaxis],\n",
    "                                band2[:, :, :, np.newaxis], \n",
    "                                third[:, :, :, np.newaxis]], axis=-1)\n",
    "    elif method=='third max':\n",
    "        third = np.max(np.concatenate([band1[:, :, :, np.newaxis], band2[:, :, :, np.newaxis]],axis = -1),axis = -1)\n",
    "        image = np.concatenate([band1[:, :, :, np.newaxis],\n",
    "                                band2[:, :, :, np.newaxis], \n",
    "                                third[:, :, :, np.newaxis]], axis=-1)\n",
    "    return image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = prepare_RGB(X_band_1,X_band_2,method = 'third mean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test = prepare_RGB(X_band_test_1,X_band_test_2,method = 'third mean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def feature_extract_vgg19(X_input):\n",
    "\n",
    "    base_model = VGG19(weights='imagenet',include_top=False,input_shape=(75,75,3),classes=1)\n",
    "    X = base_model.output\n",
    "    X = GlobalMaxPooling2D()(X)\n",
    "    model = Model(inputs=base_model.input, outputs=X)\n",
    "\n",
    "    block4_pool_features = model.predict(X_input)\n",
    "    return block4_pool_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = feature_extract_vgg19(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# scaler = MinMaxScaler().fit(X_train_others)\n",
    "# X_train_others = scaler.transform(X_train_others)\n",
    "# X_valid_others = scaler.transform(X_valid_others)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def concat_X_and_others():\n",
    "    X_whole = pd.concat([X_others,pd.DataFrame(X,columns = ['feature'+str(i+1) for i in range(X.shape[1])])],axis = 1)\n",
    "    return X_whole"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_whole = concat_X_and_others()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_whole['inc_angle'] = pd.to_numeric(X_whole['inc_angle'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "X_train_cv,X_valid,X_train_others,X_valid_others, y_train_cv, y_valid = train_test_split(X, X_others,target, random_state=2,stratify = target, train_size=0.80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\yltbe\\Anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "X_train_cv,X_valid, y_train_cv, y_valid = train_test_split(X_whole,target, random_state=2,stratify = target, train_size=0.80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del X_train_cv\n",
    "del y_train_cv\n",
    "del X_valid\n",
    "del y_valid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Machine learning models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameter tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pipeline_with_scaler(model,scaler = 'minmaxscaler'):\n",
    "    if scaler=='minmaxscaler':\n",
    "        steps = [('minmaxscaler',MinMaxScaler()),('model',model)]\n",
    "    elif scaler=='standardscaler':\n",
    "        steps = [('standardscaler',StandardScaler()),('model',model)]\n",
    "    pl = Pipeline(steps)\n",
    "    return pl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def search_param(model,param_dict,method = 'random',scaler_mode = None,n_iter = 40,scoring = 'accuracy',cv = 5,n_jobs = 4):\n",
    "    if method == 'random':\n",
    "        if scaler_mode:\n",
    "            pl = pipeline_with_scaler(model,scaler = scaler_mode)\n",
    "            param_dict = {'model__'+key:value for key,value in param_dict.items()}\n",
    "            rs = RandomizedSearchCV(pl,param_distributions=param_dict,n_iter = n_iter,scoring = scoring,cv = cv,verbose = 10,n_jobs = n_jobs).fit(X_whole,target)\n",
    "        else:\n",
    "            rs = RandomizedSearchCV(model,param_distributions=param_dict,n_iter = n_iter,scoring = scoring,cv = cv,verbose = 10,n_jobs = n_jobs).fit(X_whole,target)\n",
    "        return rs\n",
    "    elif method == 'grid':\n",
    "        if scaler_mode:\n",
    "            pl = pipeline_with_scaler(model,scaler = scaler_mode)\n",
    "            param_dict = {'model__'+key:value for key,value in param_dict.items()}\n",
    "            gs = GridSearchCV(pl,param_grid=param_dict,scoring = scoring,cv = cv,verbose = 10,n_jobs=n_jobs).fit(X_whole,target)\n",
    "        else:\n",
    "            gs = GridSearchCV(model,param_grid=param_dict,scoring = scoring,cv = cv,verbose = 10,n_jobs=n_jobs).fit(X_whole,target)\n",
    "        return gs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_search_result(bpm_search):\n",
    "    print('best_params:{}'.format(bpm_search.best_params_))\n",
    "    print('best_score:{}'.format(bpm_search.best_score_))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr = LogisticRegression(C = 0.01,tol = 1e-05)\n",
    "param_dict = {'penalty':['l1','l2'],\n",
    "              'tol':[0.001,0.0001,0.00001],\n",
    "              'C':[10,1,0.1,0.01]}\n",
    "rs_lr = search_param(model = lr,method = 'grid',scaler_mode = 'standardscaler',param_dict=param_dict,n_iter = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'C': 0.01, 'fit_intercept': True, 'penalty': 'l2', 'tol': 1e-05}"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_bpm = {'C': 0.01, 'penalty': 'l2', 'tol': 0.0001}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### K-nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "param_dict = {'n_neighbors':[5,9,13,17,21],\n",
    "              'weights':['uniform','distance'],\n",
    "              'p':[1,2]}\n",
    "knn_bpm_search = search_param(model = knn,method = 'grid',scaler_mode = 'standardscaler',param_dict=param_dict,n_iter = 20,n_jobs=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_params:{'n_neighbors': 9, 'p': 2, 'weights': 'uniform'}\n",
      "best_score:0.8223192019950125\n"
     ]
    }
   ],
   "source": [
    "knn_bpm = {'n_neighbors': 9, 'p': 2, 'weights': 'uniform'}\n",
    "knn_bsc = 0.8223192019950125"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "svm = SVC(C = 10,gamma = 0.001)\n",
    "param_dict = {'C':[100,10,1],\n",
    "              'kernel':['rbf'],\n",
    "              'gamma':[0.0001,0.001,0.01]}\n",
    "svm_bpm_search = search_param(model = svm,method = 'grid',scaler_mode = None,param_dict=param_dict,n_iter = 20,n_jobs=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_params:{'C': 10, 'gamma': 0.001, 'kernel': 'rbf'}\n",
      "best_score:0.8728179551122195\n"
     ]
    }
   ],
   "source": [
    "svm_bpm = {'C': 10, 'gamma': 0.001, 'kernel': 'rbf'}\n",
    "svm_bsc = 0.8728179551122195"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Naive bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nb = BernoulliNB()\n",
    "param_dict = {'alpha':[0.001,0.0001,0.01]+list(np.arange(0.1,1,0.1))}\n",
    "nb_bpm_search = search_param(model = nb,method = 'grid',scaler_mode = None,param_dict=param_dict,n_iter = 20,n_jobs=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_params:{'alpha': 0.001}\n",
      "best_score:0.7051122194513716\n"
     ]
    }
   ],
   "source": [
    "nb_bpm = {'alpha': 0.001}\n",
    "nb_bsc = 0.7051122194513716"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Random forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rf = RandomForestClassifier(n_estimators = 120,criterion = 'entropy',max_depth = 12,max_features = 0.5)\n",
    "param_dict = {'n_estimators':[120],\n",
    "              'criterion':['entropy'],\n",
    "              'max_depth':[12],\n",
    "              'max_features':[0.5]}\n",
    "rf_bpm_search = search_param(model = rf,method = 'grid',scaler_mode = 'minmaxscaler',param_dict=param_dict,n_iter = 40,n_jobs=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_params:{'criterion': 'entropy', 'max_depth': 12, 'max_features': 0.5, 'min_samples_leaf': 1, 'n_estimators': 120}\n",
      "best_score:0.8347880299251871\n"
     ]
    }
   ],
   "source": [
    "rf_bpm = {'criterion': 'entropy', 'max_depth': 12, 'max_features': 0.5, 'n_estimators': 120}\n",
    "rf_bsc = 0.8416458852867831"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_params:{'model__criterion': 'entropy', 'model__max_depth': 12, 'model__max_features': 0.5, 'model__n_estimators': 120}\n",
      "best_score:0.8366583541147132\n"
     ]
    }
   ],
   "source": [
    "get_search_result(rf_bpm_search)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mlp = MLPClassifier(hidden_layer_sizes = (256,256),solver = 'sgd',alpha = 1e-05,batch_size = 64,learning_rate = 'adaptive',max_iter = 150)\n",
    "param_dict = {'hidden_layer_sizes':[(512,512),(256,256),(100,100)],\n",
    "              'learning_rate_init':[0.1,0.01,0.001,0.0001],\n",
    "              'learning_rate':['constant','adaptive'],\n",
    "              'solver':['adam','sgd'],\n",
    "              'alpha':[0.001,0.0001,0.00001],\n",
    "              'batch_size':[32,64],\n",
    "              'max_iter':[50,100,150,200],\n",
    "              'early_stopping':[True,False]}\n",
    "mlp_bpm_search = search_param(model = mlp,method = 'random',scaler_mode = None,param_dict=param_dict,n_iter = 20,n_jobs=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_params:{'alpha': 1e-05, 'batch_size': 64, 'early_stopping': False, 'hidden_layer_sizes': (256, 256), 'learning_rate': 'adaptive', 'learning_rate_init': 0.001, 'max_iter': 150, 'solver': 'sgd'}\n",
      "best_score:0.8603491271820449\n"
     ]
    }
   ],
   "source": [
    "mlp_bpm = {'alpha': 1e-05, 'batch_size': 64, 'early_stopping': False, 'hidden_layer_sizes': (256, 256), 'learning_rate': 'adaptive', 'learning_rate_init': 0.001, 'max_iter': 150, 'solver': 'sgd'}\n",
    "mlp_bsc = 0.8603491271820449"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### GBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gbm = GradientBoostingClassifier(loss = 'exponential',n_estimators = 130,subsample = 0.8,max_depth = 5,max_features = 0.5)\n",
    "param_dict = {'loss':['exponential'],\n",
    "              'learning_rate':[0.1],\n",
    "              'n_estimators':[120,130,140],\n",
    "              'subsample':[0.7,0.8,0.9],\n",
    "              'max_depth':[4,5,6],\n",
    "              'max_features':[0.5,0.6,0.4]}\n",
    "gbm_bpm_search = search_param(model = gbm,method = 'grid',scaler_mode = None,param_dict=param_dict,n_iter = 40,n_jobs=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_params:{'subsample': 0.5, 'n_estimators': 150, 'max_features': 0.8, 'max_depth': 4, 'loss': 'exponential', 'learning_rate': 0.1}\n",
      "best_score:0.8466334164588528\n"
     ]
    }
   ],
   "source": [
    "gbm_bpm = {'learning_rate': 0.1, 'loss': 'exponential', 'max_depth': 5, 'max_features': 0.5, 'n_estimators': 130, 'subsample': 0.8}\n",
    "gbm_bsc = 0.8603491271820449"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_models():\n",
    "    lr = LogisticRegression(C = 0.01,tol = 1e-05)\n",
    "    knn = KNeighborsClassifier(n_neighbors = 9)\n",
    "    svm = SVC(C = 10,gamma = 0.001,probability = True)\n",
    "    nb = BernoulliNB(alpha = 0.001)\n",
    "    rf = RandomForestClassifier(n_estimators = 120,criterion = 'entropy',max_depth = 12,max_features = 0.5)\n",
    "    mlp = MLPClassifier(hidden_layer_sizes = (256,256),solver = 'sgd',alpha = 1e-05,\n",
    "                        batch_size = 64,learning_rate = 'adaptive',max_iter = 150)\n",
    "    gbm = GradientBoostingClassifier(loss = 'exponential',n_estimators = 130,\n",
    "                                     subsample = 0.8,max_depth = 5,max_features = 0.5)\n",
    "    models = dict(lr = lr,knn = knn,svm = svm,nb = nb,rf = rf,mlp = mlp,gbm = gbm)\n",
    "    return models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "models = get_models()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate_on_cv(model,X,y):\n",
    "    single_model_pred_cv = pd.Series(np.zeros(shape = y.shape[0]))\n",
    "    folds = list(StratifiedKFold(n_splits=5,shuffle = True,random_state = 7).split(X,y))\n",
    "    for j,(train_mask,valid_mask) in enumerate(folds):\n",
    "        X_train = X.iloc[train_mask]\n",
    "        y_train = y.iloc[train_mask]\n",
    "        X_valid = X.iloc[valid_mask]\n",
    "        y_valid = y.iloc[valid_mask]\n",
    "        fitted_model = model.fit(X_train,y_train)\n",
    "        preds = fitted_model.predict_proba(X_valid)\n",
    "        single_model_pred_cv.iloc[valid_mask] = preds[:,1]\n",
    "        print('===========> '+'fold{} ,done!'.format(str(j+1)))\n",
    "    acc = accuracy_score(y,single_model_pred_cv>0.5)\n",
    "    return single_model_pred_cv,acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_predict_matrix(model_dict):\n",
    "    predict_proba_matrix = np.zeros(shape = (X_whole.shape[0],len(model_dict)))\n",
    "    predict_proba_matrix = pd.DataFrame(predict_proba_matrix,columns = model_dict.keys())\n",
    "    score_dict = dict()\n",
    "    for i,(name,model) in enumerate(model_dict.items()):\n",
    "        print('number '+str(i+1)+', '+name)\n",
    "        single_pred_proba,single_acc = evaluate_on_cv(model = model,X = X_whole,y = target)\n",
    "        predict_proba_matrix[name] = single_pred_proba\n",
    "        score_dict[name] = single_acc\n",
    "        print('number '+str(i+1)+', '+name+' finished!\\n')\n",
    "    return predict_proba_matrix,score_dict\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number 1, lr\n",
      "===========> fold1 ,done!\n",
      "===========> fold2 ,done!\n",
      "===========> fold3 ,done!\n",
      "===========> fold4 ,done!\n",
      "===========> fold5 ,done!\n",
      "number 1, lr finished!\n",
      "\n",
      "number 2, knn\n",
      "===========> fold1 ,done!\n",
      "===========> fold2 ,done!\n",
      "===========> fold3 ,done!\n",
      "===========> fold4 ,done!\n",
      "===========> fold5 ,done!\n",
      "number 2, knn finished!\n",
      "\n",
      "number 3, svm\n",
      "===========> fold1 ,done!\n",
      "===========> fold2 ,done!\n",
      "===========> fold3 ,done!\n",
      "===========> fold4 ,done!\n",
      "===========> fold5 ,done!\n",
      "number 3, svm finished!\n",
      "\n",
      "number 4, nb\n",
      "===========> fold1 ,done!\n",
      "===========> fold2 ,done!\n",
      "===========> fold3 ,done!\n",
      "===========> fold4 ,done!\n",
      "===========> fold5 ,done!\n",
      "number 4, nb finished!\n",
      "\n",
      "number 5, rf\n",
      "===========> fold1 ,done!\n",
      "===========> fold2 ,done!\n",
      "===========> fold3 ,done!\n",
      "===========> fold4 ,done!\n",
      "===========> fold5 ,done!\n",
      "number 5, rf finished!\n",
      "\n",
      "number 6, mlp\n",
      "===========> fold1 ,done!\n",
      "===========> fold2 ,done!\n",
      "===========> fold3 ,done!\n",
      "===========> fold4 ,done!\n",
      "===========> fold5 ,done!\n",
      "number 6, mlp finished!\n",
      "\n",
      "number 7, gbm\n",
      "===========> fold1 ,done!\n",
      "===========> fold2 ,done!\n",
      "===========> fold3 ,done!\n",
      "===========> fold4 ,done!\n",
      "===========> fold5 ,done!\n",
      "number 7, gbm finished!\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predict_proba_matrix,score_dict = get_predict_matrix(get_models())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from seaborn import diverging_palette"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "predict_proba_matrix['target'] = target\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 408,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_base,X_meta,y_base,y_meta = train_test_split(predict_proba_matrix[predict_proba_matrix.columns[:-1]],\n",
    "                                               predict_proba_matrix['target'],test_size = 0.2,\n",
    "                                               stratify = predict_proba_matrix['target'],random_state = 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 420,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-logloss:0.512753+0.0035705\ttest-logloss:0.535215+0.00603173\n",
      "[1]\ttrain-logloss:0.407731+0.00504009\ttest-logloss:0.448227+0.00950812\n",
      "[2]\ttrain-logloss:0.339729+0.00511722\ttest-logloss:0.396575+0.0113748\n",
      "[3]\ttrain-logloss:0.292959+0.00494696\ttest-logloss:0.360675+0.0136639\n",
      "[4]\ttrain-logloss:0.257447+0.00542595\ttest-logloss:0.338018+0.0163633\n",
      "[5]\ttrain-logloss:0.232045+0.00503845\ttest-logloss:0.323615+0.0188786\n",
      "[6]\ttrain-logloss:0.211402+0.00635307\ttest-logloss:0.314356+0.0200092\n",
      "[7]\ttrain-logloss:0.196832+0.00589052\ttest-logloss:0.308533+0.0216577\n",
      "[8]\ttrain-logloss:0.183173+0.00528433\ttest-logloss:0.305862+0.0233732\n",
      "[9]\ttrain-logloss:0.17178+0.00385619\ttest-logloss:0.304048+0.0256512\n",
      "[10]\ttrain-logloss:0.163757+0.0047561\ttest-logloss:0.303335+0.0268677\n",
      "[11]\ttrain-logloss:0.156889+0.00647785\ttest-logloss:0.303786+0.0275161\n",
      "[12]\ttrain-logloss:0.150807+0.00610447\ttest-logloss:0.303824+0.0281766\n",
      "[13]\ttrain-logloss:0.146395+0.00591307\ttest-logloss:0.303511+0.0307234\n",
      "[14]\ttrain-logloss:0.141743+0.00706078\ttest-logloss:0.304027+0.0313682\n",
      "[15]\ttrain-logloss:0.13898+0.00631255\ttest-logloss:0.303752+0.0314582\n",
      "[16]\ttrain-logloss:0.135142+0.00548561\ttest-logloss:0.304638+0.033428\n",
      "[17]\ttrain-logloss:0.130847+0.00682753\ttest-logloss:0.305136+0.0339056\n",
      "[18]\ttrain-logloss:0.128409+0.00665186\ttest-logloss:0.305004+0.03466\n",
      "[19]\ttrain-logloss:0.125762+0.00684988\ttest-logloss:0.30594+0.0354148\n",
      "[20]\ttrain-logloss:0.121028+0.00606334\ttest-logloss:0.306506+0.035765\n",
      "[21]\ttrain-logloss:0.116092+0.00446224\ttest-logloss:0.307127+0.0368766\n",
      "[22]\ttrain-logloss:0.111422+0.00574466\ttest-logloss:0.308348+0.0371045\n",
      "[23]\ttrain-logloss:0.108037+0.00677875\ttest-logloss:0.310333+0.0386846\n",
      "[24]\ttrain-logloss:0.10536+0.00750482\ttest-logloss:0.312085+0.0391677\n",
      "[25]\ttrain-logloss:0.103378+0.00668267\ttest-logloss:0.312186+0.0387594\n",
      "[26]\ttrain-logloss:0.099272+0.00456991\ttest-logloss:0.314217+0.0411711\n",
      "[27]\ttrain-logloss:0.0963994+0.0054089\ttest-logloss:0.314364+0.0419192\n",
      "[28]\ttrain-logloss:0.0930494+0.00511337\ttest-logloss:0.316268+0.0417459\n",
      "[29]\ttrain-logloss:0.0892004+0.00529842\ttest-logloss:0.318499+0.0428883\n",
      "[30]\ttrain-logloss:0.0875058+0.00484518\ttest-logloss:0.319376+0.0445143\n",
      "[31]\ttrain-logloss:0.0854646+0.00551676\ttest-logloss:0.320788+0.0462073\n",
      "[32]\ttrain-logloss:0.0836116+0.00528867\ttest-logloss:0.321769+0.0472464\n",
      "[33]\ttrain-logloss:0.0820118+0.0053785\ttest-logloss:0.323165+0.047693\n",
      "[34]\ttrain-logloss:0.0807306+0.0052809\ttest-logloss:0.324949+0.0499915\n",
      "[35]\ttrain-logloss:0.0777318+0.00606111\ttest-logloss:0.32764+0.0499308\n",
      "[36]\ttrain-logloss:0.0757798+0.00635358\ttest-logloss:0.32974+0.049677\n",
      "[37]\ttrain-logloss:0.0732976+0.00675893\ttest-logloss:0.331053+0.0494365\n",
      "[38]\ttrain-logloss:0.071235+0.00663197\ttest-logloss:0.331545+0.0492326\n",
      "[39]\ttrain-logloss:0.0694116+0.0059578\ttest-logloss:0.33222+0.0487001\n",
      "[40]\ttrain-logloss:0.0677804+0.00567234\ttest-logloss:0.333146+0.0500954\n",
      "[41]\ttrain-logloss:0.066174+0.00506899\ttest-logloss:0.333044+0.050028\n",
      "[42]\ttrain-logloss:0.0642808+0.00510403\ttest-logloss:0.335138+0.0493227\n",
      "[43]\ttrain-logloss:0.0630916+0.00541829\ttest-logloss:0.335988+0.0487752\n",
      "[44]\ttrain-logloss:0.061294+0.00548768\ttest-logloss:0.336649+0.0477969\n",
      "[45]\ttrain-logloss:0.0596878+0.00518223\ttest-logloss:0.338798+0.0480533\n",
      "[46]\ttrain-logloss:0.0585336+0.00530135\ttest-logloss:0.340146+0.050049\n",
      "[47]\ttrain-logloss:0.0570756+0.0051221\ttest-logloss:0.34119+0.0506255\n",
      "[48]\ttrain-logloss:0.055824+0.00469759\ttest-logloss:0.342038+0.0513019\n",
      "[49]\ttrain-logloss:0.0547338+0.00463605\ttest-logloss:0.342877+0.0517954\n",
      "[50]\ttrain-logloss:0.053742+0.004686\ttest-logloss:0.343201+0.0512605\n",
      "[51]\ttrain-logloss:0.0525906+0.00456569\ttest-logloss:0.34495+0.0514494\n",
      "[52]\ttrain-logloss:0.0516412+0.00440208\ttest-logloss:0.346401+0.0520559\n",
      "[53]\ttrain-logloss:0.0503218+0.00417\ttest-logloss:0.346874+0.0509581\n",
      "[54]\ttrain-logloss:0.0489054+0.00368638\ttest-logloss:0.348269+0.0510692\n",
      "[55]\ttrain-logloss:0.0474928+0.00357025\ttest-logloss:0.349638+0.052471\n",
      "[56]\ttrain-logloss:0.0460522+0.00331826\ttest-logloss:0.350717+0.0541196\n",
      "[57]\ttrain-logloss:0.0452244+0.00347075\ttest-logloss:0.352452+0.0563472\n",
      "[58]\ttrain-logloss:0.044355+0.00336705\ttest-logloss:0.35344+0.057934\n",
      "[59]\ttrain-logloss:0.043589+0.00320527\ttest-logloss:0.353655+0.0577924\n"
     ]
    }
   ],
   "source": [
    "dm_data = xgb.DMatrix(data = predict_proba_matrix[predict_proba_matrix.columns[:-1]],label = target)\n",
    "params = {'objective':'binary:logistic','n_estimators':200,'colsample_bytree':0.8}\n",
    "cv_results = xgb.cv(params = params,dtrain=dm_data,verbose_eval = True,num_boost_round=100,early_stopping_rounds=50,nfold = 5,stratified=True,metrics='logloss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.32701484098543204"
      ]
     },
     "execution_count": 417,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_loss(target,predict_proba_matrix[predict_proba_matrix.columns[:-1]].mean(axis = 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'lr': 4}"
      ]
     },
     "execution_count": 370,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def try_a_model(model,X_train,y_train,X_valid,y_valid):\n",
    "    train = model.fit(X_train,y_train)\n",
    "    train_acc = train.score(X_train,y_train)\n",
    "    train_loss = log_loss(y_train,train.predict_proba(X_train))\n",
    "    valid_acc = train.score(X_valid,y_valid)\n",
    "    valid_loss = log_loss(y_valid,train.predict_proba(X_valid))\n",
    "    print('Train Acc: {}'.format(train_acc))\n",
    "    print('Valid Acc: {}'.format(valid_acc))\n",
    "    print('Train Loss: {}'.format(train_loss))\n",
    "    print('Valid Loss: {}'.format(valid_loss))\n",
    "    return train_acc,valid_acc,train_loss,valid_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Acc: 0.8472330475448169\n",
      "Valid Acc: 0.7850467289719626\n",
      "Train Loss: 0.34870159122119554\n",
      "Valid Loss: 0.4028870357934383\n"
     ]
    }
   ],
   "source": [
    "train_acc,valid_acc,train_loss,valid_loss = try_a_model(knn,X_train_cv,y_train_cv,X_valid,y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cross_validation(model,X,y,cv = 5):\n",
    "    cv_score_loss = -np.mean(cross_val_score(model,X,y,scoring='neg_log_loss',cv = cv))\n",
    "    cv_score_acc = np.mean(cross_val_score(model,X,y,scoring='accuracy',cv = cv))\n",
    "    print('Loss cv score: {}'.format(cv_score_loss))\n",
    "    print('Acc cv score: {}'.format(cv_score_acc))\n",
    "    return cv_score_loss,cv_score_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss cv score: 0.46218617811203205\n",
      "Acc cv score: 0.7917612009249047\n"
     ]
    }
   ],
   "source": [
    "cv_score_loss,cv_score_acc = cross_validation(rf,X_whole,target,cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def grid_search(model,X,y,param_grid,cv = 5):\n",
    "    \n",
    "    gdsearch1 = GridSearchCV(model,param_grid=param_grid,scoring = 'neg_log_loss',cv = cv)\n",
    "    result1 = gdsearch1.fit(X,y)\n",
    "    print('best param for loss: {}'.format(result1.best_params_))\n",
    "    print('best score for loss: {}'.format(result1.best_score_))\n",
    "    gdsearch2 = GridSearchCV(model,param_grid=param_grid,scoring = 'accuracy',cv = cv)\n",
    "    result2 = gdsearch2.fit(X,y)\n",
    "    print('best param for loss: {}'.format(result2.best_params_))\n",
    "    print('best score for loss: {}'.format(result2.best_score_))\n",
    "    return result1,result2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best param for loss: {'max_features': 256, 'n_estimators': 40}\n",
      "best score for loss: -0.4627297026361883\n",
      "best param for loss: {'max_features': 128, 'n_estimators': 30}\n",
      "best score for loss: 0.7992518703241895\n"
     ]
    }
   ],
   "source": [
    "r1,r2 = grid_search(rf,X,target,param_grid = dict(n_estimators = list(range(10,60,10)),max_features = [128,256,512]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### gridsearch with preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "def pipelined_gridsearch(X,y,mlmodel,param_grid,scoring = 'accuracy',cv = 5):\n",
    "    first = MinMaxScaler()\n",
    "    normed_ml_search = Pipeline(steps=[('first',first),('mlmodel',mlmodel)])\n",
    "    grid = GridSearchCV(normed_ml_search,param_grid=param_grid,scoring = scoring,cv = cv)\n",
    "    grid.fit(X,y)\n",
    "    best_param = grid.best_params_\n",
    "    best_score = grid.best_score_\n",
    "    print('Best param: {}'.format(best_param))\n",
    "    print('Best score: {}'.format(best_score))\n",
    "    return best_param,best_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best param: {'mlmodel__C': 1.1}\n",
      "Best score: 0.837281795511222\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "({'mlmodel__C': 1.1}, 0.83728179551122195)"
      ]
     },
     "execution_count": 343,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr = LogisticRegression()\n",
    "pipelined_gridsearch(X_whole,target,lr,param_grid={'mlmodel__C':[1+0.1*i for i in range(1,11)]},cv = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best param: {'mlmodel__n_neighbors': 9}\n",
      "Best score: 0.7743142144638404\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "({'mlmodel__n_neighbors': 9}, 0.77431421446384041)"
      ]
     },
     "execution_count": 381,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn = KNeighborsClassifier()\n",
    "pipelined_gridsearch(X_whole,target,knn,param_grid={'mlmodel__n_neighbors':list(range(1,25,4))},cv = 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def ml_kfolds_evaluate(mlmodel,X,y,K=5,normalization = True):\n",
    "    folds = list(StratifiedKFold(n_splits=K, shuffle=True).split(X,y))\n",
    "    y_valid_pred_log = 0.0*y\n",
    "    for j, (train_idx, valid_idx) in enumerate(folds):\n",
    "        print('\\n===================FOLD=',j+1)\n",
    "        X_train_cv = np.array(X)[train_idx]\n",
    "        y_train_cv = y[train_idx]\n",
    "        X_valid = np.array(X)[valid_idx]\n",
    "        y_valid= y[valid_idx]\n",
    "        if normalization==True:\n",
    "            first = MinMaxScaler()\n",
    "            normed_ml_search = Pipeline(steps=[('first',first),('mlmodel',mlmodel)]).fit(X_train_cv,y_train_cv)\n",
    "        else:\n",
    "            normed_ml_search = mlmodel.fit(X_train_cv,y_train_cv)\n",
    "        preds = normed_ml_search.predict(X_valid)\n",
    "        y_valid_pred_log[valid_idx] = preds\n",
    "    return y_valid_pred_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================FOLD= 1\n",
      "\n",
      "===================FOLD= 2"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\sklearn\\utils\\validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\sklearn\\utils\\validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================FOLD= 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\sklearn\\utils\\validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================FOLD= 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\sklearn\\utils\\validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================FOLD= 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\sklearn\\utils\\validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.84413965087281795"
      ]
     },
     "execution_count": 379,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr = LogisticRegression(C=1)\n",
    "preds = ml_kfolds_evaluate(lr,X_whole,target,K=5,normalization=True)\n",
    "accuracy_score(target,preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### xgboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.84423676012461057"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "xgbclf = xgb.XGBClassifier(n_estimators = 200,colsample_bytree=0.8)\n",
    "xgbclf.fit(X_train_cv,y_train_cv)\n",
    "preds = xgbclf.predict(X_valid)\n",
    "xgbclf.score(X_train_cv,y_train_cv)\n",
    "xgbclf.score(X_valid,y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "t1 = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.167266368865967"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t2 = time.time()\n",
    "t2-t1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DM_train = xgb.DMatrix(X_train_cv,y_train_cv)\n",
    "DM_valid = xgb.DMatrix(X_valid,y_valid)\n",
    "t1 = time.time()\n",
    "params = {'objective':'binary:logistic','alpha':1,'tree_method':'gpu_hist'}\n",
    "xgbclf = xgb.train(params = params,dtrain = DM_train,num_boost_round=100,evals = [(DM_train,'train'),(DM_valid,'valid')],early_stopping_rounds=100)\n",
    "t2 = time.time()\n",
    "t2-t1\n",
    "#xgbclf.predict(DM_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-logloss:0.550286+0.00206975\ttest-logloss:0.586198+0.00890863\n",
      "[1]\ttrain-logloss:0.470306+0.00958932\ttest-logloss:0.530621+0.0147537\n",
      "[2]\ttrain-logloss:0.40707+0.00296914\ttest-logloss:0.49179+0.0104228\n",
      "[3]\ttrain-logloss:0.351958+0.0070862\ttest-logloss:0.465135+0.00752298\n",
      "[4]\ttrain-logloss:0.312793+0.00864052\ttest-logloss:0.442968+0.0087691\n",
      "[5]\ttrain-logloss:0.279917+0.0076576\ttest-logloss:0.426425+0.00709965\n",
      "[6]\ttrain-logloss:0.254217+0.00845198\ttest-logloss:0.413488+0.00684292\n",
      "[7]\ttrain-logloss:0.232065+0.00947995\ttest-logloss:0.402179+0.00766479\n",
      "[8]\ttrain-logloss:0.212765+0.00987895\ttest-logloss:0.393354+0.00817723\n",
      "[9]\ttrain-logloss:0.19542+0.00722489\ttest-logloss:0.384648+0.00819415\n",
      "[10]\ttrain-logloss:0.176595+0.0048563\ttest-logloss:0.381758+0.00888178\n",
      "[11]\ttrain-logloss:0.162076+0.00497059\ttest-logloss:0.378541+0.00773131\n",
      "[12]\ttrain-logloss:0.149078+0.00638988\ttest-logloss:0.373771+0.0110737\n",
      "[13]\ttrain-logloss:0.135587+0.00562881\ttest-logloss:0.368797+0.0128146\n",
      "[14]\ttrain-logloss:0.125637+0.00561703\ttest-logloss:0.365943+0.016452\n",
      "[15]\ttrain-logloss:0.114937+0.00531465\ttest-logloss:0.361229+0.0155064\n",
      "[16]\ttrain-logloss:0.106603+0.00524422\ttest-logloss:0.360909+0.0149014\n",
      "[17]\ttrain-logloss:0.0987812+0.00450467\ttest-logloss:0.360811+0.0155593\n",
      "[18]\ttrain-logloss:0.0919362+0.00449189\ttest-logloss:0.359301+0.0188363\n",
      "[19]\ttrain-logloss:0.086129+0.00410915\ttest-logloss:0.358926+0.0178405\n",
      "[20]\ttrain-logloss:0.0796336+0.00335109\ttest-logloss:0.357568+0.0176213\n",
      "[21]\ttrain-logloss:0.0739234+0.00316352\ttest-logloss:0.355402+0.0176634\n",
      "[22]\ttrain-logloss:0.0693182+0.00360962\ttest-logloss:0.352717+0.0183559\n",
      "[23]\ttrain-logloss:0.0643468+0.00330346\ttest-logloss:0.351321+0.0173511\n",
      "[24]\ttrain-logloss:0.0608798+0.00309187\ttest-logloss:0.350959+0.0185566\n",
      "[25]\ttrain-logloss:0.0574808+0.003433\ttest-logloss:0.349632+0.0195056\n",
      "[26]\ttrain-logloss:0.0535572+0.00300986\ttest-logloss:0.349429+0.0198071\n",
      "[27]\ttrain-logloss:0.0503034+0.00292963\ttest-logloss:0.348299+0.0202172\n",
      "[28]\ttrain-logloss:0.0472784+0.00269655\ttest-logloss:0.349124+0.0211353\n",
      "[29]\ttrain-logloss:0.0448658+0.00266175\ttest-logloss:0.349615+0.0217596\n",
      "[30]\ttrain-logloss:0.042429+0.00246805\ttest-logloss:0.349474+0.0228652\n",
      "[31]\ttrain-logloss:0.04029+0.00223234\ttest-logloss:0.351116+0.0234197\n"
     ]
    }
   ],
   "source": [
    "dm_data = xgb.DMatrix(data = X_whole,label = target)\n",
    "params = {'objective':'binary:logistic','n_estimators':200,'colsample_bytree':0.5}\n",
    "cv_results = xgb.cv(params = params,dtrain=dm_data,verbose_eval = True,num_boost_round=100,early_stopping_rounds=5,nfold = 5,stratified=True,metrics='logloss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.34829900000000003"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_results['test-logloss-mean'].iloc[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test-logloss-mean</th>\n",
       "      <th>test-logloss-std</th>\n",
       "      <th>train-logloss-mean</th>\n",
       "      <th>train-logloss-std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.586198</td>\n",
       "      <td>0.008909</td>\n",
       "      <td>0.550286</td>\n",
       "      <td>0.002070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.530621</td>\n",
       "      <td>0.014754</td>\n",
       "      <td>0.470306</td>\n",
       "      <td>0.009589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.491790</td>\n",
       "      <td>0.010423</td>\n",
       "      <td>0.407070</td>\n",
       "      <td>0.002969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.465135</td>\n",
       "      <td>0.007523</td>\n",
       "      <td>0.351958</td>\n",
       "      <td>0.007086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.442968</td>\n",
       "      <td>0.008769</td>\n",
       "      <td>0.312793</td>\n",
       "      <td>0.008641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.426425</td>\n",
       "      <td>0.007100</td>\n",
       "      <td>0.279917</td>\n",
       "      <td>0.007658</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.413488</td>\n",
       "      <td>0.006843</td>\n",
       "      <td>0.254217</td>\n",
       "      <td>0.008452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.402179</td>\n",
       "      <td>0.007665</td>\n",
       "      <td>0.232065</td>\n",
       "      <td>0.009480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.393354</td>\n",
       "      <td>0.008177</td>\n",
       "      <td>0.212765</td>\n",
       "      <td>0.009879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.384648</td>\n",
       "      <td>0.008194</td>\n",
       "      <td>0.195420</td>\n",
       "      <td>0.007225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.381758</td>\n",
       "      <td>0.008882</td>\n",
       "      <td>0.176595</td>\n",
       "      <td>0.004856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.378541</td>\n",
       "      <td>0.007731</td>\n",
       "      <td>0.162076</td>\n",
       "      <td>0.004971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.373771</td>\n",
       "      <td>0.011074</td>\n",
       "      <td>0.149078</td>\n",
       "      <td>0.006390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.368797</td>\n",
       "      <td>0.012815</td>\n",
       "      <td>0.135587</td>\n",
       "      <td>0.005629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.365943</td>\n",
       "      <td>0.016452</td>\n",
       "      <td>0.125637</td>\n",
       "      <td>0.005617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.361229</td>\n",
       "      <td>0.015506</td>\n",
       "      <td>0.114937</td>\n",
       "      <td>0.005315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.360909</td>\n",
       "      <td>0.014901</td>\n",
       "      <td>0.106603</td>\n",
       "      <td>0.005244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.360811</td>\n",
       "      <td>0.015559</td>\n",
       "      <td>0.098781</td>\n",
       "      <td>0.004505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.359301</td>\n",
       "      <td>0.018836</td>\n",
       "      <td>0.091936</td>\n",
       "      <td>0.004492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.358926</td>\n",
       "      <td>0.017841</td>\n",
       "      <td>0.086129</td>\n",
       "      <td>0.004109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.357568</td>\n",
       "      <td>0.017621</td>\n",
       "      <td>0.079634</td>\n",
       "      <td>0.003351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.355402</td>\n",
       "      <td>0.017663</td>\n",
       "      <td>0.073923</td>\n",
       "      <td>0.003164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.352717</td>\n",
       "      <td>0.018356</td>\n",
       "      <td>0.069318</td>\n",
       "      <td>0.003610</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.351321</td>\n",
       "      <td>0.017351</td>\n",
       "      <td>0.064347</td>\n",
       "      <td>0.003303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.350959</td>\n",
       "      <td>0.018557</td>\n",
       "      <td>0.060880</td>\n",
       "      <td>0.003092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.349632</td>\n",
       "      <td>0.019506</td>\n",
       "      <td>0.057481</td>\n",
       "      <td>0.003433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.349429</td>\n",
       "      <td>0.019807</td>\n",
       "      <td>0.053557</td>\n",
       "      <td>0.003010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0.348299</td>\n",
       "      <td>0.020217</td>\n",
       "      <td>0.050303</td>\n",
       "      <td>0.002930</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    test-logloss-mean  test-logloss-std  train-logloss-mean  train-logloss-std\n",
       "0            0.586198          0.008909            0.550286           0.002070\n",
       "1            0.530621          0.014754            0.470306           0.009589\n",
       "2            0.491790          0.010423            0.407070           0.002969\n",
       "3            0.465135          0.007523            0.351958           0.007086\n",
       "4            0.442968          0.008769            0.312793           0.008641\n",
       "5            0.426425          0.007100            0.279917           0.007658\n",
       "6            0.413488          0.006843            0.254217           0.008452\n",
       "7            0.402179          0.007665            0.232065           0.009480\n",
       "8            0.393354          0.008177            0.212765           0.009879\n",
       "9            0.384648          0.008194            0.195420           0.007225\n",
       "10           0.381758          0.008882            0.176595           0.004856\n",
       "11           0.378541          0.007731            0.162076           0.004971\n",
       "12           0.373771          0.011074            0.149078           0.006390\n",
       "13           0.368797          0.012815            0.135587           0.005629\n",
       "14           0.365943          0.016452            0.125637           0.005617\n",
       "15           0.361229          0.015506            0.114937           0.005315\n",
       "16           0.360909          0.014901            0.106603           0.005244\n",
       "17           0.360811          0.015559            0.098781           0.004505\n",
       "18           0.359301          0.018836            0.091936           0.004492\n",
       "19           0.358926          0.017841            0.086129           0.004109\n",
       "20           0.357568          0.017621            0.079634           0.003351\n",
       "21           0.355402          0.017663            0.073923           0.003164\n",
       "22           0.352717          0.018356            0.069318           0.003610\n",
       "23           0.351321          0.017351            0.064347           0.003303\n",
       "24           0.350959          0.018557            0.060880           0.003092\n",
       "25           0.349632          0.019506            0.057481           0.003433\n",
       "26           0.349429          0.019807            0.053557           0.003010\n",
       "27           0.348299          0.020217            0.050303           0.002930"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### xgboost grid search 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 6 candidates, totalling 30 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=4)]: Done   5 tasks      | elapsed:   37.9s\n",
      "[Parallel(n_jobs=4)]: Done  10 tasks      | elapsed:  1.0min\n",
      "[Parallel(n_jobs=4)]: Done  17 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=4)]: Done  27 out of  30 | elapsed:  2.4min remaining:   15.9s\n",
      "[Parallel(n_jobs=4)]: Done  30 out of  30 | elapsed:  2.6min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=3, min_child_weight=1, missing=None, n_estimators=100,\n",
       "       n_jobs=1, nthread=None, objective='binary:logistic', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "       silent=True, subsample=1),\n",
       "       fit_params=None, iid=True, n_jobs=4,\n",
       "       param_grid={'learning_rate': [0.25], 'max_depth': [6], 'n_estimators': [130], 'subsample': [1], 'colsample_bytree': [1], 'gamma': [0, 2, 4, 6, 8, 10]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='accuracy', verbose=10)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dm_data = xgb.DMatrix(data = X_whole,label = target)\n",
    "gbm = xgb.XGBClassifier()\n",
    "gs_params1 = {'learning_rate':[0.25],'max_depth':[6],'n_estimators':[130],'subsample':[1],'colsample_bytree':[1],'gamma':[0,2,4,6,8,10]}\n",
    "gs1 = GridSearchCV(gbm,param_grid=gs_params1,scoring = 'accuracy',cv = 5,verbose = 10,n_jobs=4)\n",
    "gs1.fit(X_whole,target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': 1,\n",
       " 'gamma': 0,\n",
       " 'learning_rate': 0.25,\n",
       " 'max_depth': 6,\n",
       " 'n_estimators': 130,\n",
       " 'subsample': 1}"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs1.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8559850374064838"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs1.cv_results_['mean_test_score'][np.argmax(gs1.cv_results_['mean_test_score'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\yltbe\\Anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs1.cv_results_['mean_train_score'][np.argmax(gs1.cv_results_['mean_test_score'])]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### xgb Random Search 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def random_search(num = 10):\n",
    "    \n",
    "    dm_data = xgb.DMatrix(data = X_whole,label = target)\n",
    "    gbm2 = xgb.XGBClassifier()\n",
    "    gs_params2 = {'learning_rate':[0.05,0.1,0.2],\n",
    "                  'max_depth':[6,7],\n",
    "                  'n_estimators':[130,150,170],\n",
    "                  'subsample':[1,0.9,0.8],\n",
    "                  'colsample_bytree':[0.9,0.8,0.7,0.6],\n",
    "                  'gamma':[0,1,2],\n",
    "                  'min_child_weight':[1,2,3]}\n",
    "    gs2 = RandomizedSearchCV(gbm2,param_distributions=gs_params2,n_iter=40,scoring = 'accuracy',cv = StratifiedKFold(n_splits = 5,shuffle = True,random_state = 2),verbose = 10,n_jobs=4)\n",
    "    all_scores = []\n",
    "    all_params = pd.DataFrame({'colsample_bytree': [],\n",
    "                  'gamma': [],\n",
    "                  'learning_rate': [],\n",
    "                  'max_depth': [],\n",
    "                  'min_child_weight': [],\n",
    "                  'n_estimators': [],\n",
    "                  'subsample': []})\n",
    "    for i in range(num):\n",
    "        gs2.fit(X_whole,target)\n",
    "        best_param = gs2.best_params_\n",
    "        best_score = gs2.best_score_\n",
    "        all_scores.append(best_score)\n",
    "        cur_best_param= pd.DataFrame({key:[value] for key,value in best_param.items()})\n",
    "        all_params = pd.concat([all_params,cur_best_param],axis = 0)\n",
    "        print('num_{},done!'.format(i+1))\n",
    "    all_params['acc'] = all_scores\n",
    "    return all_params\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_params4 = random_search(num = 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_params3 = random_search(num = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xgb_param = all_params4.iloc[1].to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8572319201995012"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_param.pop('acc',None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xgb_param = {'colsample_bytree': 0.9,\n",
    "             'gamma': 0.0,\n",
    "             'learning_rate': 0.1,\n",
    "             'max_depth': 6,\n",
    "             'min_child_weight': 2.0,\n",
    "             'subsample': 1.0,'objective':'binary:logistic'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-error:0.266563\tvalid-error:0.286604\n",
      "[1]\ttrain-error:0.254092\tvalid-error:0.267913\n",
      "[2]\ttrain-error:0.217459\tvalid-error:0.221184\n",
      "[3]\ttrain-error:0.199532\tvalid-error:0.211838\n",
      "[4]\ttrain-error:0.198753\tvalid-error:0.190031\n",
      "[5]\ttrain-error:0.190959\tvalid-error:0.186916\n",
      "[6]\ttrain-error:0.195635\tvalid-error:0.183801\n",
      "[7]\ttrain-error:0.195635\tvalid-error:0.193146\n",
      "[8]\ttrain-error:0.191738\tvalid-error:0.196262\n",
      "[9]\ttrain-error:0.173811\tvalid-error:0.174455\n",
      "[10]\ttrain-error:0.172253\tvalid-error:0.183801\n",
      "[11]\ttrain-error:0.166797\tvalid-error:0.174455\n",
      "[12]\ttrain-error:0.16212\tvalid-error:0.158879\n",
      "[13]\ttrain-error:0.151208\tvalid-error:0.17134\n",
      "[14]\ttrain-error:0.14887\tvalid-error:0.161994\n",
      "[15]\ttrain-error:0.146532\tvalid-error:0.168224\n",
      "[16]\ttrain-error:0.141076\tvalid-error:0.168224\n",
      "[17]\ttrain-error:0.139517\tvalid-error:0.158879\n",
      "[18]\ttrain-error:0.138737\tvalid-error:0.155763\n",
      "[19]\ttrain-error:0.141855\tvalid-error:0.152648\n",
      "[20]\ttrain-error:0.13562\tvalid-error:0.158879\n",
      "[21]\ttrain-error:0.140296\tvalid-error:0.155763\n",
      "[22]\ttrain-error:0.137178\tvalid-error:0.158879\n",
      "[23]\ttrain-error:0.132502\tvalid-error:0.158879\n",
      "[24]\ttrain-error:0.124708\tvalid-error:0.158879\n",
      "[25]\ttrain-error:0.124708\tvalid-error:0.152648\n",
      "[26]\ttrain-error:0.12159\tvalid-error:0.155763\n",
      "[27]\ttrain-error:0.12159\tvalid-error:0.152648\n",
      "[28]\ttrain-error:0.124708\tvalid-error:0.158879\n",
      "[29]\ttrain-error:0.117693\tvalid-error:0.146417\n",
      "[30]\ttrain-error:0.109119\tvalid-error:0.137072\n",
      "[31]\ttrain-error:0.10756\tvalid-error:0.133956\n",
      "[32]\ttrain-error:0.09431\tvalid-error:0.130841\n",
      "[33]\ttrain-error:0.091972\tvalid-error:0.137072\n",
      "[34]\ttrain-error:0.099766\tvalid-error:0.140187\n",
      "[35]\ttrain-error:0.091193\tvalid-error:0.140187\n",
      "[36]\ttrain-error:0.09509\tvalid-error:0.143302\n",
      "[37]\ttrain-error:0.090413\tvalid-error:0.143302\n",
      "[38]\ttrain-error:0.091193\tvalid-error:0.140187\n",
      "[39]\ttrain-error:0.091972\tvalid-error:0.137072\n",
      "[40]\ttrain-error:0.085737\tvalid-error:0.133956\n",
      "[41]\ttrain-error:0.086516\tvalid-error:0.130841\n",
      "[42]\ttrain-error:0.084957\tvalid-error:0.127726\n",
      "[43]\ttrain-error:0.08106\tvalid-error:0.133956\n",
      "[44]\ttrain-error:0.080281\tvalid-error:0.130841\n",
      "[45]\ttrain-error:0.078722\tvalid-error:0.124611\n",
      "[46]\ttrain-error:0.072486\tvalid-error:0.124611\n",
      "[47]\ttrain-error:0.069369\tvalid-error:0.124611\n",
      "[48]\ttrain-error:0.072486\tvalid-error:0.127726\n",
      "[49]\ttrain-error:0.072486\tvalid-error:0.127726\n",
      "[50]\ttrain-error:0.069369\tvalid-error:0.130841\n",
      "[51]\ttrain-error:0.064692\tvalid-error:0.127726\n",
      "[52]\ttrain-error:0.064692\tvalid-error:0.124611\n",
      "[53]\ttrain-error:0.066251\tvalid-error:0.11838\n",
      "[54]\ttrain-error:0.069369\tvalid-error:0.11838\n",
      "[55]\ttrain-error:0.06703\tvalid-error:0.11838\n",
      "[56]\ttrain-error:0.064692\tvalid-error:0.11215\n",
      "[57]\ttrain-error:0.064692\tvalid-error:0.115265\n",
      "[58]\ttrain-error:0.066251\tvalid-error:0.121495\n",
      "[59]\ttrain-error:0.063913\tvalid-error:0.127726\n",
      "[60]\ttrain-error:0.062354\tvalid-error:0.124611\n",
      "[61]\ttrain-error:0.063133\tvalid-error:0.121495\n",
      "[62]\ttrain-error:0.062354\tvalid-error:0.121495\n",
      "[63]\ttrain-error:0.060795\tvalid-error:0.127726\n",
      "[64]\ttrain-error:0.060795\tvalid-error:0.127726\n",
      "[65]\ttrain-error:0.058457\tvalid-error:0.127726\n",
      "[66]\ttrain-error:0.057677\tvalid-error:0.124611\n",
      "[67]\ttrain-error:0.05456\tvalid-error:0.127726\n",
      "[68]\ttrain-error:0.05456\tvalid-error:0.127726\n",
      "[69]\ttrain-error:0.055339\tvalid-error:0.127726\n",
      "[70]\ttrain-error:0.053001\tvalid-error:0.127726\n",
      "[71]\ttrain-error:0.049883\tvalid-error:0.124611\n",
      "[72]\ttrain-error:0.049104\tvalid-error:0.121495\n",
      "[73]\ttrain-error:0.049104\tvalid-error:0.11838\n",
      "[74]\ttrain-error:0.047545\tvalid-error:0.124611\n",
      "[75]\ttrain-error:0.048324\tvalid-error:0.11838\n",
      "[76]\ttrain-error:0.046765\tvalid-error:0.11838\n",
      "[77]\ttrain-error:0.046765\tvalid-error:0.11838\n",
      "[78]\ttrain-error:0.044427\tvalid-error:0.11838\n",
      "[79]\ttrain-error:0.045986\tvalid-error:0.124611\n",
      "[80]\ttrain-error:0.045986\tvalid-error:0.124611\n",
      "[81]\ttrain-error:0.045986\tvalid-error:0.121495\n",
      "[82]\ttrain-error:0.041309\tvalid-error:0.124611\n",
      "[83]\ttrain-error:0.042089\tvalid-error:0.124611\n",
      "[84]\ttrain-error:0.041309\tvalid-error:0.127726\n",
      "[85]\ttrain-error:0.037412\tvalid-error:0.121495\n",
      "[86]\ttrain-error:0.037412\tvalid-error:0.124611\n",
      "[87]\ttrain-error:0.037412\tvalid-error:0.124611\n",
      "[88]\ttrain-error:0.035853\tvalid-error:0.127726\n",
      "[89]\ttrain-error:0.036633\tvalid-error:0.127726\n",
      "[90]\ttrain-error:0.035853\tvalid-error:0.127726\n",
      "[91]\ttrain-error:0.036633\tvalid-error:0.124611\n",
      "[92]\ttrain-error:0.031956\tvalid-error:0.124611\n",
      "[93]\ttrain-error:0.031956\tvalid-error:0.11838\n",
      "[94]\ttrain-error:0.031956\tvalid-error:0.11838\n",
      "[95]\ttrain-error:0.033515\tvalid-error:0.124611\n",
      "[96]\ttrain-error:0.032736\tvalid-error:0.11838\n",
      "[97]\ttrain-error:0.031177\tvalid-error:0.11838\n",
      "[98]\ttrain-error:0.030398\tvalid-error:0.115265\n",
      "[99]\ttrain-error:0.030398\tvalid-error:0.115265\n",
      "[100]\ttrain-error:0.028839\tvalid-error:0.121495\n",
      "[101]\ttrain-error:0.028059\tvalid-error:0.121495\n",
      "[102]\ttrain-error:0.02728\tvalid-error:0.121495\n",
      "[103]\ttrain-error:0.025721\tvalid-error:0.11838\n",
      "[104]\ttrain-error:0.0265\tvalid-error:0.121495\n",
      "[105]\ttrain-error:0.025721\tvalid-error:0.121495\n",
      "[106]\ttrain-error:0.024942\tvalid-error:0.127726\n",
      "[107]\ttrain-error:0.025721\tvalid-error:0.124611\n",
      "[108]\ttrain-error:0.024942\tvalid-error:0.124611\n",
      "[109]\ttrain-error:0.025721\tvalid-error:0.115265\n",
      "[110]\ttrain-error:0.024942\tvalid-error:0.115265\n",
      "[111]\ttrain-error:0.025721\tvalid-error:0.11215\n",
      "[112]\ttrain-error:0.024162\tvalid-error:0.11215\n",
      "[113]\ttrain-error:0.022603\tvalid-error:0.11838\n",
      "[114]\ttrain-error:0.021044\tvalid-error:0.115265\n",
      "[115]\ttrain-error:0.021044\tvalid-error:0.11838\n",
      "[116]\ttrain-error:0.022603\tvalid-error:0.121495\n",
      "[117]\ttrain-error:0.021044\tvalid-error:0.11838\n",
      "[118]\ttrain-error:0.018706\tvalid-error:0.121495\n",
      "[119]\ttrain-error:0.021044\tvalid-error:0.11215\n",
      "[120]\ttrain-error:0.019486\tvalid-error:0.115265\n",
      "[121]\ttrain-error:0.019486\tvalid-error:0.115265\n",
      "[122]\ttrain-error:0.018706\tvalid-error:0.11838\n",
      "[123]\ttrain-error:0.018706\tvalid-error:0.115265\n",
      "[124]\ttrain-error:0.018706\tvalid-error:0.11838\n"
     ]
    }
   ],
   "source": [
    "X_tra,X_val,y_tra,y_val = train_test_split(X_whole,target)\n",
    "dmtra = xgb.DMatrix(data = X_tra,label = y_tra)\n",
    "dmval = xgb.DMatrix(data = X_val,label = y_val)\n",
    "xgb_param = {'colsample_bytree': 0.8,\n",
    "                     'gamma': 0,\n",
    "                     'learning_rate': 0.1,\n",
    "                     'max_depth': 3,\n",
    "                     'min_child_weight': 2.0,\n",
    "                     'subsample': 0.8,\n",
    "                     'objective':'binary:logistic',\n",
    "                     'booster':'gbtree','save_period':2,'model_dir':'xgbmodels'}\n",
    "xgbmodel = xgb.train(params = xgb_param,\n",
    "                  dtrain = dmtrain,\n",
    "                  evals = [(dmtrain,'train'),(dmvalid,'valid')],\n",
    "                  num_boost_round=125,\n",
    "                  verbose_eval=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.85785580000000006"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1-xgbcv['test-error-mean'].iloc[-1]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "xgbmodel.load_model('xgbmodels/1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def xgb_cv5_ensemble():\n",
    "    folds = list(StratifiedKFold(n_splits=5,shuffle=True,random_state=2).split(X_whole,target))\n",
    "    all_data_valid_score = target*0\n",
    "    a_test = target*0\n",
    "    for i,(train_mask,valid_mask) in enumerate(folds):\n",
    "        X_train = X_whole.iloc[train_mask]\n",
    "        y_train = target.iloc[train_mask]\n",
    "        X_valid = X_whole.iloc[valid_mask]\n",
    "        y_valid = target.iloc[valid_mask]\n",
    "        dmtrain = xgb.DMatrix(data = X_train,label = y_train)\n",
    "        dmvalid = xgb.DMatrix(data = X_valid,label = y_valid)\n",
    "        xgb_param = {'colsample_bytree': 0.8,\n",
    "                     'gamma': 0,\n",
    "                     'learning_rate': 0.1,\n",
    "                     'max_depth': 3,\n",
    "                     'min_child_weight': 2.0,\n",
    "                     'subsample': 0.8,\n",
    "                     'objective':'binary:logistic',\n",
    "                     'booster':'gbtree','save_period':2,'model_dir':'xgbmodels'}\n",
    "        xgbmodel = xgb.train(params = xgb_param,\n",
    "                          dtrain = dmtrain,\n",
    "                          evals = [(dmtrain,'train'),(dmvalid,'valid')],\n",
    "                          num_boost_round=125,\n",
    "                          verbose_eval=True)\n",
    "        xgbmodel.save_model('xgbmodels/'+str(i+1))\n",
    "        temp_valid = xgbmodel.predict(dmvalid)\n",
    "        all_data_valid_score.iloc[valid_mask] = temp_valid\n",
    "        test = xgb.Booster(model_file = 'xgbmodels/'+str(i+1))\n",
    "        temp2 = test.predict(dmvalid)\n",
    "        a_test.iloc[valid_mask] = temp2\n",
    "        print('Fold'+str(i+1)+',Done!')\n",
    "    all_cv_valid_score = accuracy_score(target,1*(all_data_valid_score>0.5))\n",
    "    just_a_test = accuracy_score(target,1*(a_test>0.5))\n",
    "    print('all_cv_valid_score')\n",
    "    return all_cv_valid_score,just_a_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-error:0.226989\tvalid-error:0.267081\n",
      "[1]\ttrain-error:0.229329\tvalid-error:0.267081\n",
      "[2]\ttrain-error:0.225429\tvalid-error:0.257764\n",
      "[3]\ttrain-error:0.180967\tvalid-error:0.226708\n",
      "[4]\ttrain-error:0.178627\tvalid-error:0.232919\n",
      "[5]\ttrain-error:0.179407\tvalid-error:0.242236\n",
      "[6]\ttrain-error:0.173167\tvalid-error:0.242236\n",
      "[7]\ttrain-error:0.170827\tvalid-error:0.23913\n",
      "[8]\ttrain-error:0.176287\tvalid-error:0.242236\n",
      "[9]\ttrain-error:0.160686\tvalid-error:0.23913\n",
      "[10]\ttrain-error:0.148986\tvalid-error:0.21118\n",
      "[11]\ttrain-error:0.148986\tvalid-error:0.214286\n",
      "[12]\ttrain-error:0.144306\tvalid-error:0.217391\n",
      "[13]\ttrain-error:0.139626\tvalid-error:0.220497\n",
      "[14]\ttrain-error:0.136505\tvalid-error:0.214286\n",
      "[15]\ttrain-error:0.134165\tvalid-error:0.217391\n",
      "[16]\ttrain-error:0.133385\tvalid-error:0.217391\n",
      "[17]\ttrain-error:0.131825\tvalid-error:0.226708\n",
      "[18]\ttrain-error:0.127925\tvalid-error:0.220497\n",
      "[19]\ttrain-error:0.124805\tvalid-error:0.214286\n",
      "[20]\ttrain-error:0.119345\tvalid-error:0.214286\n",
      "[21]\ttrain-error:0.116225\tvalid-error:0.220497\n",
      "[22]\ttrain-error:0.113885\tvalid-error:0.21118\n",
      "[23]\ttrain-error:0.113885\tvalid-error:0.198758\n",
      "[24]\ttrain-error:0.107644\tvalid-error:0.21118\n",
      "[25]\ttrain-error:0.108424\tvalid-error:0.201863\n",
      "[26]\ttrain-error:0.107644\tvalid-error:0.214286\n",
      "[27]\ttrain-error:0.104524\tvalid-error:0.198758\n",
      "[28]\ttrain-error:0.098284\tvalid-error:0.198758\n",
      "[29]\ttrain-error:0.092824\tvalid-error:0.201863\n",
      "[30]\ttrain-error:0.095164\tvalid-error:0.204969\n",
      "[31]\ttrain-error:0.092824\tvalid-error:0.21118\n",
      "[32]\ttrain-error:0.092824\tvalid-error:0.192547\n",
      "[33]\ttrain-error:0.090484\tvalid-error:0.198758\n",
      "[34]\ttrain-error:0.093604\tvalid-error:0.208075\n",
      "[35]\ttrain-error:0.090484\tvalid-error:0.204969\n",
      "[36]\ttrain-error:0.087363\tvalid-error:0.198758\n",
      "[37]\ttrain-error:0.083463\tvalid-error:0.192547\n",
      "[38]\ttrain-error:0.081123\tvalid-error:0.192547\n",
      "[39]\ttrain-error:0.085023\tvalid-error:0.18323\n",
      "[40]\ttrain-error:0.083463\tvalid-error:0.177019\n",
      "[41]\ttrain-error:0.077223\tvalid-error:0.189441\n",
      "[42]\ttrain-error:0.078003\tvalid-error:0.186335\n",
      "[43]\ttrain-error:0.074883\tvalid-error:0.180124\n",
      "[44]\ttrain-error:0.072543\tvalid-error:0.177019\n",
      "[45]\ttrain-error:0.072543\tvalid-error:0.173913\n",
      "[46]\ttrain-error:0.072543\tvalid-error:0.180124\n",
      "[47]\ttrain-error:0.071763\tvalid-error:0.177019\n",
      "[48]\ttrain-error:0.069423\tvalid-error:0.177019\n",
      "[49]\ttrain-error:0.069423\tvalid-error:0.180124\n",
      "[50]\ttrain-error:0.063963\tvalid-error:0.177019\n",
      "[51]\ttrain-error:0.067083\tvalid-error:0.167702\n",
      "[52]\ttrain-error:0.063963\tvalid-error:0.167702\n",
      "[53]\ttrain-error:0.060062\tvalid-error:0.164596\n",
      "[54]\ttrain-error:0.062402\tvalid-error:0.164596\n",
      "[55]\ttrain-error:0.062402\tvalid-error:0.161491\n",
      "[56]\ttrain-error:0.058502\tvalid-error:0.167702\n",
      "[57]\ttrain-error:0.051482\tvalid-error:0.170807\n",
      "[58]\ttrain-error:0.054602\tvalid-error:0.167702\n",
      "[59]\ttrain-error:0.053042\tvalid-error:0.167702\n",
      "[60]\ttrain-error:0.052262\tvalid-error:0.164596\n",
      "[61]\ttrain-error:0.052262\tvalid-error:0.161491\n",
      "[62]\ttrain-error:0.049142\tvalid-error:0.161491\n",
      "[63]\ttrain-error:0.049922\tvalid-error:0.173913\n",
      "[64]\ttrain-error:0.047582\tvalid-error:0.167702\n",
      "[65]\ttrain-error:0.046802\tvalid-error:0.164596\n",
      "[66]\ttrain-error:0.045242\tvalid-error:0.161491\n",
      "[67]\ttrain-error:0.046802\tvalid-error:0.161491\n",
      "[68]\ttrain-error:0.046802\tvalid-error:0.158385\n",
      "[69]\ttrain-error:0.044462\tvalid-error:0.164596\n",
      "[70]\ttrain-error:0.043682\tvalid-error:0.167702\n",
      "[71]\ttrain-error:0.042122\tvalid-error:0.167702\n",
      "[72]\ttrain-error:0.040562\tvalid-error:0.167702\n",
      "[73]\ttrain-error:0.036661\tvalid-error:0.164596\n",
      "[74]\ttrain-error:0.035101\tvalid-error:0.167702\n",
      "[75]\ttrain-error:0.035101\tvalid-error:0.167702\n",
      "[76]\ttrain-error:0.035101\tvalid-error:0.173913\n",
      "[77]\ttrain-error:0.034321\tvalid-error:0.177019\n",
      "[78]\ttrain-error:0.036661\tvalid-error:0.173913\n",
      "[79]\ttrain-error:0.033541\tvalid-error:0.170807\n",
      "[80]\ttrain-error:0.033541\tvalid-error:0.170807\n",
      "[81]\ttrain-error:0.034321\tvalid-error:0.173913\n",
      "[82]\ttrain-error:0.031201\tvalid-error:0.170807\n",
      "[83]\ttrain-error:0.031201\tvalid-error:0.170807\n",
      "[84]\ttrain-error:0.030421\tvalid-error:0.170807\n",
      "[85]\ttrain-error:0.028861\tvalid-error:0.170807\n",
      "[86]\ttrain-error:0.029641\tvalid-error:0.170807\n",
      "[87]\ttrain-error:0.028861\tvalid-error:0.164596\n",
      "[88]\ttrain-error:0.028861\tvalid-error:0.167702\n",
      "[89]\ttrain-error:0.028861\tvalid-error:0.167702\n",
      "[90]\ttrain-error:0.028081\tvalid-error:0.167702\n",
      "[91]\ttrain-error:0.027301\tvalid-error:0.170807\n",
      "[92]\ttrain-error:0.027301\tvalid-error:0.170807\n",
      "[93]\ttrain-error:0.027301\tvalid-error:0.170807\n",
      "[94]\ttrain-error:0.026521\tvalid-error:0.170807\n",
      "[95]\ttrain-error:0.025741\tvalid-error:0.170807\n",
      "[96]\ttrain-error:0.024961\tvalid-error:0.173913\n",
      "[97]\ttrain-error:0.025741\tvalid-error:0.173913\n",
      "[98]\ttrain-error:0.025741\tvalid-error:0.177019\n",
      "[99]\ttrain-error:0.025741\tvalid-error:0.170807\n",
      "[100]\ttrain-error:0.025741\tvalid-error:0.177019\n",
      "[101]\ttrain-error:0.025741\tvalid-error:0.170807\n",
      "[102]\ttrain-error:0.024181\tvalid-error:0.173913\n",
      "[103]\ttrain-error:0.023401\tvalid-error:0.177019\n",
      "[104]\ttrain-error:0.024181\tvalid-error:0.180124\n",
      "[105]\ttrain-error:0.024181\tvalid-error:0.180124\n",
      "[106]\ttrain-error:0.023401\tvalid-error:0.177019\n",
      "[107]\ttrain-error:0.023401\tvalid-error:0.180124\n",
      "[108]\ttrain-error:0.022621\tvalid-error:0.177019\n",
      "[109]\ttrain-error:0.023401\tvalid-error:0.177019\n",
      "[110]\ttrain-error:0.023401\tvalid-error:0.170807\n",
      "[111]\ttrain-error:0.022621\tvalid-error:0.173913\n",
      "[112]\ttrain-error:0.021841\tvalid-error:0.173913\n",
      "[113]\ttrain-error:0.020281\tvalid-error:0.173913\n",
      "[114]\ttrain-error:0.020281\tvalid-error:0.170807\n",
      "[115]\ttrain-error:0.020281\tvalid-error:0.173913\n",
      "[116]\ttrain-error:0.016381\tvalid-error:0.173913\n",
      "[117]\ttrain-error:0.015601\tvalid-error:0.173913\n",
      "[118]\ttrain-error:0.015601\tvalid-error:0.173913\n",
      "[119]\ttrain-error:0.016381\tvalid-error:0.173913\n",
      "[120]\ttrain-error:0.015601\tvalid-error:0.177019\n",
      "[121]\ttrain-error:0.017161\tvalid-error:0.177019\n",
      "[122]\ttrain-error:0.016381\tvalid-error:0.180124\n",
      "[123]\ttrain-error:0.016381\tvalid-error:0.177019\n",
      "[124]\ttrain-error:0.016381\tvalid-error:0.177019\n",
      "Fold1,Done!\n",
      "[0]\ttrain-error:0.249415\tvalid-error:0.23676\n",
      "[1]\ttrain-error:0.230709\tvalid-error:0.227414\n",
      "[2]\ttrain-error:0.211224\tvalid-error:0.249221\n",
      "[3]\ttrain-error:0.1894\tvalid-error:0.221184\n",
      "[4]\ttrain-error:0.194076\tvalid-error:0.23053\n",
      "[5]\ttrain-error:0.180826\tvalid-error:0.205607\n",
      "[6]\ttrain-error:0.183164\tvalid-error:0.218069\n",
      "[7]\ttrain-error:0.17615\tvalid-error:0.199377\n",
      "[8]\ttrain-error:0.178488\tvalid-error:0.221184\n",
      "[9]\ttrain-error:0.171473\tvalid-error:0.202492\n",
      "[10]\ttrain-error:0.171473\tvalid-error:0.196262\n",
      "[11]\ttrain-error:0.166797\tvalid-error:0.205607\n",
      "[12]\ttrain-error:0.163679\tvalid-error:0.208723\n",
      "[13]\ttrain-error:0.158223\tvalid-error:0.180685\n",
      "[14]\ttrain-error:0.159782\tvalid-error:0.186916\n",
      "[15]\ttrain-error:0.151208\tvalid-error:0.190031\n",
      "[16]\ttrain-error:0.144973\tvalid-error:0.190031\n",
      "[17]\ttrain-error:0.145752\tvalid-error:0.183801\n",
      "[18]\ttrain-error:0.144193\tvalid-error:0.17134\n",
      "[19]\ttrain-error:0.133281\tvalid-error:0.17134\n",
      "[20]\ttrain-error:0.134061\tvalid-error:0.17757\n",
      "[21]\ttrain-error:0.124708\tvalid-error:0.183801\n",
      "[22]\ttrain-error:0.118472\tvalid-error:0.180685\n",
      "[23]\ttrain-error:0.122369\tvalid-error:0.17757\n",
      "[24]\ttrain-error:0.122369\tvalid-error:0.180685\n",
      "[25]\ttrain-error:0.119252\tvalid-error:0.186916\n",
      "[26]\ttrain-error:0.117693\tvalid-error:0.17134\n",
      "[27]\ttrain-error:0.113796\tvalid-error:0.168224\n",
      "[28]\ttrain-error:0.113796\tvalid-error:0.158879\n",
      "[29]\ttrain-error:0.116134\tvalid-error:0.158879\n",
      "[30]\ttrain-error:0.113016\tvalid-error:0.165109\n",
      "[31]\ttrain-error:0.115355\tvalid-error:0.161994\n",
      "[32]\ttrain-error:0.111458\tvalid-error:0.158879\n",
      "[33]\ttrain-error:0.111458\tvalid-error:0.161994\n",
      "[34]\ttrain-error:0.104443\tvalid-error:0.161994\n",
      "[35]\ttrain-error:0.105222\tvalid-error:0.168224\n",
      "[36]\ttrain-error:0.102884\tvalid-error:0.161994\n",
      "[37]\ttrain-error:0.097428\tvalid-error:0.165109\n",
      "[38]\ttrain-error:0.098987\tvalid-error:0.161994\n",
      "[39]\ttrain-error:0.096648\tvalid-error:0.155763\n",
      "[40]\ttrain-error:0.095869\tvalid-error:0.161994\n",
      "[41]\ttrain-error:0.09509\tvalid-error:0.152648\n",
      "[42]\ttrain-error:0.096648\tvalid-error:0.152648\n",
      "[43]\ttrain-error:0.092751\tvalid-error:0.149533\n",
      "[44]\ttrain-error:0.088075\tvalid-error:0.146417\n",
      "[45]\ttrain-error:0.086516\tvalid-error:0.146417\n",
      "[46]\ttrain-error:0.088075\tvalid-error:0.158879\n",
      "[47]\ttrain-error:0.087295\tvalid-error:0.152648\n",
      "[48]\ttrain-error:0.085737\tvalid-error:0.158879\n",
      "[49]\ttrain-error:0.083398\tvalid-error:0.165109\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[50]\ttrain-error:0.080281\tvalid-error:0.155763\n",
      "[51]\ttrain-error:0.076383\tvalid-error:0.155763\n",
      "[52]\ttrain-error:0.073266\tvalid-error:0.152648\n",
      "[53]\ttrain-error:0.073266\tvalid-error:0.158879\n",
      "[54]\ttrain-error:0.074045\tvalid-error:0.155763\n",
      "[55]\ttrain-error:0.070928\tvalid-error:0.155763\n",
      "[56]\ttrain-error:0.070148\tvalid-error:0.161994\n",
      "[57]\ttrain-error:0.068589\tvalid-error:0.155763\n",
      "[58]\ttrain-error:0.06703\tvalid-error:0.158879\n",
      "[59]\ttrain-error:0.068589\tvalid-error:0.152648\n",
      "[60]\ttrain-error:0.06781\tvalid-error:0.168224\n",
      "[61]\ttrain-error:0.066251\tvalid-error:0.158879\n",
      "[62]\ttrain-error:0.065472\tvalid-error:0.168224\n",
      "[63]\ttrain-error:0.063133\tvalid-error:0.152648\n",
      "[64]\ttrain-error:0.061574\tvalid-error:0.152648\n",
      "[65]\ttrain-error:0.059236\tvalid-error:0.155763\n",
      "[66]\ttrain-error:0.056898\tvalid-error:0.146417\n",
      "[67]\ttrain-error:0.056118\tvalid-error:0.149533\n",
      "[68]\ttrain-error:0.055339\tvalid-error:0.149533\n",
      "[69]\ttrain-error:0.051442\tvalid-error:0.152648\n",
      "[70]\ttrain-error:0.056898\tvalid-error:0.143302\n",
      "[71]\ttrain-error:0.052221\tvalid-error:0.149533\n",
      "[72]\ttrain-error:0.053001\tvalid-error:0.146417\n",
      "[73]\ttrain-error:0.050663\tvalid-error:0.149533\n",
      "[74]\ttrain-error:0.046765\tvalid-error:0.149533\n",
      "[75]\ttrain-error:0.047545\tvalid-error:0.149533\n",
      "[76]\ttrain-error:0.045986\tvalid-error:0.149533\n",
      "[77]\ttrain-error:0.046765\tvalid-error:0.149533\n",
      "[78]\ttrain-error:0.041309\tvalid-error:0.146417\n",
      "[79]\ttrain-error:0.041309\tvalid-error:0.146417\n",
      "[80]\ttrain-error:0.042089\tvalid-error:0.146417\n",
      "[81]\ttrain-error:0.043648\tvalid-error:0.143302\n",
      "[82]\ttrain-error:0.043648\tvalid-error:0.149533\n",
      "[83]\ttrain-error:0.042089\tvalid-error:0.146417\n",
      "[84]\ttrain-error:0.042089\tvalid-error:0.146417\n",
      "[85]\ttrain-error:0.042089\tvalid-error:0.152648\n",
      "[86]\ttrain-error:0.042089\tvalid-error:0.146417\n",
      "[87]\ttrain-error:0.042089\tvalid-error:0.143302\n",
      "[88]\ttrain-error:0.041309\tvalid-error:0.143302\n",
      "[89]\ttrain-error:0.042868\tvalid-error:0.143302\n",
      "[90]\ttrain-error:0.039751\tvalid-error:0.149533\n",
      "[91]\ttrain-error:0.037412\tvalid-error:0.143302\n",
      "[92]\ttrain-error:0.035853\tvalid-error:0.149533\n",
      "[93]\ttrain-error:0.035853\tvalid-error:0.152648\n",
      "[94]\ttrain-error:0.035853\tvalid-error:0.152648\n",
      "[95]\ttrain-error:0.035074\tvalid-error:0.155763\n",
      "[96]\ttrain-error:0.035853\tvalid-error:0.152648\n",
      "[97]\ttrain-error:0.035074\tvalid-error:0.149533\n",
      "[98]\ttrain-error:0.033515\tvalid-error:0.152648\n",
      "[99]\ttrain-error:0.033515\tvalid-error:0.152648\n",
      "[100]\ttrain-error:0.032736\tvalid-error:0.155763\n",
      "[101]\ttrain-error:0.031177\tvalid-error:0.155763\n",
      "[102]\ttrain-error:0.031177\tvalid-error:0.158879\n",
      "[103]\ttrain-error:0.029618\tvalid-error:0.152648\n",
      "[104]\ttrain-error:0.028839\tvalid-error:0.158879\n",
      "[105]\ttrain-error:0.02728\tvalid-error:0.158879\n",
      "[106]\ttrain-error:0.02728\tvalid-error:0.155763\n",
      "[107]\ttrain-error:0.029618\tvalid-error:0.152648\n",
      "[108]\ttrain-error:0.02728\tvalid-error:0.161994\n",
      "[109]\ttrain-error:0.024162\tvalid-error:0.161994\n",
      "[110]\ttrain-error:0.024942\tvalid-error:0.158879\n",
      "[111]\ttrain-error:0.024942\tvalid-error:0.155763\n",
      "[112]\ttrain-error:0.024942\tvalid-error:0.155763\n",
      "[113]\ttrain-error:0.022603\tvalid-error:0.152648\n",
      "[114]\ttrain-error:0.023383\tvalid-error:0.152648\n",
      "[115]\ttrain-error:0.022603\tvalid-error:0.149533\n",
      "[116]\ttrain-error:0.024162\tvalid-error:0.152648\n",
      "[117]\ttrain-error:0.024162\tvalid-error:0.149533\n",
      "[118]\ttrain-error:0.023383\tvalid-error:0.149533\n",
      "[119]\ttrain-error:0.023383\tvalid-error:0.146417\n",
      "[120]\ttrain-error:0.023383\tvalid-error:0.140187\n",
      "[121]\ttrain-error:0.023383\tvalid-error:0.146417\n",
      "[122]\ttrain-error:0.021824\tvalid-error:0.133956\n",
      "[123]\ttrain-error:0.020265\tvalid-error:0.133956\n",
      "[124]\ttrain-error:0.021044\tvalid-error:0.140187\n",
      "Fold2,Done!\n",
      "[0]\ttrain-error:0.258768\tvalid-error:0.29595\n",
      "[1]\ttrain-error:0.222915\tvalid-error:0.261682\n",
      "[2]\ttrain-error:0.204988\tvalid-error:0.242991\n",
      "[3]\ttrain-error:0.176929\tvalid-error:0.199377\n",
      "[4]\ttrain-error:0.187062\tvalid-error:0.208723\n",
      "[5]\ttrain-error:0.186282\tvalid-error:0.218069\n",
      "[6]\ttrain-error:0.181606\tvalid-error:0.205607\n",
      "[7]\ttrain-error:0.178488\tvalid-error:0.205607\n",
      "[8]\ttrain-error:0.171473\tvalid-error:0.202492\n",
      "[9]\ttrain-error:0.171473\tvalid-error:0.193146\n",
      "[10]\ttrain-error:0.169914\tvalid-error:0.205607\n",
      "[11]\ttrain-error:0.159002\tvalid-error:0.186916\n",
      "[12]\ttrain-error:0.153546\tvalid-error:0.186916\n",
      "[13]\ttrain-error:0.147311\tvalid-error:0.190031\n",
      "[14]\ttrain-error:0.149649\tvalid-error:0.193146\n",
      "[15]\ttrain-error:0.14809\tvalid-error:0.196262\n",
      "[16]\ttrain-error:0.144973\tvalid-error:0.199377\n",
      "[17]\ttrain-error:0.151988\tvalid-error:0.186916\n",
      "[18]\ttrain-error:0.140296\tvalid-error:0.186916\n",
      "[19]\ttrain-error:0.140296\tvalid-error:0.193146\n",
      "[20]\ttrain-error:0.133281\tvalid-error:0.193146\n",
      "[21]\ttrain-error:0.13484\tvalid-error:0.190031\n",
      "[22]\ttrain-error:0.134061\tvalid-error:0.180685\n",
      "[23]\ttrain-error:0.132502\tvalid-error:0.193146\n",
      "[24]\ttrain-error:0.125487\tvalid-error:0.186916\n",
      "[25]\ttrain-error:0.123149\tvalid-error:0.180685\n",
      "[26]\ttrain-error:0.12159\tvalid-error:0.183801\n",
      "[27]\ttrain-error:0.12159\tvalid-error:0.17757\n",
      "[28]\ttrain-error:0.113796\tvalid-error:0.180685\n",
      "[29]\ttrain-error:0.113016\tvalid-error:0.183801\n",
      "[30]\ttrain-error:0.109899\tvalid-error:0.17757\n",
      "[31]\ttrain-error:0.102104\tvalid-error:0.161994\n",
      "[32]\ttrain-error:0.101325\tvalid-error:0.17134\n",
      "[33]\ttrain-error:0.098987\tvalid-error:0.17134\n",
      "[34]\ttrain-error:0.095869\tvalid-error:0.165109\n",
      "[35]\ttrain-error:0.092751\tvalid-error:0.17757\n",
      "[36]\ttrain-error:0.089634\tvalid-error:0.183801\n",
      "[37]\ttrain-error:0.088854\tvalid-error:0.180685\n",
      "[38]\ttrain-error:0.091193\tvalid-error:0.17134\n",
      "[39]\ttrain-error:0.091193\tvalid-error:0.174455\n",
      "[40]\ttrain-error:0.088854\tvalid-error:0.17757\n",
      "[41]\ttrain-error:0.091193\tvalid-error:0.168224\n",
      "[42]\ttrain-error:0.088075\tvalid-error:0.17134\n",
      "[43]\ttrain-error:0.090413\tvalid-error:0.165109\n",
      "[44]\ttrain-error:0.083398\tvalid-error:0.168224\n",
      "[45]\ttrain-error:0.081839\tvalid-error:0.168224\n",
      "[46]\ttrain-error:0.077942\tvalid-error:0.174455\n",
      "[47]\ttrain-error:0.076383\tvalid-error:0.17134\n",
      "[48]\ttrain-error:0.072486\tvalid-error:0.17757\n",
      "[49]\ttrain-error:0.070148\tvalid-error:0.174455\n",
      "[50]\ttrain-error:0.068589\tvalid-error:0.174455\n",
      "[51]\ttrain-error:0.069369\tvalid-error:0.168224\n",
      "[52]\ttrain-error:0.06703\tvalid-error:0.174455\n",
      "[53]\ttrain-error:0.06703\tvalid-error:0.174455\n",
      "[54]\ttrain-error:0.068589\tvalid-error:0.168224\n",
      "[55]\ttrain-error:0.066251\tvalid-error:0.168224\n",
      "[56]\ttrain-error:0.065472\tvalid-error:0.17134\n",
      "[57]\ttrain-error:0.062354\tvalid-error:0.174455\n",
      "[58]\ttrain-error:0.058457\tvalid-error:0.174455\n",
      "[59]\ttrain-error:0.060795\tvalid-error:0.168224\n",
      "[60]\ttrain-error:0.060795\tvalid-error:0.168224\n",
      "[61]\ttrain-error:0.056898\tvalid-error:0.17757\n",
      "[62]\ttrain-error:0.05456\tvalid-error:0.180685\n",
      "[63]\ttrain-error:0.057677\tvalid-error:0.174455\n",
      "[64]\ttrain-error:0.051442\tvalid-error:0.17757\n",
      "[65]\ttrain-error:0.048324\tvalid-error:0.17757\n",
      "[66]\ttrain-error:0.048324\tvalid-error:0.17757\n",
      "[67]\ttrain-error:0.051442\tvalid-error:0.168224\n",
      "[68]\ttrain-error:0.048324\tvalid-error:0.168224\n",
      "[69]\ttrain-error:0.045986\tvalid-error:0.174455\n",
      "[70]\ttrain-error:0.045986\tvalid-error:0.17134\n",
      "[71]\ttrain-error:0.046765\tvalid-error:0.17134\n",
      "[72]\ttrain-error:0.042868\tvalid-error:0.174455\n",
      "[73]\ttrain-error:0.044427\tvalid-error:0.174455\n",
      "[74]\ttrain-error:0.043648\tvalid-error:0.174455\n",
      "[75]\ttrain-error:0.042089\tvalid-error:0.168224\n",
      "[76]\ttrain-error:0.046765\tvalid-error:0.17134\n",
      "[77]\ttrain-error:0.045207\tvalid-error:0.174455\n",
      "[78]\ttrain-error:0.046765\tvalid-error:0.168224\n",
      "[79]\ttrain-error:0.045986\tvalid-error:0.17134\n",
      "[80]\ttrain-error:0.044427\tvalid-error:0.168224\n",
      "[81]\ttrain-error:0.043648\tvalid-error:0.17134\n",
      "[82]\ttrain-error:0.042089\tvalid-error:0.161994\n",
      "[83]\ttrain-error:0.039751\tvalid-error:0.158879\n",
      "[84]\ttrain-error:0.042089\tvalid-error:0.158879\n",
      "[85]\ttrain-error:0.04053\tvalid-error:0.165109\n",
      "[86]\ttrain-error:0.038971\tvalid-error:0.161994\n",
      "[87]\ttrain-error:0.038971\tvalid-error:0.165109\n",
      "[88]\ttrain-error:0.037412\tvalid-error:0.161994\n",
      "[89]\ttrain-error:0.035853\tvalid-error:0.168224\n",
      "[90]\ttrain-error:0.037412\tvalid-error:0.161994\n",
      "[91]\ttrain-error:0.035853\tvalid-error:0.158879\n",
      "[92]\ttrain-error:0.038192\tvalid-error:0.165109\n",
      "[93]\ttrain-error:0.036633\tvalid-error:0.149533\n",
      "[94]\ttrain-error:0.035853\tvalid-error:0.158879\n",
      "[95]\ttrain-error:0.034295\tvalid-error:0.158879\n",
      "[96]\ttrain-error:0.032736\tvalid-error:0.158879\n",
      "[97]\ttrain-error:0.033515\tvalid-error:0.158879\n",
      "[98]\ttrain-error:0.035074\tvalid-error:0.155763\n",
      "[99]\ttrain-error:0.031956\tvalid-error:0.152648\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[100]\ttrain-error:0.032736\tvalid-error:0.152648\n",
      "[101]\ttrain-error:0.034295\tvalid-error:0.152648\n",
      "[102]\ttrain-error:0.031956\tvalid-error:0.155763\n",
      "[103]\ttrain-error:0.030398\tvalid-error:0.161994\n",
      "[104]\ttrain-error:0.031956\tvalid-error:0.168224\n",
      "[105]\ttrain-error:0.02728\tvalid-error:0.161994\n",
      "[106]\ttrain-error:0.024942\tvalid-error:0.158879\n",
      "[107]\ttrain-error:0.02728\tvalid-error:0.158879\n",
      "[108]\ttrain-error:0.025721\tvalid-error:0.158879\n",
      "[109]\ttrain-error:0.024942\tvalid-error:0.161994\n",
      "[110]\ttrain-error:0.025721\tvalid-error:0.158879\n",
      "[111]\ttrain-error:0.024162\tvalid-error:0.155763\n",
      "[112]\ttrain-error:0.0265\tvalid-error:0.161994\n",
      "[113]\ttrain-error:0.023383\tvalid-error:0.165109\n",
      "[114]\ttrain-error:0.022603\tvalid-error:0.165109\n",
      "[115]\ttrain-error:0.022603\tvalid-error:0.165109\n",
      "[116]\ttrain-error:0.021044\tvalid-error:0.161994\n",
      "[117]\ttrain-error:0.020265\tvalid-error:0.158879\n",
      "[118]\ttrain-error:0.020265\tvalid-error:0.158879\n",
      "[119]\ttrain-error:0.020265\tvalid-error:0.158879\n",
      "[120]\ttrain-error:0.019486\tvalid-error:0.161994\n",
      "[121]\ttrain-error:0.017927\tvalid-error:0.158879\n",
      "[122]\ttrain-error:0.017927\tvalid-error:0.17134\n",
      "[123]\ttrain-error:0.018706\tvalid-error:0.168224\n",
      "[124]\ttrain-error:0.016368\tvalid-error:0.161994\n",
      "Fold3,Done!\n",
      "[0]\ttrain-error:0.281153\tvalid-error:0.290625\n",
      "[1]\ttrain-error:0.21729\tvalid-error:0.24375\n",
      "[2]\ttrain-error:0.213396\tvalid-error:0.25625\n",
      "[3]\ttrain-error:0.204829\tvalid-error:0.240625\n",
      "[4]\ttrain-error:0.207944\tvalid-error:0.234375\n",
      "[5]\ttrain-error:0.207165\tvalid-error:0.246875\n",
      "[6]\ttrain-error:0.193925\tvalid-error:0.24375\n",
      "[7]\ttrain-error:0.175234\tvalid-error:0.234375\n",
      "[8]\ttrain-error:0.183022\tvalid-error:0.234375\n",
      "[9]\ttrain-error:0.170561\tvalid-error:0.228125\n",
      "[10]\ttrain-error:0.162773\tvalid-error:0.221875\n",
      "[11]\ttrain-error:0.162773\tvalid-error:0.225\n",
      "[12]\ttrain-error:0.165109\tvalid-error:0.215625\n",
      "[13]\ttrain-error:0.154206\tvalid-error:0.215625\n",
      "[14]\ttrain-error:0.151869\tvalid-error:0.209375\n",
      "[15]\ttrain-error:0.143302\tvalid-error:0.203125\n",
      "[16]\ttrain-error:0.140187\tvalid-error:0.2125\n",
      "[17]\ttrain-error:0.134735\tvalid-error:0.203125\n",
      "[18]\ttrain-error:0.126947\tvalid-error:0.19375\n",
      "[19]\ttrain-error:0.132399\tvalid-error:0.19375\n",
      "[20]\ttrain-error:0.130841\tvalid-error:0.20625\n",
      "[21]\ttrain-error:0.127726\tvalid-error:0.1875\n",
      "[22]\ttrain-error:0.126168\tvalid-error:0.1875\n",
      "[23]\ttrain-error:0.125389\tvalid-error:0.18125\n",
      "[24]\ttrain-error:0.124611\tvalid-error:0.175\n",
      "[25]\ttrain-error:0.122274\tvalid-error:0.153125\n",
      "[26]\ttrain-error:0.125389\tvalid-error:0.153125\n",
      "[27]\ttrain-error:0.122274\tvalid-error:0.165625\n",
      "[28]\ttrain-error:0.119938\tvalid-error:0.16875\n",
      "[29]\ttrain-error:0.116822\tvalid-error:0.171875\n",
      "[30]\ttrain-error:0.11215\tvalid-error:0.171875\n",
      "[31]\ttrain-error:0.115265\tvalid-error:0.16875\n",
      "[32]\ttrain-error:0.109813\tvalid-error:0.16875\n",
      "[33]\ttrain-error:0.109034\tvalid-error:0.1625\n",
      "[34]\ttrain-error:0.103583\tvalid-error:0.15625\n",
      "[35]\ttrain-error:0.09891\tvalid-error:0.153125\n",
      "[36]\ttrain-error:0.09891\tvalid-error:0.14375\n",
      "[37]\ttrain-error:0.094237\tvalid-error:0.140625\n",
      "[38]\ttrain-error:0.094237\tvalid-error:0.14375\n",
      "[39]\ttrain-error:0.092679\tvalid-error:0.1375\n",
      "[40]\ttrain-error:0.0919\tvalid-error:0.134375\n",
      "[41]\ttrain-error:0.089564\tvalid-error:0.140625\n",
      "[42]\ttrain-error:0.091121\tvalid-error:0.140625\n",
      "[43]\ttrain-error:0.084891\tvalid-error:0.140625\n",
      "[44]\ttrain-error:0.084891\tvalid-error:0.14375\n",
      "[45]\ttrain-error:0.084112\tvalid-error:0.1375\n",
      "[46]\ttrain-error:0.07866\tvalid-error:0.140625\n",
      "[47]\ttrain-error:0.076324\tvalid-error:0.14375\n",
      "[48]\ttrain-error:0.07866\tvalid-error:0.1375\n",
      "[49]\ttrain-error:0.076324\tvalid-error:0.140625\n",
      "[50]\ttrain-error:0.073209\tvalid-error:0.140625\n",
      "[51]\ttrain-error:0.074766\tvalid-error:0.14375\n",
      "[52]\ttrain-error:0.075545\tvalid-error:0.140625\n",
      "[53]\ttrain-error:0.07243\tvalid-error:0.1375\n",
      "[54]\ttrain-error:0.068536\tvalid-error:0.128125\n",
      "[55]\ttrain-error:0.070093\tvalid-error:0.13125\n",
      "[56]\ttrain-error:0.068536\tvalid-error:0.140625\n",
      "[57]\ttrain-error:0.067757\tvalid-error:0.140625\n",
      "[58]\ttrain-error:0.063084\tvalid-error:0.1375\n",
      "[59]\ttrain-error:0.064642\tvalid-error:0.1375\n",
      "[60]\ttrain-error:0.063863\tvalid-error:0.134375\n",
      "[61]\ttrain-error:0.065421\tvalid-error:0.1375\n",
      "[62]\ttrain-error:0.063084\tvalid-error:0.128125\n",
      "[63]\ttrain-error:0.062305\tvalid-error:0.13125\n",
      "[64]\ttrain-error:0.063084\tvalid-error:0.134375\n",
      "[65]\ttrain-error:0.060748\tvalid-error:0.125\n",
      "[66]\ttrain-error:0.05919\tvalid-error:0.115625\n",
      "[67]\ttrain-error:0.058411\tvalid-error:0.1125\n",
      "[68]\ttrain-error:0.057632\tvalid-error:0.115625\n",
      "[69]\ttrain-error:0.057632\tvalid-error:0.115625\n",
      "[70]\ttrain-error:0.05296\tvalid-error:0.11875\n",
      "[71]\ttrain-error:0.052181\tvalid-error:0.125\n",
      "[72]\ttrain-error:0.049065\tvalid-error:0.11875\n",
      "[73]\ttrain-error:0.049065\tvalid-error:0.128125\n",
      "[74]\ttrain-error:0.047508\tvalid-error:0.125\n",
      "[75]\ttrain-error:0.045171\tvalid-error:0.13125\n",
      "[76]\ttrain-error:0.044393\tvalid-error:0.125\n",
      "[77]\ttrain-error:0.04595\tvalid-error:0.125\n",
      "[78]\ttrain-error:0.043614\tvalid-error:0.125\n",
      "[79]\ttrain-error:0.043614\tvalid-error:0.13125\n",
      "[80]\ttrain-error:0.040498\tvalid-error:0.134375\n",
      "[81]\ttrain-error:0.042056\tvalid-error:0.13125\n",
      "[82]\ttrain-error:0.03972\tvalid-error:0.13125\n",
      "[83]\ttrain-error:0.040498\tvalid-error:0.13125\n",
      "[84]\ttrain-error:0.038162\tvalid-error:0.125\n",
      "[85]\ttrain-error:0.038162\tvalid-error:0.1125\n",
      "[86]\ttrain-error:0.038162\tvalid-error:0.115625\n",
      "[87]\ttrain-error:0.035826\tvalid-error:0.11875\n",
      "[88]\ttrain-error:0.033489\tvalid-error:0.121875\n",
      "[89]\ttrain-error:0.034268\tvalid-error:0.121875\n",
      "[90]\ttrain-error:0.033489\tvalid-error:0.121875\n",
      "[91]\ttrain-error:0.031931\tvalid-error:0.11875\n",
      "[92]\ttrain-error:0.030374\tvalid-error:0.11875\n",
      "[93]\ttrain-error:0.030374\tvalid-error:0.121875\n",
      "[94]\ttrain-error:0.028037\tvalid-error:0.115625\n",
      "[95]\ttrain-error:0.027259\tvalid-error:0.11875\n",
      "[96]\ttrain-error:0.024922\tvalid-error:0.121875\n",
      "[97]\ttrain-error:0.024922\tvalid-error:0.125\n",
      "[98]\ttrain-error:0.025701\tvalid-error:0.11875\n",
      "[99]\ttrain-error:0.025701\tvalid-error:0.1125\n",
      "[100]\ttrain-error:0.024143\tvalid-error:0.11875\n",
      "[101]\ttrain-error:0.023364\tvalid-error:0.11875\n",
      "[102]\ttrain-error:0.023364\tvalid-error:0.115625\n",
      "[103]\ttrain-error:0.021028\tvalid-error:0.121875\n",
      "[104]\ttrain-error:0.01947\tvalid-error:0.128125\n",
      "[105]\ttrain-error:0.01947\tvalid-error:0.121875\n",
      "[106]\ttrain-error:0.01947\tvalid-error:0.121875\n",
      "[107]\ttrain-error:0.017913\tvalid-error:0.11875\n",
      "[108]\ttrain-error:0.018692\tvalid-error:0.11875\n",
      "[109]\ttrain-error:0.017913\tvalid-error:0.121875\n",
      "[110]\ttrain-error:0.017913\tvalid-error:0.11875\n",
      "[111]\ttrain-error:0.018692\tvalid-error:0.115625\n",
      "[112]\ttrain-error:0.017913\tvalid-error:0.115625\n",
      "[113]\ttrain-error:0.017913\tvalid-error:0.115625\n",
      "[114]\ttrain-error:0.017134\tvalid-error:0.1125\n",
      "[115]\ttrain-error:0.017134\tvalid-error:0.1125\n",
      "[116]\ttrain-error:0.01947\tvalid-error:0.1125\n",
      "[117]\ttrain-error:0.015576\tvalid-error:0.115625\n",
      "[118]\ttrain-error:0.014798\tvalid-error:0.115625\n",
      "[119]\ttrain-error:0.014798\tvalid-error:0.115625\n",
      "[120]\ttrain-error:0.016355\tvalid-error:0.115625\n",
      "[121]\ttrain-error:0.016355\tvalid-error:0.115625\n",
      "[122]\ttrain-error:0.016355\tvalid-error:0.1125\n",
      "[123]\ttrain-error:0.016355\tvalid-error:0.11875\n",
      "[124]\ttrain-error:0.016355\tvalid-error:0.115625\n",
      "Fold4,Done!\n",
      "[0]\ttrain-error:0.25\tvalid-error:0.309375\n",
      "[1]\ttrain-error:0.203271\tvalid-error:0.2875\n",
      "[2]\ttrain-error:0.211059\tvalid-error:0.309375\n",
      "[3]\ttrain-error:0.185358\tvalid-error:0.3125\n",
      "[4]\ttrain-error:0.185358\tvalid-error:0.278125\n",
      "[5]\ttrain-error:0.184579\tvalid-error:0.3\n",
      "[6]\ttrain-error:0.186137\tvalid-error:0.2875\n",
      "[7]\ttrain-error:0.183022\tvalid-error:0.28125\n",
      "[8]\ttrain-error:0.176791\tvalid-error:0.275\n",
      "[9]\ttrain-error:0.176012\tvalid-error:0.278125\n",
      "[10]\ttrain-error:0.166667\tvalid-error:0.28125\n",
      "[11]\ttrain-error:0.169782\tvalid-error:0.2875\n",
      "[12]\ttrain-error:0.160436\tvalid-error:0.29375\n",
      "[13]\ttrain-error:0.16433\tvalid-error:0.284375\n",
      "[14]\ttrain-error:0.154984\tvalid-error:0.2875\n",
      "[15]\ttrain-error:0.145639\tvalid-error:0.271875\n",
      "[16]\ttrain-error:0.135514\tvalid-error:0.265625\n",
      "[17]\ttrain-error:0.138629\tvalid-error:0.253125\n",
      "[18]\ttrain-error:0.135514\tvalid-error:0.25\n",
      "[19]\ttrain-error:0.133956\tvalid-error:0.265625\n",
      "[20]\ttrain-error:0.123832\tvalid-error:0.2625\n",
      "[21]\ttrain-error:0.11838\tvalid-error:0.240625\n",
      "[22]\ttrain-error:0.114486\tvalid-error:0.25625\n",
      "[23]\ttrain-error:0.112928\tvalid-error:0.246875\n",
      "[24]\ttrain-error:0.11215\tvalid-error:0.234375\n",
      "[25]\ttrain-error:0.107477\tvalid-error:0.2375\n",
      "[26]\ttrain-error:0.104361\tvalid-error:0.2375\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[27]\ttrain-error:0.103583\tvalid-error:0.24375\n",
      "[28]\ttrain-error:0.102804\tvalid-error:0.25\n",
      "[29]\ttrain-error:0.101246\tvalid-error:0.24375\n",
      "[30]\ttrain-error:0.096573\tvalid-error:0.2375\n",
      "[31]\ttrain-error:0.095016\tvalid-error:0.246875\n",
      "[32]\ttrain-error:0.093458\tvalid-error:0.221875\n",
      "[33]\ttrain-error:0.094237\tvalid-error:0.21875\n",
      "[34]\ttrain-error:0.093458\tvalid-error:0.215625\n",
      "[35]\ttrain-error:0.0919\tvalid-error:0.19375\n",
      "[36]\ttrain-error:0.0919\tvalid-error:0.2125\n",
      "[37]\ttrain-error:0.088785\tvalid-error:0.215625\n",
      "[38]\ttrain-error:0.088006\tvalid-error:0.2\n",
      "[39]\ttrain-error:0.084891\tvalid-error:0.1875\n",
      "[40]\ttrain-error:0.081776\tvalid-error:0.184375\n",
      "[41]\ttrain-error:0.080997\tvalid-error:0.1875\n",
      "[42]\ttrain-error:0.081776\tvalid-error:0.190625\n",
      "[43]\ttrain-error:0.07866\tvalid-error:0.19375\n",
      "[44]\ttrain-error:0.076324\tvalid-error:0.18125\n",
      "[45]\ttrain-error:0.077882\tvalid-error:0.178125\n",
      "[46]\ttrain-error:0.077103\tvalid-error:0.18125\n",
      "[47]\ttrain-error:0.077103\tvalid-error:0.171875\n",
      "[48]\ttrain-error:0.076324\tvalid-error:0.171875\n",
      "[49]\ttrain-error:0.074766\tvalid-error:0.16875\n",
      "[50]\ttrain-error:0.071651\tvalid-error:0.178125\n",
      "[51]\ttrain-error:0.065421\tvalid-error:0.171875\n",
      "[52]\ttrain-error:0.067757\tvalid-error:0.175\n",
      "[53]\ttrain-error:0.066199\tvalid-error:0.16875\n",
      "[54]\ttrain-error:0.063084\tvalid-error:0.1625\n",
      "[55]\ttrain-error:0.063863\tvalid-error:0.171875\n",
      "[56]\ttrain-error:0.056854\tvalid-error:0.165625\n",
      "[57]\ttrain-error:0.059969\tvalid-error:0.165625\n",
      "[58]\ttrain-error:0.056854\tvalid-error:0.159375\n",
      "[59]\ttrain-error:0.056854\tvalid-error:0.165625\n",
      "[60]\ttrain-error:0.056075\tvalid-error:0.159375\n",
      "[61]\ttrain-error:0.057632\tvalid-error:0.16875\n",
      "[62]\ttrain-error:0.056854\tvalid-error:0.16875\n",
      "[63]\ttrain-error:0.053738\tvalid-error:0.178125\n",
      "[64]\ttrain-error:0.050623\tvalid-error:0.178125\n",
      "[65]\ttrain-error:0.051402\tvalid-error:0.171875\n",
      "[66]\ttrain-error:0.051402\tvalid-error:0.16875\n",
      "[67]\ttrain-error:0.049065\tvalid-error:0.165625\n",
      "[68]\ttrain-error:0.049065\tvalid-error:0.175\n",
      "[69]\ttrain-error:0.048287\tvalid-error:0.175\n",
      "[70]\ttrain-error:0.044393\tvalid-error:0.18125\n",
      "[71]\ttrain-error:0.04595\tvalid-error:0.178125\n",
      "[72]\ttrain-error:0.046729\tvalid-error:0.178125\n",
      "[73]\ttrain-error:0.045171\tvalid-error:0.175\n",
      "[74]\ttrain-error:0.04595\tvalid-error:0.178125\n",
      "[75]\ttrain-error:0.043614\tvalid-error:0.171875\n",
      "[76]\ttrain-error:0.041277\tvalid-error:0.175\n",
      "[77]\ttrain-error:0.042835\tvalid-error:0.175\n",
      "[78]\ttrain-error:0.042835\tvalid-error:0.16875\n",
      "[79]\ttrain-error:0.042835\tvalid-error:0.16875\n",
      "[80]\ttrain-error:0.040498\tvalid-error:0.16875\n",
      "[81]\ttrain-error:0.03972\tvalid-error:0.165625\n",
      "[82]\ttrain-error:0.041277\tvalid-error:0.165625\n",
      "[83]\ttrain-error:0.038162\tvalid-error:0.16875\n",
      "[84]\ttrain-error:0.037383\tvalid-error:0.16875\n",
      "[85]\ttrain-error:0.036604\tvalid-error:0.165625\n",
      "[86]\ttrain-error:0.035047\tvalid-error:0.165625\n",
      "[87]\ttrain-error:0.033489\tvalid-error:0.16875\n",
      "[88]\ttrain-error:0.035047\tvalid-error:0.16875\n",
      "[89]\ttrain-error:0.031931\tvalid-error:0.171875\n",
      "[90]\ttrain-error:0.028816\tvalid-error:0.171875\n",
      "[91]\ttrain-error:0.028816\tvalid-error:0.171875\n",
      "[92]\ttrain-error:0.02648\tvalid-error:0.171875\n",
      "[93]\ttrain-error:0.027259\tvalid-error:0.175\n",
      "[94]\ttrain-error:0.02648\tvalid-error:0.175\n",
      "[95]\ttrain-error:0.024922\tvalid-error:0.171875\n",
      "[96]\ttrain-error:0.022586\tvalid-error:0.178125\n",
      "[97]\ttrain-error:0.023364\tvalid-error:0.175\n",
      "[98]\ttrain-error:0.023364\tvalid-error:0.171875\n",
      "[99]\ttrain-error:0.023364\tvalid-error:0.16875\n",
      "[100]\ttrain-error:0.023364\tvalid-error:0.171875\n",
      "[101]\ttrain-error:0.022586\tvalid-error:0.175\n",
      "[102]\ttrain-error:0.021807\tvalid-error:0.175\n",
      "[103]\ttrain-error:0.021028\tvalid-error:0.171875\n",
      "[104]\ttrain-error:0.023364\tvalid-error:0.175\n",
      "[105]\ttrain-error:0.023364\tvalid-error:0.171875\n",
      "[106]\ttrain-error:0.022586\tvalid-error:0.171875\n",
      "[107]\ttrain-error:0.021807\tvalid-error:0.171875\n",
      "[108]\ttrain-error:0.021028\tvalid-error:0.16875\n",
      "[109]\ttrain-error:0.021028\tvalid-error:0.165625\n",
      "[110]\ttrain-error:0.021807\tvalid-error:0.178125\n",
      "[111]\ttrain-error:0.021807\tvalid-error:0.18125\n",
      "[112]\ttrain-error:0.020249\tvalid-error:0.184375\n",
      "[113]\ttrain-error:0.020249\tvalid-error:0.171875\n",
      "[114]\ttrain-error:0.021028\tvalid-error:0.171875\n",
      "[115]\ttrain-error:0.017913\tvalid-error:0.165625\n",
      "[116]\ttrain-error:0.016355\tvalid-error:0.1625\n",
      "[117]\ttrain-error:0.017134\tvalid-error:0.1625\n",
      "[118]\ttrain-error:0.016355\tvalid-error:0.159375\n",
      "[119]\ttrain-error:0.014798\tvalid-error:0.15625\n",
      "[120]\ttrain-error:0.016355\tvalid-error:0.15625\n",
      "[121]\ttrain-error:0.014798\tvalid-error:0.153125\n",
      "[122]\ttrain-error:0.014798\tvalid-error:0.15\n",
      "[123]\ttrain-error:0.014798\tvalid-error:0.153125\n",
      "[124]\ttrain-error:0.015576\tvalid-error:0.159375\n",
      "Fold5,Done!\n",
      "all_cv_valid_score\n"
     ]
    }
   ],
   "source": [
    "cv5_score,jt = xgb_cv5_ensemble()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test = xgb.Booster(model_file = 'xgbmodels/1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  9.30216193e-01,   1.18495345e-01,   9.33574736e-01,\n",
       "         1.19841965e-02,   3.89860310e-02,   3.05081531e-03,\n",
       "         8.43689561e-01,   2.02422030e-02,   9.72251713e-01,\n",
       "         8.12476221e-03,   9.59637284e-01,   9.42460179e-01,\n",
       "         5.10363579e-01,   9.57942784e-01,   9.61259305e-01,\n",
       "         7.74938893e-03,   9.10757542e-01,   9.02308524e-01,\n",
       "         9.58071887e-01,   4.71686721e-02,   9.70711827e-01,\n",
       "         1.60330147e-01,   7.19150960e-01,   9.55066085e-01,\n",
       "         2.88649917e-01,   3.55880171e-01,   8.69590044e-01,\n",
       "         8.60727787e-01,   5.44600114e-02,   8.00786078e-01,\n",
       "         2.81799883e-01,   3.84505749e-01,   4.48864279e-03,\n",
       "         9.22111332e-01,   2.82749087e-02,   8.20679545e-01,\n",
       "         8.00827861e-01,   1.75811059e-03,   2.88332105e-02,\n",
       "         9.20137286e-01,   1.43489540e-01,   9.79234219e-01,\n",
       "         9.70732033e-01,   5.23065403e-02,   2.53462702e-01,\n",
       "         8.12394321e-01,   4.48300660e-01,   2.83244699e-01,\n",
       "         2.71717072e-01,   1.81745328e-02,   9.77809727e-01,\n",
       "         1.76787302e-02,   6.85934734e-04,   4.64497097e-02,\n",
       "         8.03104937e-01,   8.10490847e-01,   8.26814353e-01,\n",
       "         9.72751617e-01,   9.16757286e-01,   5.26939612e-03,\n",
       "         8.47905427e-02,   9.07273412e-01,   2.45327242e-02,\n",
       "         2.16308340e-01,   8.83885741e-01,   7.86632393e-03,\n",
       "         9.60356772e-01,   7.19796300e-01,   9.74905074e-01,\n",
       "         4.76279948e-03,   3.71378586e-02,   1.66951809e-02,\n",
       "         5.19335091e-01,   9.85787153e-01,   7.20100760e-01,\n",
       "         7.14031219e-01,   8.10990691e-01,   8.69682789e-01,\n",
       "         1.70857534e-01,   9.38140035e-01,   2.64439899e-02,\n",
       "         9.39296722e-01,   5.90141527e-02,   7.40804791e-01,\n",
       "         2.32456103e-02,   9.04043019e-01,   8.83161068e-01,\n",
       "         2.90338546e-01,   1.63521189e-02,   1.14857361e-01,\n",
       "         9.90475357e-01,   8.14928412e-01,   8.81177425e-01,\n",
       "         8.35163519e-02,   5.44095179e-03,   8.88015687e-01,\n",
       "         9.63422656e-01,   9.70897436e-01,   9.73475873e-01,\n",
       "         3.29289101e-02,   1.51719609e-02,   9.93070066e-01,\n",
       "         6.85021222e-01,   3.70716482e-01,   8.81014671e-03,\n",
       "         1.22801460e-01,   8.42610151e-02,   3.92771870e-01,\n",
       "         1.58321008e-01,   9.70638156e-01,   6.84712315e-03,\n",
       "         1.53770261e-02,   8.76731217e-01,   1.95729464e-01,\n",
       "         1.08846426e-02,   8.55797529e-01,   3.14854202e-03,\n",
       "         3.82876508e-02,   9.38605666e-01,   1.71970084e-01,\n",
       "         1.77005108e-03,   9.43323493e-01,   8.35517585e-01,\n",
       "         9.86714661e-01,   9.04801250e-01,   7.85528958e-01,\n",
       "         9.54098642e-01,   9.42374945e-01,   5.67416800e-03,\n",
       "         7.52711833e-01,   3.03518564e-01,   2.59069115e-01,\n",
       "         9.63657320e-01,   2.06235945e-01,   9.43675101e-01,\n",
       "         6.20614946e-01,   7.56669521e-01,   7.02917278e-02,\n",
       "         8.26284409e-01,   7.98845053e-01,   1.20461788e-02,\n",
       "         5.50196134e-03,   2.31156155e-01,   9.64416325e-01,\n",
       "         9.17228937e-01,   9.78213251e-01,   7.61264609e-03,\n",
       "         9.54856072e-03,   7.44751617e-02,   3.49809736e-01,\n",
       "         5.58358524e-03,   9.08520341e-01,   4.62624401e-01,\n",
       "         2.10551787e-02,   9.10650253e-01,   1.23895213e-01,\n",
       "         5.07857442e-01,   1.14266574e-02,   8.05878043e-01,\n",
       "         6.81088626e-01,   1.48125187e-01,   8.19333971e-01,\n",
       "         1.03277527e-02,   1.93003789e-01,   8.15828502e-01,\n",
       "         1.20045394e-02,   8.91307175e-01,   5.64466119e-01,\n",
       "         9.60891306e-01,   8.87829483e-01,   6.14610594e-03,\n",
       "         2.41941144e-03,   8.17723334e-01,   3.77867371e-03,\n",
       "         1.25790790e-01,   8.35976183e-01,   7.61422217e-02,\n",
       "         2.96547338e-02,   4.72757779e-02,   1.96422860e-01,\n",
       "         8.21121156e-01,   3.81050631e-02,   7.88264632e-01,\n",
       "         8.54496099e-03,   8.76131505e-02,   7.73218095e-01,\n",
       "         7.88073421e-01,   9.75692391e-01,   9.48026538e-01,\n",
       "         8.40608682e-03,   8.35783362e-01,   5.99914134e-01,\n",
       "         5.34624746e-03,   8.11550796e-01,   8.20633948e-01,\n",
       "         1.02794595e-01,   9.81324971e-01,   3.53859901e-01,\n",
       "         3.72248590e-02,   5.29943081e-03,   9.29780975e-02,\n",
       "         9.27646518e-01,   3.38560250e-03,   8.61501880e-03,\n",
       "         3.55202984e-03,   6.35114849e-01,   2.24059168e-02,\n",
       "         6.04532426e-03,   9.84598026e-02,   9.48167026e-01,\n",
       "         2.27608941e-02,   2.51652151e-01,   2.87247319e-02,\n",
       "         7.92493880e-01,   1.24331377e-02,   8.50601673e-01,\n",
       "         5.93342446e-02,   9.75640059e-01,   1.03218965e-02,\n",
       "         7.87244961e-02,   9.73928154e-01,   9.75010693e-01,\n",
       "         9.18065906e-01,   1.40244206e-02,   2.12788694e-02,\n",
       "         9.57315385e-01,   1.41077891e-01,   1.40674591e-01,\n",
       "         2.95905471e-01,   2.12716892e-01,   5.44778407e-01,\n",
       "         9.83033955e-01,   1.41091822e-02,   9.31163788e-01,\n",
       "         6.63361311e-01,   6.50969818e-02,   8.38015437e-01,\n",
       "         3.00110847e-01,   9.89335496e-03,   2.98247561e-02,\n",
       "         9.33113277e-01,   2.12691445e-02,   8.00078571e-01,\n",
       "         9.81345952e-01,   9.58344817e-01,   6.71349373e-03,\n",
       "         7.89498806e-01,   1.65701941e-01,   3.00693642e-02,\n",
       "         1.92438643e-02,   9.62163389e-01,   9.24468935e-01,\n",
       "         9.77942646e-01,   8.54043126e-01,   6.70854628e-01,\n",
       "         2.70838380e-01,   7.23462045e-01,   6.36049137e-02,\n",
       "         7.40888000e-01,   9.69725568e-03,   1.20322496e-01,\n",
       "         8.06655228e-01,   9.64608729e-01,   6.88051462e-01,\n",
       "         8.18267226e-01,   9.55296457e-01,   3.98977757e-01,\n",
       "         3.59553695e-02,   5.05929887e-01,   9.67301309e-01,\n",
       "         7.69534521e-03,   3.95089477e-01,   9.23763216e-01,\n",
       "         3.87786515e-02,   4.69770432e-02,   7.27661312e-01,\n",
       "         9.71514940e-01,   4.74098474e-02,   8.52859378e-01,\n",
       "         1.88485868e-02,   3.26445639e-01,   9.38196719e-01,\n",
       "         6.09565914e-01,   7.28595436e-01,   9.38856840e-01,\n",
       "         2.93112546e-02,   9.67732787e-01,   2.37561569e-01,\n",
       "         8.67206752e-01,   8.90740156e-01,   9.57501650e-01,\n",
       "         8.06432217e-02,   2.84447940e-03,   9.65627730e-01,\n",
       "         8.95189345e-01,   1.39969110e-01,   2.24651173e-01,\n",
       "         8.95813759e-03,   9.76136863e-01,   2.45311949e-02,\n",
       "         4.46028449e-03,   1.98235869e-01,   1.31425247e-01,\n",
       "         9.09668088e-01,   5.09201922e-02,   3.05887293e-02,\n",
       "         3.60113710e-01,   9.51136556e-03,   4.25715327e-01,\n",
       "         5.39621174e-01,   1.88858360e-01,   4.02777530e-02,\n",
       "         8.89890671e-01,   8.90379965e-01,   3.27088058e-01,\n",
       "         9.87842560e-01,   9.02688384e-01,   7.62480199e-02,\n",
       "         2.62235433e-01,   1.60053372e-01,   5.37307118e-04], dtype=float32)"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def xgb_cv5_predict():\n",
    "    folds = list(StratifiedKFold(n_splits=5,shuffle=True,random_state=2).split(X_whole,target))\n",
    "    all_data_valid_score = target*0\n",
    "    a_test = target*0\n",
    "    for i,(train_mask,valid_mask) in enumerate(folds):\n",
    "        X_train = X_whole.iloc[train_mask]\n",
    "        y_train = target.iloc[train_mask]\n",
    "        X_valid = X_whole.iloc[valid_mask]\n",
    "        y_valid = target.iloc[valid_mask]\n",
    "        dmtrain = xgb.DMatrix(data = X_train,label = y_train)\n",
    "        dmvalid = xgb.DMatrix(data = X_valid,label = y_valid)\n",
    "        xgb_param = {'colsample_bytree': 0.8,\n",
    "                     'gamma': 0,\n",
    "                     'learning_rate': 0.1,\n",
    "                     'max_depth': 3,\n",
    "                     'min_child_weight': 2.0,\n",
    "                     'subsample': 0.8,\n",
    "                     'objective':'binary:logistic',\n",
    "                     'booster':'gbtree','save_period':2,'model_dir':'xgbmodels'}\n",
    "        xgbmodel = xgb.train(params = xgb_param,\n",
    "                          dtrain = dmtrain,\n",
    "                          evals = [(dmtrain,'train'),(dmvalid,'valid')],\n",
    "                          num_boost_round=125,\n",
    "                          verbose_eval=True)\n",
    "        xgbmodel.save_model('xgbmodels/'+str(i+1))\n",
    "        temp_valid = xgbmodel.predict(dmvalid)\n",
    "        all_data_valid_score.iloc[valid_mask] = temp_valid\n",
    "        test = xgb.Booster(model_file = 'xgbmodels/'+str(i+1))\n",
    "        temp2 = test.predict(dmvalid)\n",
    "        a_test.iloc[valid_mask] = temp2\n",
    "        print('Fold'+str(i+1)+',Done!')\n",
    "    all_cv_valid_score = accuracy_score(target,1*(all_data_valid_score>0.5))\n",
    "    just_a_test = accuracy_score(target,1*(a_test>0.5))\n",
    "    print('all_cv_valid_score')\n",
    "    return all_cv_valid_score,just_a_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[MLENS] backend: threading\n"
     ]
    }
   ],
   "source": [
    "from mlens.ensemble import SuperLearner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "Cannot clone object ''a'' (type <class 'str'>): it does not seem to be a scikit-learn estimator as it does not implement a 'get_params' methods.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-141-cdf503612629>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mclone\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'a'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\sklearn\\base.py\u001b[0m in \u001b[0;36mclone\u001b[1;34m(estimator, safe)\u001b[0m\n\u001b[0;32m     56\u001b[0m                             \u001b[1;34m\"it does not seem to be a scikit-learn estimator \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     57\u001b[0m                             \u001b[1;34m\"as it does not implement a 'get_params' methods.\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 58\u001b[1;33m                             % (repr(estimator), type(estimator)))\n\u001b[0m\u001b[0;32m     59\u001b[0m     \u001b[0mklass\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mestimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     60\u001b[0m     \u001b[0mnew_object_params\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mestimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_params\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdeep\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: Cannot clone object ''a'' (type <class 'str'>): it does not seem to be a scikit-learn estimator as it does not implement a 'get_params' methods."
     ]
    }
   ],
   "source": [
    "clone('a')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross Validation Folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def data_augmentation(X_input,Y_input,batch_size = 32):\n",
    "    data_aug = ImageDataGenerator(#featurewise_center=True,\n",
    "                             #featurewise_std_normalization=True,\n",
    "                             #zca_whitening=True,\n",
    "                             rotation_range=0,\n",
    "                             width_shift_range = 0,\n",
    "                             height_shift_range = 0,\n",
    "                             zoom_range = 0,\n",
    "                             data_format = 'channels_last',\n",
    "                             horizontal_flip = True,\n",
    "                             vertical_flip = True)#,fill_mode = 'constant',cval = 0)\n",
    "    data_aug_batches = data_aug.flow(X_input,Y_input,batch_size = batch_size)\n",
    "    return data_aug_batches\n",
    "\n",
    "def data_augmentation_with_others(X_input,X_others,Y_input,batch_size = 32):\n",
    "    data_aug = ImageDataGenerator(#featurewise_center=True,\n",
    "                             #featurewise_std_normalization=True,\n",
    "                             #zca_whitening=True,\n",
    "                             rotation_range=0,\n",
    "                             width_shift_range = 0,\n",
    "                             height_shift_range = 0,\n",
    "                             zoom_range = 0,\n",
    "                             data_format = 'channels_last',\n",
    "                             horizontal_flip = True,\n",
    "                             vertical_flip = True)#,fill_mode = 'constant',cval = 0)\n",
    "    gen1 = data_aug.flow(X_input,Y_input,batch_size = batch_size,seed = 1990)\n",
    "    gen2 = data_aug.flow(X_input,X_others,batch_size = batch_size,seed = 1990)\n",
    "    while True:\n",
    "            Xi1 = gen1.next()\n",
    "            Xi2 = gen2.next()\n",
    "            #Assert arrays are equal - this was for peace of mind, but slows down training\n",
    "            #np.testing.assert_array_equal(X1i[0],X2i[0])\n",
    "            yield [Xi1[0], Xi2[1]], Xi1[1]\n",
    "\n",
    "def data_no_augmentation_with_others(X_input,X_others,Y_input,batch_size = 32):\n",
    "    data_aug = ImageDataGenerator(#featurewise_center=True,\n",
    "                             #featurewise_std_normalization=True,\n",
    "                             #zca_whitening=True,\n",
    "                             rotation_range=0,\n",
    "                             width_shift_range = 0,\n",
    "                             height_shift_range = 0,\n",
    "                             zoom_range = 0,\n",
    "                             data_format = 'channels_last',\n",
    "                             horizontal_flip = False,\n",
    "                             vertical_flip = False)#,fill_mode = 'constant',cval = 0)\n",
    "    gen1 = data_aug.flow(X_input,Y_input,batch_size = batch_size,seed = 1990)\n",
    "    gen2 = data_aug.flow(X_input,X_others,batch_size = batch_size,seed = 1990)\n",
    "    while True:\n",
    "            Xi1 = gen1.next()\n",
    "            Xi2 = gen2.next()\n",
    "            #Assert arrays are equal - this was for peace of mind, but slows down training\n",
    "            #np.testing.assert_array_equal(X1i[0],X2i[0])\n",
    "            yield [Xi1[0], Xi2[1]], Xi1[1]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "def cnnmodel():\n",
    "    X_input = Input(shape = (75,75,3))\n",
    "    #First composite layer\n",
    "    X = Conv2D(64,kernel_size = (5,5))(X_input)\n",
    "    X = Activation('relu')(X)\n",
    "    X = MaxPooling2D(pool_size=(3,3),strides =(2,2))(X)\n",
    "    X = Dropout(rate = 0.2)(X)\n",
    "    \n",
    "    #Second composite layer\n",
    "    X = Conv2D(128,kernel_size = (3,3))(X)\n",
    "    X = Activation('relu')(X)\n",
    "    X = MaxPooling2D(pool_size=(3,3),strides =(2,2))(X)\n",
    "    X = Dropout(rate = 0.2)(X)\n",
    "    \n",
    "    #Third composite layer\n",
    "    X = Conv2D(128,kernel_size = (3,3))(X)\n",
    "    X = Activation('relu')(X)\n",
    "    X = MaxPooling2D(pool_size=(2,2),strides =(2,2))(X)\n",
    "    X = Dropout(rate = 0.2)(X)\n",
    "\n",
    "    #Forth composite layer\n",
    "    X = Conv2D(64,kernel_size = (3,3))(X)\n",
    "    X = Activation('relu')(X)\n",
    "    X = MaxPooling2D(pool_size=(3,3),strides =(2,2))(X)\n",
    "    X = Dropout(rate = 0.2)(X)\n",
    "\n",
    "    #Flatten layer\n",
    "    X = Flatten()(X)\n",
    "    \n",
    "    #First dense layer\n",
    "    X = Dense(256,activation = 'relu')(X)\n",
    "    X = Dropout(rate = 0.2)(X)\n",
    "   \n",
    "    #Second dense layer\n",
    "    X = Dense(256,activation = 'relu')(X)\n",
    "    X = Dropout(rate = 0.2)(X)\n",
    "   \n",
    "    #Decision layer\n",
    "    X = Dense(1,activation='sigmoid')(X)\n",
    "    \n",
    "    model = Model(inputs=X_input,outputs=X)\n",
    "\n",
    "    return model\n",
    "\n",
    "def cnnmodel_with_others():\n",
    "    input_other = Input(shape=(13,), name=\"other\")\n",
    "    X_other = Dense(16,activation = 'relu')(input_other)\n",
    "    X_other = Dense(16,activation = 'relu')(X_other)\n",
    "    X_input = Input(shape = (75,75,3))\n",
    "    #First composite layer\n",
    "    X = Conv2D(64,kernel_size = (5,5))(X_input)\n",
    "    X = Activation('relu')(X)\n",
    "    X = MaxPooling2D(pool_size=(3,3),strides =(2,2))(X)\n",
    "    X = Dropout(rate = 0.2)(X)\n",
    "    \n",
    "    #Second composite layer\n",
    "    X = Conv2D(128,kernel_size = (3,3))(X)\n",
    "    X = Activation('relu')(X)\n",
    "    X = MaxPooling2D(pool_size=(3,3),strides =(2,2))(X)\n",
    "    X = Dropout(rate = 0.2)(X)\n",
    "    \n",
    "    #Third composite layer\n",
    "    X = Conv2D(128,kernel_size = (3,3))(X)\n",
    "    X = Activation('relu')(X)\n",
    "    X = MaxPooling2D(pool_size=(2,2),strides =(2,2))(X)\n",
    "    X = Dropout(rate = 0.2)(X)\n",
    "\n",
    "    #Forth composite layer\n",
    "    X = Conv2D(64,kernel_size = (3,3))(X)\n",
    "    X = Activation('relu')(X)\n",
    "    X = MaxPooling2D(pool_size=(3,3),strides =(2,2))(X)\n",
    "    X = Dropout(rate = 0.2)(X)\n",
    "\n",
    "    #Flatten layer\n",
    "    X = Flatten()(X)\n",
    "    X = concatenate([X,X_other])\n",
    "    \n",
    "    #First dense layer\n",
    "    X = Dense(256,activation = 'relu')(X)\n",
    "    X = Dropout(rate = 0.2)(X)\n",
    "   \n",
    "    #Second dense layer\n",
    "    X = Dense(256,activation = 'relu')(X)\n",
    "    X = Dropout(rate = 0.2)(X)\n",
    "   \n",
    "    #Decision layer\n",
    "    X = Dense(1,activation='sigmoid')(X)\n",
    "    \n",
    "    model = Model(inputs=[X_input,input_other],outputs=X)\n",
    "\n",
    "    return model\n",
    "\n",
    "def model_compile(model,lr = 0.0001,decay = 0, freezing_layers = None,weights_path = None):\n",
    "   \n",
    "    if weights_path:\n",
    "        model.load_weights(weights_path)\n",
    "    else:\n",
    "        pass\n",
    "    if freezing_layers:\n",
    "        for layer in model.layers[:freezing_layers]:\n",
    "            layer.trainable = False\n",
    "    else:\n",
    "        pass\n",
    "    optimizer = Adam(lr = lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=decay)\n",
    "    #optimizer = SGD(lr, decay=0, momentum=0.9, nesterov=True)\n",
    "    model.compile(loss='binary_crossentropy',optimizer=optimizer,metrics=['accuracy'])\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "def get_callbacks(filepath = \".model_weights.hdf5\", patience=7, monitor = 'val_loss', verbose = 1, save_best_only = True):\n",
    "    es = EarlyStopping('val_loss', patience=patience, mode=\"min\")\n",
    "    msave = ModelCheckpoint(filepath, monitor=monitor, verbose = verbose, save_best_only=True)\n",
    "    return [es,msave]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "def normal_fit(model,callbacks,epochs = 100,batch_size = 32,verbose = 1):\n",
    "    \n",
    "    result = model.fit(X_train_cv, y_train_cv,\n",
    "                          batch_size=batch_size,\n",
    "                          epochs=epochs,\n",
    "                          verbose=verbose,\n",
    "                          validation_data=(X_valid, y_valid),\n",
    "                          callbacks=callbacks)\n",
    "    return result\n",
    "\n",
    "def data_augment_fit(model,callbacks,epochs = 100,batch_size = 32,verbose = 1):\n",
    "    \n",
    "    result = model.fit_generator(data_augmentation(X_train_cv,y_train_cv,batch_size = batch_size),\n",
    "                          steps_per_epoch = len(X_train_cv)/batch_size,\n",
    "                          epochs=epochs,\n",
    "                          verbose=verbose,\n",
    "                          validation_data=(X_valid, y_valid),\n",
    "                          callbacks=callbacks)\n",
    "    return result\n",
    "\n",
    "def data_augment_fit_with_others(model,callbacks,epochs = 100,batch_size = 32,verbose = 1,da = False):\n",
    "    \n",
    "    if da == False:\n",
    "        result = model.fit_generator(data_no_augmentation_with_others(X_train_cv,X_train_others,y_train_cv,batch_size = 32),\n",
    "                          steps_per_epoch = len(X_train_cv)/batch_size,\n",
    "                          epochs=epochs,\n",
    "                          verbose=verbose,\n",
    "                          validation_data=([X_valid,X_valid_others], y_valid),\n",
    "                          callbacks=callbacks)\n",
    "    else:\n",
    "        result = model.fit_generator(data_augmentation_with_others(X_train_cv,X_train_others,y_train_cv,batch_size = 32),\n",
    "                          steps_per_epoch = len(X_train_cv)/batch_size,\n",
    "                          epochs=epochs,\n",
    "                          verbose=verbose,\n",
    "                          validation_data=([X_valid,X_valid_others], y_valid),\n",
    "                          callbacks=callbacks)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 75, 75, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 71, 71, 64)   4864        input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 71, 71, 64)   0           conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)  (None, 35, 35, 64)   0           activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 35, 35, 64)   0           max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 33, 33, 128)  73856       dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 33, 33, 128)  0           conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2D)  (None, 16, 16, 128)  0           activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 16, 16, 128)  0           max_pooling2d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 14, 14, 128)  147584      dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 14, 14, 128)  0           conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2D)  (None, 7, 7, 128)    0           activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 7, 7, 128)    0           max_pooling2d_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 5, 5, 64)     73792       dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 5, 5, 64)     0           conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2D)  (None, 2, 2, 64)     0           activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "other (InputLayer)              (None, 13)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 2, 2, 64)     0           max_pooling2d_4[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 16)           224         other[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 256)          0           dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 16)           272         dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 272)          0           flatten_1[0][0]                  \n",
      "                                                                 dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 256)          69888       concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 256)          0           dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 256)          65792       dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)             (None, 256)          0           dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 1)            257         dropout_6[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 436,529\n",
      "Trainable params: 436,529\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'X_train_cv' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-21-2e963bee9483>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mCnnmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel_compile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcnnmodel_with_others\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mlr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0.0001\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mdecay\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfreezing_layers\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mweights_path\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mcallbacks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_callbacks\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/2018.1.4.hdf5'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpatience\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m30\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msave_best_only\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdata_augment_fit_with_others\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCnnmodel\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mverbose\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-20-03a312f2d1cf>\u001b[0m in \u001b[0;36mdata_augment_fit_with_others\u001b[1;34m(model, callbacks, epochs, batch_size, verbose, da)\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mda\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 25\u001b[1;33m         result = model.fit_generator(data_no_augmentation_with_others(X_train_cv,X_train_others,y_train_cv,batch_size = 32),\n\u001b[0m\u001b[0;32m     26\u001b[0m                           \u001b[0msteps_per_epoch\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     27\u001b[0m                           \u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mepochs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'X_train_cv' is not defined"
     ]
    }
   ],
   "source": [
    "Cnnmodel = model_compile(cnnmodel_with_others(),lr = 0.0001,decay = 0, freezing_layers = None,weights_path = None)\n",
    "callbacks = get_callbacks(filepath = 'model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/2018.1.4.hdf5', patience=30, save_best_only = True)\n",
    "result = data_augment_fit_with_others(Cnnmodel,callbacks=callbacks,verbose = 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Cnnmodel = model_compile(cnnmodel_with_others(),lr = 0.0001,decay = 0, freezing_layers = 5,weights_path = 'model save/Model 7-2x filters Advanced CNN with DA and self TL/2018.1.2.hdf5')\n",
    "callbacks = get_callbacks(filepath = 'model save/Model 7-2x filters Advanced CNN with DA and self TL/2018.1.2_stage2.hdf5', patience=30, save_best_only = True)\n",
    "result = data_augment_fit(Cnnmodel,callbacks = callbacks,epochs = 100,batch_size=32,verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "Cnnmodel.load_weights('model save/Model 7-2x filters Advanced CNN with DA and self TL/Model-7 cv1_stage1.hdf5')\n",
    "Cnnmodel.evaluate(X_valid, y_valid)\n",
    "#Cnnmodel.evaluate(X_train_cv, y_train_cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def save_history(result,name = 'fit_history.csv',path = os.getcwd()):\n",
    "    train_loss = result.history['loss']\n",
    "    val_loss = result.history['val_loss']\n",
    "    train_accuracy = result.history['acc']\n",
    "    val_accuracy = result.history['val_acc']\n",
    "    fit_history = pd.DataFrame({'train_loss':train_loss,'val_loss':val_loss,'train_accuracy':train_accuracy,'val_accuracy':val_accuracy})\n",
    "    fit_history.to_csv(os.path.join(path,name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "save_history(result_Cnn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.47316885,  0.43016976,  0.49740523, ...,  0.49190184,\n",
       "        0.48540697,  0.48072121], dtype=float32)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Cnnmodel.predict([X_train_cv,X_train_others]).reshape(1283)+0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Model11_CV(K=3):\n",
    "    folds = list(StratifiedKFold(n_splits=K, shuffle=True,random_state = 3).split(X, target))\n",
    "    y_train_pred_log=0\n",
    "    y_valid_pred_log = 0.0*target\n",
    "    for j, (train_idx, valid_idx) in enumerate(folds):\n",
    "        print('\\n===================FOLD=',j+1)\n",
    "        global X_train_cv\n",
    "        X_train_cv = X[train_idx]\n",
    "        global y_train_cv\n",
    "        y_train_cv = target[train_idx]\n",
    "        global X_valid\n",
    "        X_valid = X[valid_idx]\n",
    "        global y_valid\n",
    "        y_valid= target[valid_idx]\n",
    "        global X_train_others\n",
    "        X_train_others = np.array(X_others)[train_idx]\n",
    "        global X_valid_others\n",
    "        X_valid_others = np.array(X_others)[valid_idx]\n",
    "        \n",
    "        scaler = MinMaxScaler().fit(X_train_others)\n",
    "        X_train_others = scaler.transform(X_train_others)\n",
    "        X_valid_others = scaler.transform(X_valid_others)\n",
    "        #Angle\n",
    "#         X_angle_cv=X_angle[train_idx]\n",
    "#         X_angle_hold=X_angle[test_idx]\n",
    "\n",
    "        #define file path and get callbacks\n",
    "    \n",
    "        file_path_stage1 = 'model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv'+str(j+1)+'_stage1.hdf5'\n",
    "        Cnnmodel = model_compile(cnnmodel_with_others(),lr = 0.0001,decay = 0, freezing_layers = None,weights_path = None)\n",
    "        callbacks = get_callbacks(filepath = file_path_stage1, patience=30, save_best_only = True)\n",
    "        result1 = data_augment_fit_with_others(Cnnmodel,callbacks = callbacks,epochs = 150,verbose=2)\n",
    "        print('CV'+str(j+1)+', Done!')\n",
    "        \n",
    "        file_path_stage2 = 'model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv'+str(j+1)+'_stage2.hdf5'\n",
    "        Cnnmodel = model_compile(cnnmodel_with_others(),lr = 0.0001,decay = 0, freezing_layers = 5,weights_path = file_path_stage1)\n",
    "        callbacks = get_callbacks(filepath = file_path_stage2, patience=30, save_best_only = True)\n",
    "        result2 = data_augment_fit_with_others(Cnnmodel,callbacks = callbacks,epochs = 150,verbose=2,da=True)\n",
    "        print('CV'+str(j+1)+', Done!')\n",
    "#         callbacks = get_callbacks(filepath=file_path, patience=5)\n",
    "#         gen_flow = gen_flow_for_two_inputs(X_train_cv, X_angle_cv, y_train_cv)\n",
    "#         galaxyModel= getVggAngleModel()\n",
    "#         galaxyModel.fit_generator(\n",
    "#                 gen_flow,\n",
    "#                 steps_per_epoch=24,\n",
    "#                 epochs=100,\n",
    "#                 shuffle=True,\n",
    "#                 verbose=1,\n",
    "#                 validation_data=([X_holdout,X_angle_hold], Y_holdout),\n",
    "#                 callbacks=callbacks)\n",
    "\n",
    "        #Getting the Best Model\n",
    "        Cnnmodel.load_weights(filepath=file_path_stage2)\n",
    "        #Getting Training Score\n",
    "        score = Cnnmodel.evaluate([X_train_cv,X_train_others],y_train_cv)                                   \n",
    "        print('Train loss:', score[0])\n",
    "        print('Train accuracy:', score[1])\n",
    "        #Getting Test Score\n",
    "        score = Cnnmodel.evaluate([X_valid,X_valid_others], y_valid)\n",
    "        print('Valid loss:', score[0])\n",
    "        print('Valid accuracy:', score[1])\n",
    "\n",
    "        #Getting validation Score.\n",
    "        pred_valid=Cnnmodel.predict([X_valid,X_valid_others])\n",
    "        y_valid_pred_log[valid_idx] = pred_valid.reshape(pred_valid.shape[0])\n",
    "\n",
    "        #Getting Test Scores\n",
    "#         temp_test=Cnnmodel.predict(X_test)\n",
    "#         y_test_pred_log+=temp_test.reshape(temp_test.shape[0])\n",
    "\n",
    "        #Getting Train Scores\n",
    "#         temp_train=Cnnmodel.predict([X_train_cv,X_train_others])\n",
    "#         y_train_pred_log+=temp_train.reshape(temp_train.shape[0])\n",
    "\n",
    "#     y_valid_pred_log=y_valid_pred_log/K\n",
    "    #y_train_pred_log=y_train_pred_log/K\n",
    "\n",
    "#     print('\\n Train Log Loss Validation= ',log_loss(target, y_train_pred_log))\n",
    "    print(' Test Log Loss Validation= ',log_loss(target, y_valid_pred_log))\n",
    "    return y_valid_pred_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================FOLD= 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\sklearn\\utils\\validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_16 (InputLayer)           (None, 75, 75, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_61 (Conv2D)              (None, 71, 71, 64)   4864        input_16[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_61 (Activation)      (None, 71, 71, 64)   0           conv2d_61[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_61 (MaxPooling2D) (None, 35, 35, 64)   0           activation_61[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_91 (Dropout)            (None, 35, 35, 64)   0           max_pooling2d_61[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_62 (Conv2D)              (None, 33, 33, 128)  73856       dropout_91[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_62 (Activation)      (None, 33, 33, 128)  0           conv2d_62[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_62 (MaxPooling2D) (None, 16, 16, 128)  0           activation_62[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_92 (Dropout)            (None, 16, 16, 128)  0           max_pooling2d_62[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_63 (Conv2D)              (None, 14, 14, 128)  147584      dropout_92[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_63 (Activation)      (None, 14, 14, 128)  0           conv2d_63[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_63 (MaxPooling2D) (None, 7, 7, 128)    0           activation_63[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_93 (Dropout)            (None, 7, 7, 128)    0           max_pooling2d_63[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_64 (Conv2D)              (None, 5, 5, 64)     73792       dropout_93[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_64 (Activation)      (None, 5, 5, 64)     0           conv2d_64[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_64 (MaxPooling2D) (None, 2, 2, 64)     0           activation_64[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "other (InputLayer)              (None, 13)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_94 (Dropout)            (None, 2, 2, 64)     0           max_pooling2d_64[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_76 (Dense)                (None, 16)           224         other[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_16 (Flatten)            (None, 256)          0           dropout_94[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_77 (Dense)                (None, 16)           272         dense_76[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_16 (Concatenate)    (None, 272)          0           flatten_16[0][0]                 \n",
      "                                                                 dense_77[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_78 (Dense)                (None, 256)          69888       concatenate_16[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_95 (Dropout)            (None, 256)          0           dense_78[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_79 (Dense)                (None, 256)          65792       dropout_95[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_96 (Dropout)            (None, 256)          0           dense_79[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_80 (Dense)                (None, 1)            257         dropout_96[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 436,529\n",
      "Trainable params: 436,529\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/150\n",
      "Epoch 00001: val_loss improved from inf to 0.69096, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 11s - loss: 1.2622 - acc: 0.5432 - val_loss: 0.6910 - val_acc: 0.5404\n",
      "Epoch 2/150\n",
      "Epoch 00002: val_loss improved from 0.69096 to 0.68520, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.7441 - acc: 0.5336 - val_loss: 0.6852 - val_acc: 0.5311\n",
      "Epoch 3/150\n",
      "Epoch 00003: val_loss improved from 0.68520 to 0.67124, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.7063 - acc: 0.5424 - val_loss: 0.6712 - val_acc: 0.5062\n",
      "Epoch 4/150\n",
      "Epoch 00004: val_loss improved from 0.67124 to 0.63197, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.6748 - acc: 0.5572 - val_loss: 0.6320 - val_acc: 0.6335\n",
      "Epoch 5/150\n",
      "Epoch 00005: val_loss improved from 0.63197 to 0.59607, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.6320 - acc: 0.5996 - val_loss: 0.5961 - val_acc: 0.6739\n",
      "Epoch 6/150\n",
      "Epoch 00006: val_loss improved from 0.59607 to 0.57357, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.5956 - acc: 0.6286 - val_loss: 0.5736 - val_acc: 0.6615\n",
      "Epoch 7/150\n",
      "Epoch 00007: val_loss improved from 0.57357 to 0.55663, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.5674 - acc: 0.6583 - val_loss: 0.5566 - val_acc: 0.6894\n",
      "Epoch 8/150\n",
      "Epoch 00008: val_loss improved from 0.55663 to 0.53519, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.5432 - acc: 0.6766 - val_loss: 0.5352 - val_acc: 0.7174\n",
      "Epoch 9/150\n",
      "Epoch 00009: val_loss improved from 0.53519 to 0.52083, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.5292 - acc: 0.7018 - val_loss: 0.5208 - val_acc: 0.7360\n",
      "Epoch 10/150\n",
      "Epoch 00010: val_loss improved from 0.52083 to 0.50535, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.5082 - acc: 0.7186 - val_loss: 0.5053 - val_acc: 0.7453\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11/150\n",
      "Epoch 00011: val_loss improved from 0.50535 to 0.45251, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.4855 - acc: 0.7514 - val_loss: 0.4525 - val_acc: 0.7733\n",
      "Epoch 12/150\n",
      "Epoch 00012: val_loss improved from 0.45251 to 0.43462, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.4472 - acc: 0.7712 - val_loss: 0.4346 - val_acc: 0.7857\n",
      "Epoch 13/150\n",
      "Epoch 00013: val_loss improved from 0.43462 to 0.40216, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.4229 - acc: 0.7895 - val_loss: 0.4022 - val_acc: 0.7981\n",
      "Epoch 14/150\n",
      "Epoch 00014: val_loss improved from 0.40216 to 0.37459, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.4001 - acc: 0.8154 - val_loss: 0.3746 - val_acc: 0.8323\n",
      "Epoch 15/150\n",
      "Epoch 00015: val_loss improved from 0.37459 to 0.36593, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.3931 - acc: 0.8170 - val_loss: 0.3659 - val_acc: 0.8602\n",
      "Epoch 16/150\n",
      "Epoch 00016: val_loss improved from 0.36593 to 0.36204, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.3613 - acc: 0.8322 - val_loss: 0.3620 - val_acc: 0.8230\n",
      "Epoch 17/150\n",
      "Epoch 00017: val_loss did not improve\n",
      " - 6s - loss: 0.3848 - acc: 0.8150 - val_loss: 0.3872 - val_acc: 0.7950\n",
      "Epoch 18/150\n",
      "Epoch 00018: val_loss improved from 0.36204 to 0.34722, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.3756 - acc: 0.8279 - val_loss: 0.3472 - val_acc: 0.8261\n",
      "Epoch 19/150\n",
      "Epoch 00019: val_loss improved from 0.34722 to 0.34009, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.3372 - acc: 0.8467 - val_loss: 0.3401 - val_acc: 0.8261\n",
      "Epoch 20/150\n",
      "Epoch 00020: val_loss improved from 0.34009 to 0.33335, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.3287 - acc: 0.8482 - val_loss: 0.3333 - val_acc: 0.8323\n",
      "Epoch 21/150\n",
      "Epoch 00021: val_loss did not improve\n",
      " - 5s - loss: 0.3149 - acc: 0.8622 - val_loss: 0.3720 - val_acc: 0.8199\n",
      "Epoch 22/150\n",
      "Epoch 00022: val_loss improved from 0.33335 to 0.32553, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.3133 - acc: 0.8582 - val_loss: 0.3255 - val_acc: 0.8634\n",
      "Epoch 23/150\n",
      "Epoch 00023: val_loss improved from 0.32553 to 0.31816, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.3180 - acc: 0.8615 - val_loss: 0.3182 - val_acc: 0.8478\n",
      "Epoch 24/150\n",
      "Epoch 00024: val_loss improved from 0.31816 to 0.30983, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.2973 - acc: 0.8658 - val_loss: 0.3098 - val_acc: 0.8665\n",
      "Epoch 25/150\n",
      "Epoch 00025: val_loss did not improve\n",
      " - 6s - loss: 0.2854 - acc: 0.8742 - val_loss: 0.3568 - val_acc: 0.8354\n",
      "Epoch 26/150\n",
      "Epoch 00026: val_loss did not improve\n",
      " - 6s - loss: 0.3073 - acc: 0.8673 - val_loss: 0.3895 - val_acc: 0.8075\n",
      "Epoch 27/150\n",
      "Epoch 00027: val_loss did not improve\n",
      " - 6s - loss: 0.2998 - acc: 0.8630 - val_loss: 0.3117 - val_acc: 0.8509\n",
      "Epoch 28/150\n",
      "Epoch 00028: val_loss improved from 0.30983 to 0.30781, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.2733 - acc: 0.8841 - val_loss: 0.3078 - val_acc: 0.8602\n",
      "Epoch 29/150\n",
      "Epoch 00029: val_loss improved from 0.30781 to 0.30010, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.2623 - acc: 0.8887 - val_loss: 0.3001 - val_acc: 0.8851\n",
      "Epoch 30/150\n",
      "Epoch 00030: val_loss did not improve\n",
      " - 6s - loss: 0.2511 - acc: 0.8993 - val_loss: 0.3156 - val_acc: 0.8540\n",
      "Epoch 31/150\n",
      "Epoch 00031: val_loss improved from 0.30010 to 0.29409, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.2619 - acc: 0.8775 - val_loss: 0.2941 - val_acc: 0.8665\n",
      "Epoch 32/150\n",
      "Epoch 00032: val_loss did not improve\n",
      " - 6s - loss: 0.2445 - acc: 0.8940 - val_loss: 0.3247 - val_acc: 0.8292\n",
      "Epoch 33/150\n",
      "Epoch 00033: val_loss improved from 0.29409 to 0.29180, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.2427 - acc: 0.8909 - val_loss: 0.2918 - val_acc: 0.8696\n",
      "Epoch 34/150\n",
      "Epoch 00034: val_loss did not improve\n",
      " - 6s - loss: 0.2265 - acc: 0.8993 - val_loss: 0.2973 - val_acc: 0.8882\n",
      "Epoch 35/150\n",
      "Epoch 00035: val_loss did not improve\n",
      " - 6s - loss: 0.2418 - acc: 0.8897 - val_loss: 0.3088 - val_acc: 0.8571\n",
      "Epoch 36/150\n",
      "Epoch 00036: val_loss improved from 0.29180 to 0.28804, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.2455 - acc: 0.8948 - val_loss: 0.2880 - val_acc: 0.8634\n",
      "Epoch 37/150\n",
      "Epoch 00037: val_loss did not improve\n",
      " - 6s - loss: 0.2380 - acc: 0.9031 - val_loss: 0.3202 - val_acc: 0.8385\n",
      "Epoch 38/150\n",
      "Epoch 00038: val_loss improved from 0.28804 to 0.27161, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.2615 - acc: 0.8813 - val_loss: 0.2716 - val_acc: 0.8727\n",
      "Epoch 39/150\n",
      "Epoch 00039: val_loss did not improve\n",
      " - 6s - loss: 0.2533 - acc: 0.8894 - val_loss: 0.2764 - val_acc: 0.8758\n",
      "Epoch 40/150\n",
      "Epoch 00040: val_loss did not improve\n",
      " - 5s - loss: 0.2482 - acc: 0.8821 - val_loss: 0.3570 - val_acc: 0.8261\n",
      "Epoch 41/150\n",
      "Epoch 00041: val_loss did not improve\n",
      " - 6s - loss: 0.3232 - acc: 0.8536 - val_loss: 0.2960 - val_acc: 0.8602\n",
      "Epoch 42/150\n",
      "Epoch 00042: val_loss did not improve\n",
      " - 6s - loss: 0.2239 - acc: 0.9062 - val_loss: 0.2753 - val_acc: 0.8820\n",
      "Epoch 43/150\n",
      "Epoch 00043: val_loss did not improve\n",
      " - 6s - loss: 0.2185 - acc: 0.9024 - val_loss: 0.2876 - val_acc: 0.8665\n",
      "Epoch 44/150\n",
      "Epoch 00044: val_loss did not improve\n",
      " - 6s - loss: 0.2327 - acc: 0.8993 - val_loss: 0.2936 - val_acc: 0.8540\n",
      "Epoch 45/150\n",
      "Epoch 00045: val_loss improved from 0.27161 to 0.26724, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.2154 - acc: 0.9085 - val_loss: 0.2672 - val_acc: 0.8696\n",
      "Epoch 46/150\n",
      "Epoch 00046: val_loss did not improve\n",
      " - 6s - loss: 0.2172 - acc: 0.9009 - val_loss: 0.2820 - val_acc: 0.8696\n",
      "Epoch 47/150\n",
      "Epoch 00047: val_loss did not improve\n",
      " - 6s - loss: 0.2013 - acc: 0.9138 - val_loss: 0.2703 - val_acc: 0.8758\n",
      "Epoch 48/150\n",
      "Epoch 00048: val_loss did not improve\n",
      " - 5s - loss: 0.2175 - acc: 0.9042 - val_loss: 0.2725 - val_acc: 0.8789\n",
      "Epoch 49/150\n",
      "Epoch 00049: val_loss did not improve\n",
      " - 6s - loss: 0.1991 - acc: 0.9161 - val_loss: 0.2789 - val_acc: 0.8820\n",
      "Epoch 50/150\n",
      "Epoch 00050: val_loss did not improve\n",
      " - 6s - loss: 0.2194 - acc: 0.8986 - val_loss: 0.2850 - val_acc: 0.8758\n",
      "Epoch 51/150\n",
      "Epoch 00051: val_loss did not improve\n",
      " - 6s - loss: 0.2034 - acc: 0.9169 - val_loss: 0.3025 - val_acc: 0.8665\n",
      "Epoch 52/150\n",
      "Epoch 00052: val_loss did not improve\n",
      " - 6s - loss: 0.1882 - acc: 0.9276 - val_loss: 0.2899 - val_acc: 0.8758\n",
      "Epoch 53/150\n",
      "Epoch 00053: val_loss did not improve\n",
      " - 6s - loss: 0.2098 - acc: 0.9161 - val_loss: 0.2814 - val_acc: 0.8758\n",
      "Epoch 54/150\n",
      "Epoch 00054: val_loss improved from 0.26724 to 0.26651, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.2206 - acc: 0.9001 - val_loss: 0.2665 - val_acc: 0.8758\n",
      "Epoch 55/150\n",
      "Epoch 00055: val_loss improved from 0.26651 to 0.26556, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.1950 - acc: 0.9161 - val_loss: 0.2656 - val_acc: 0.8851\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 56/150\n",
      "Epoch 00056: val_loss improved from 0.26556 to 0.25648, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.2174 - acc: 0.9088 - val_loss: 0.2565 - val_acc: 0.8882\n",
      "Epoch 57/150\n",
      "Epoch 00057: val_loss did not improve\n",
      " - 6s - loss: 0.1864 - acc: 0.9237 - val_loss: 0.2641 - val_acc: 0.8851\n",
      "Epoch 58/150\n",
      "Epoch 00058: val_loss did not improve\n",
      " - 6s - loss: 0.1986 - acc: 0.9222 - val_loss: 0.2620 - val_acc: 0.8851\n",
      "Epoch 59/150\n",
      "Epoch 00059: val_loss did not improve\n",
      " - 5s - loss: 0.1822 - acc: 0.9222 - val_loss: 0.2701 - val_acc: 0.8851\n",
      "Epoch 60/150\n",
      "Epoch 00060: val_loss improved from 0.25648 to 0.25325, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.1755 - acc: 0.9321 - val_loss: 0.2532 - val_acc: 0.8913\n",
      "Epoch 61/150\n",
      "Epoch 00061: val_loss did not improve\n",
      " - 6s - loss: 0.1822 - acc: 0.9306 - val_loss: 0.2627 - val_acc: 0.8913\n",
      "Epoch 62/150\n",
      "Epoch 00062: val_loss improved from 0.25325 to 0.25220, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.1845 - acc: 0.9253 - val_loss: 0.2522 - val_acc: 0.8975\n",
      "Epoch 63/150\n",
      "Epoch 00063: val_loss did not improve\n",
      " - 6s - loss: 0.1787 - acc: 0.9283 - val_loss: 0.2703 - val_acc: 0.8851\n",
      "Epoch 64/150\n",
      "Epoch 00064: val_loss did not improve\n",
      " - 6s - loss: 0.1938 - acc: 0.9108 - val_loss: 0.2538 - val_acc: 0.9006\n",
      "Epoch 65/150\n",
      "Epoch 00065: val_loss did not improve\n",
      " - 6s - loss: 0.1783 - acc: 0.9291 - val_loss: 0.2635 - val_acc: 0.8851\n",
      "Epoch 66/150\n",
      "Epoch 00066: val_loss did not improve\n",
      " - 6s - loss: 0.1833 - acc: 0.9283 - val_loss: 0.2712 - val_acc: 0.8882\n",
      "Epoch 67/150\n",
      "Epoch 00067: val_loss did not improve\n",
      " - 6s - loss: 0.1928 - acc: 0.9245 - val_loss: 0.2555 - val_acc: 0.8944\n",
      "Epoch 68/150\n",
      "Epoch 00068: val_loss did not improve\n",
      " - 6s - loss: 0.1779 - acc: 0.9337 - val_loss: 0.2613 - val_acc: 0.8944\n",
      "Epoch 69/150\n",
      "Epoch 00069: val_loss did not improve\n",
      " - 6s - loss: 0.1663 - acc: 0.9298 - val_loss: 0.2727 - val_acc: 0.8913\n",
      "Epoch 70/150\n",
      "Epoch 00070: val_loss did not improve\n",
      " - 6s - loss: 0.1703 - acc: 0.9306 - val_loss: 0.2714 - val_acc: 0.8789\n",
      "Epoch 71/150\n",
      "Epoch 00071: val_loss improved from 0.25220 to 0.24926, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage1.hdf5\n",
      " - 6s - loss: 0.2500 - acc: 0.9103 - val_loss: 0.2493 - val_acc: 0.9037\n",
      "Epoch 72/150\n",
      "Epoch 00072: val_loss did not improve\n",
      " - 6s - loss: 0.2356 - acc: 0.8940 - val_loss: 0.2727 - val_acc: 0.8851\n",
      "Epoch 73/150\n",
      "Epoch 00073: val_loss did not improve\n",
      " - 5s - loss: 0.1694 - acc: 0.9367 - val_loss: 0.2755 - val_acc: 0.8789\n",
      "Epoch 74/150\n",
      "Epoch 00074: val_loss did not improve\n",
      " - 6s - loss: 0.2065 - acc: 0.9092 - val_loss: 0.2650 - val_acc: 0.8820\n",
      "Epoch 75/150\n",
      "Epoch 00075: val_loss did not improve\n",
      " - 6s - loss: 0.1652 - acc: 0.9291 - val_loss: 0.2541 - val_acc: 0.8882\n",
      "Epoch 76/150\n",
      "Epoch 00076: val_loss did not improve\n",
      " - 6s - loss: 0.1545 - acc: 0.9382 - val_loss: 0.2684 - val_acc: 0.8882\n",
      "Epoch 77/150\n",
      "Epoch 00077: val_loss did not improve\n",
      " - 6s - loss: 0.1444 - acc: 0.9413 - val_loss: 0.2525 - val_acc: 0.8882\n",
      "Epoch 78/150\n",
      "Epoch 00078: val_loss did not improve\n",
      " - 6s - loss: 0.1586 - acc: 0.9413 - val_loss: 0.3077 - val_acc: 0.8820\n",
      "Epoch 79/150\n",
      "Epoch 00079: val_loss did not improve\n",
      " - 6s - loss: 0.1537 - acc: 0.9413 - val_loss: 0.2737 - val_acc: 0.9068\n",
      "Epoch 80/150\n",
      "Epoch 00080: val_loss did not improve\n",
      " - 6s - loss: 0.1399 - acc: 0.9352 - val_loss: 0.2881 - val_acc: 0.8789\n",
      "Epoch 81/150\n",
      "Epoch 00081: val_loss did not improve\n",
      " - 5s - loss: 0.1460 - acc: 0.9329 - val_loss: 0.2760 - val_acc: 0.9037\n",
      "Epoch 82/150\n",
      "Epoch 00082: val_loss did not improve\n",
      " - 6s - loss: 0.1398 - acc: 0.9398 - val_loss: 0.2935 - val_acc: 0.8851\n",
      "Epoch 83/150\n",
      "Epoch 00083: val_loss did not improve\n",
      " - 6s - loss: 0.1421 - acc: 0.9405 - val_loss: 0.2583 - val_acc: 0.9068\n",
      "Epoch 84/150\n",
      "Epoch 00084: val_loss did not improve\n",
      " - 6s - loss: 0.1683 - acc: 0.9271 - val_loss: 0.2950 - val_acc: 0.8820\n",
      "Epoch 85/150\n",
      "Epoch 00085: val_loss did not improve\n",
      " - 6s - loss: 0.1612 - acc: 0.9283 - val_loss: 0.2794 - val_acc: 0.8758\n",
      "Epoch 86/150\n",
      "Epoch 00086: val_loss did not improve\n",
      " - 6s - loss: 0.1520 - acc: 0.9375 - val_loss: 0.2896 - val_acc: 0.9037\n",
      "Epoch 87/150\n",
      "Epoch 00087: val_loss did not improve\n",
      " - 6s - loss: 0.1684 - acc: 0.9344 - val_loss: 0.2869 - val_acc: 0.9037\n",
      "Epoch 88/150\n",
      "Epoch 00088: val_loss did not improve\n",
      " - 5s - loss: 0.1404 - acc: 0.9443 - val_loss: 0.2852 - val_acc: 0.9068\n",
      "Epoch 89/150\n",
      "Epoch 00089: val_loss did not improve\n",
      " - 6s - loss: 0.1388 - acc: 0.9375 - val_loss: 0.2736 - val_acc: 0.9006\n",
      "Epoch 90/150\n",
      "Epoch 00090: val_loss did not improve\n",
      " - 6s - loss: 0.1402 - acc: 0.9355 - val_loss: 0.2639 - val_acc: 0.8944\n",
      "Epoch 91/150\n",
      "Epoch 00091: val_loss did not improve\n",
      " - 5s - loss: 0.1698 - acc: 0.9291 - val_loss: 0.3114 - val_acc: 0.8789\n",
      "Epoch 92/150\n",
      "Epoch 00092: val_loss did not improve\n",
      " - 6s - loss: 0.1559 - acc: 0.9367 - val_loss: 0.2567 - val_acc: 0.9099\n",
      "Epoch 93/150\n",
      "Epoch 00093: val_loss did not improve\n",
      " - 6s - loss: 0.1233 - acc: 0.9497 - val_loss: 0.2716 - val_acc: 0.9037\n",
      "Epoch 94/150\n",
      "Epoch 00094: val_loss did not improve\n",
      " - 6s - loss: 0.1359 - acc: 0.9393 - val_loss: 0.2789 - val_acc: 0.9037\n",
      "Epoch 95/150\n",
      "Epoch 00095: val_loss did not improve\n",
      " - 6s - loss: 0.1341 - acc: 0.9375 - val_loss: 0.2781 - val_acc: 0.9099\n",
      "Epoch 96/150\n",
      "Epoch 00096: val_loss did not improve\n",
      " - 6s - loss: 0.1235 - acc: 0.9527 - val_loss: 0.2619 - val_acc: 0.8975\n",
      "Epoch 97/150\n",
      "Epoch 00097: val_loss did not improve\n",
      " - 6s - loss: 0.1320 - acc: 0.9497 - val_loss: 0.2739 - val_acc: 0.9037\n",
      "Epoch 98/150\n",
      "Epoch 00098: val_loss did not improve\n",
      " - 6s - loss: 0.1193 - acc: 0.9497 - val_loss: 0.3121 - val_acc: 0.8820\n",
      "Epoch 99/150\n",
      "Epoch 00099: val_loss did not improve\n",
      " - 6s - loss: 0.1271 - acc: 0.9497 - val_loss: 0.2732 - val_acc: 0.9161\n",
      "Epoch 100/150\n",
      "Epoch 00100: val_loss did not improve\n",
      " - 6s - loss: 0.1272 - acc: 0.9512 - val_loss: 0.2897 - val_acc: 0.8944\n",
      "Epoch 101/150\n",
      "Epoch 00101: val_loss did not improve\n",
      " - 6s - loss: 0.1219 - acc: 0.9474 - val_loss: 0.3145 - val_acc: 0.8851\n",
      "CV1, Done!\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_17 (InputLayer)           (None, 75, 75, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_65 (Conv2D)              (None, 71, 71, 64)   4864        input_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_65 (Activation)      (None, 71, 71, 64)   0           conv2d_65[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_65 (MaxPooling2D) (None, 35, 35, 64)   0           activation_65[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_97 (Dropout)            (None, 35, 35, 64)   0           max_pooling2d_65[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_66 (Conv2D)              (None, 33, 33, 128)  73856       dropout_97[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_66 (Activation)      (None, 33, 33, 128)  0           conv2d_66[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_66 (MaxPooling2D) (None, 16, 16, 128)  0           activation_66[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_98 (Dropout)            (None, 16, 16, 128)  0           max_pooling2d_66[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_67 (Conv2D)              (None, 14, 14, 128)  147584      dropout_98[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_67 (Activation)      (None, 14, 14, 128)  0           conv2d_67[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_67 (MaxPooling2D) (None, 7, 7, 128)    0           activation_67[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_99 (Dropout)            (None, 7, 7, 128)    0           max_pooling2d_67[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_68 (Conv2D)              (None, 5, 5, 64)     73792       dropout_99[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_68 (Activation)      (None, 5, 5, 64)     0           conv2d_68[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_68 (MaxPooling2D) (None, 2, 2, 64)     0           activation_68[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "other (InputLayer)              (None, 13)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_100 (Dropout)           (None, 2, 2, 64)     0           max_pooling2d_68[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_81 (Dense)                (None, 16)           224         other[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_17 (Flatten)            (None, 256)          0           dropout_100[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_82 (Dense)                (None, 16)           272         dense_81[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_17 (Concatenate)    (None, 272)          0           flatten_17[0][0]                 \n",
      "                                                                 dense_82[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_83 (Dense)                (None, 256)          69888       concatenate_17[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_101 (Dropout)           (None, 256)          0           dense_83[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_84 (Dense)                (None, 256)          65792       dropout_101[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_102 (Dropout)           (None, 256)          0           dense_84[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_85 (Dense)                (None, 1)            257         dropout_102[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 436,529\n",
      "Trainable params: 431,665\n",
      "Non-trainable params: 4,864\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/150\n",
      "Epoch 00001: val_loss improved from inf to 0.26013, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage2.hdf5\n",
      " - 9s - loss: 0.2948 - acc: 0.8803 - val_loss: 0.2601 - val_acc: 0.8913\n",
      "Epoch 2/150\n",
      "Epoch 00002: val_loss improved from 0.26013 to 0.25624, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage2.hdf5\n",
      " - 4s - loss: 0.2683 - acc: 0.8826 - val_loss: 0.2562 - val_acc: 0.9037\n",
      "Epoch 3/150\n",
      "Epoch 00003: val_loss did not improve\n",
      " - 4s - loss: 0.2830 - acc: 0.8848 - val_loss: 0.2706 - val_acc: 0.8851\n",
      "Epoch 4/150\n",
      "Epoch 00004: val_loss did not improve\n",
      " - 4s - loss: 0.2536 - acc: 0.9009 - val_loss: 0.2636 - val_acc: 0.8882\n",
      "Epoch 5/150\n",
      "Epoch 00005: val_loss did not improve\n",
      " - 4s - loss: 0.2518 - acc: 0.8970 - val_loss: 0.2570 - val_acc: 0.8882\n",
      "Epoch 6/150\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 4s - loss: 0.2503 - acc: 0.8993 - val_loss: 0.2569 - val_acc: 0.8851\n",
      "Epoch 7/150\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 4s - loss: 0.2441 - acc: 0.8864 - val_loss: 0.2566 - val_acc: 0.8913\n",
      "Epoch 8/150\n",
      "Epoch 00008: val_loss improved from 0.25624 to 0.25505, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage2.hdf5\n",
      " - 4s - loss: 0.2402 - acc: 0.8948 - val_loss: 0.2551 - val_acc: 0.8913\n",
      "Epoch 9/150\n",
      "Epoch 00009: val_loss improved from 0.25505 to 0.25027, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage2.hdf5\n",
      " - 4s - loss: 0.2239 - acc: 0.9047 - val_loss: 0.2503 - val_acc: 0.8913\n",
      "Epoch 10/150\n",
      "Epoch 00010: val_loss improved from 0.25027 to 0.24484, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage2.hdf5\n",
      " - 4s - loss: 0.2322 - acc: 0.9039 - val_loss: 0.2448 - val_acc: 0.9006\n",
      "Epoch 11/150\n",
      "Epoch 00011: val_loss did not improve\n",
      " - 4s - loss: 0.2421 - acc: 0.8970 - val_loss: 0.2612 - val_acc: 0.8727\n",
      "Epoch 12/150\n",
      "Epoch 00012: val_loss did not improve\n",
      " - 4s - loss: 0.2391 - acc: 0.8948 - val_loss: 0.2599 - val_acc: 0.8851\n",
      "Epoch 13/150\n",
      "Epoch 00013: val_loss did not improve\n",
      " - 4s - loss: 0.2445 - acc: 0.8978 - val_loss: 0.2502 - val_acc: 0.8913\n",
      "Epoch 14/150\n",
      "Epoch 00014: val_loss did not improve\n",
      " - 4s - loss: 0.2318 - acc: 0.8986 - val_loss: 0.2482 - val_acc: 0.8851\n",
      "Epoch 15/150\n",
      "Epoch 00015: val_loss did not improve\n",
      " - 4s - loss: 0.2203 - acc: 0.9085 - val_loss: 0.2492 - val_acc: 0.8944\n",
      "Epoch 16/150\n",
      "Epoch 00016: val_loss did not improve\n",
      " - 4s - loss: 0.2158 - acc: 0.9176 - val_loss: 0.2597 - val_acc: 0.8882\n",
      "Epoch 17/150\n",
      "Epoch 00017: val_loss did not improve\n",
      " - 4s - loss: 0.2319 - acc: 0.8948 - val_loss: 0.2530 - val_acc: 0.8851\n",
      "Epoch 18/150\n",
      "Epoch 00018: val_loss improved from 0.24484 to 0.24463, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv1_stage2.hdf5\n",
      " - 4s - loss: 0.2431 - acc: 0.9042 - val_loss: 0.2446 - val_acc: 0.8913\n",
      "Epoch 19/150\n",
      "Epoch 00019: val_loss did not improve\n",
      " - 4s - loss: 0.2437 - acc: 0.8897 - val_loss: 0.2574 - val_acc: 0.8882\n",
      "Epoch 20/150\n",
      "Epoch 00020: val_loss did not improve\n",
      " - 4s - loss: 0.2166 - acc: 0.9108 - val_loss: 0.2543 - val_acc: 0.8851\n",
      "Epoch 21/150\n",
      "Epoch 00021: val_loss did not improve\n",
      " - 4s - loss: 0.2151 - acc: 0.9062 - val_loss: 0.2768 - val_acc: 0.8665\n",
      "Epoch 22/150\n",
      "Epoch 00022: val_loss did not improve\n",
      " - 4s - loss: 0.2247 - acc: 0.9131 - val_loss: 0.2511 - val_acc: 0.8975\n",
      "Epoch 23/150\n",
      "Epoch 00023: val_loss did not improve\n",
      " - 4s - loss: 0.2099 - acc: 0.8966 - val_loss: 0.2492 - val_acc: 0.8944\n",
      "Epoch 24/150\n",
      "Epoch 00024: val_loss did not improve\n",
      " - 4s - loss: 0.1974 - acc: 0.9215 - val_loss: 0.2587 - val_acc: 0.8851\n",
      "Epoch 25/150\n",
      "Epoch 00025: val_loss did not improve\n",
      " - 5s - loss: 0.2125 - acc: 0.9062 - val_loss: 0.2510 - val_acc: 0.8944\n",
      "Epoch 26/150\n",
      "Epoch 00026: val_loss did not improve\n",
      " - 4s - loss: 0.2044 - acc: 0.9092 - val_loss: 0.2523 - val_acc: 0.8944\n",
      "Epoch 27/150\n",
      "Epoch 00027: val_loss did not improve\n",
      " - 4s - loss: 0.2024 - acc: 0.9085 - val_loss: 0.2577 - val_acc: 0.8820\n",
      "Epoch 28/150\n",
      "Epoch 00028: val_loss did not improve\n",
      " - 4s - loss: 0.2037 - acc: 0.9146 - val_loss: 0.2447 - val_acc: 0.8913\n",
      "Epoch 29/150\n",
      "Epoch 00029: val_loss did not improve\n",
      " - 4s - loss: 0.1796 - acc: 0.9283 - val_loss: 0.2508 - val_acc: 0.8913\n",
      "Epoch 30/150\n",
      "Epoch 00030: val_loss did not improve\n",
      " - 4s - loss: 0.2059 - acc: 0.9161 - val_loss: 0.2466 - val_acc: 0.8944\n",
      "Epoch 31/150\n",
      "Epoch 00031: val_loss did not improve\n",
      " - 4s - loss: 0.2013 - acc: 0.9184 - val_loss: 0.2544 - val_acc: 0.8851\n",
      "Epoch 32/150\n",
      "Epoch 00032: val_loss did not improve\n",
      " - 4s - loss: 0.1844 - acc: 0.9138 - val_loss: 0.2841 - val_acc: 0.8758\n",
      "Epoch 33/150\n",
      "Epoch 00033: val_loss did not improve\n",
      " - 4s - loss: 0.1945 - acc: 0.9123 - val_loss: 0.2572 - val_acc: 0.8944\n",
      "Epoch 34/150\n",
      "Epoch 00034: val_loss did not improve\n",
      " - 4s - loss: 0.1834 - acc: 0.9276 - val_loss: 0.2488 - val_acc: 0.8975\n",
      "Epoch 35/150\n",
      "Epoch 00035: val_loss did not improve\n",
      " - 4s - loss: 0.1899 - acc: 0.9050 - val_loss: 0.2604 - val_acc: 0.8975\n",
      "Epoch 36/150\n",
      "Epoch 00036: val_loss did not improve\n",
      " - 4s - loss: 0.2008 - acc: 0.9176 - val_loss: 0.2528 - val_acc: 0.8882\n",
      "Epoch 37/150\n",
      "Epoch 00037: val_loss did not improve\n",
      " - 4s - loss: 0.1812 - acc: 0.9260 - val_loss: 0.2747 - val_acc: 0.8882\n",
      "Epoch 38/150\n",
      "Epoch 00038: val_loss did not improve\n",
      " - 4s - loss: 0.1864 - acc: 0.9260 - val_loss: 0.2537 - val_acc: 0.8975\n",
      "Epoch 39/150\n",
      "Epoch 00039: val_loss did not improve\n",
      " - 4s - loss: 0.1787 - acc: 0.9253 - val_loss: 0.2594 - val_acc: 0.8944\n",
      "Epoch 40/150\n",
      "Epoch 00040: val_loss did not improve\n",
      " - 4s - loss: 0.1907 - acc: 0.9199 - val_loss: 0.2602 - val_acc: 0.8913\n",
      "Epoch 41/150\n",
      "Epoch 00041: val_loss did not improve\n",
      " - 4s - loss: 0.1895 - acc: 0.9276 - val_loss: 0.2523 - val_acc: 0.8913\n",
      "Epoch 42/150\n",
      "Epoch 00042: val_loss did not improve\n",
      " - 5s - loss: 0.1788 - acc: 0.9314 - val_loss: 0.2671 - val_acc: 0.8820\n",
      "Epoch 43/150\n",
      "Epoch 00043: val_loss did not improve\n",
      " - 4s - loss: 0.2031 - acc: 0.9138 - val_loss: 0.2556 - val_acc: 0.8913\n",
      "Epoch 44/150\n",
      "Epoch 00044: val_loss did not improve\n",
      " - 4s - loss: 0.1976 - acc: 0.9192 - val_loss: 0.2631 - val_acc: 0.8882\n",
      "Epoch 45/150\n",
      "Epoch 00045: val_loss did not improve\n",
      " - 4s - loss: 0.1747 - acc: 0.9298 - val_loss: 0.2607 - val_acc: 0.8913\n",
      "Epoch 46/150\n",
      "Epoch 00046: val_loss did not improve\n",
      " - 4s - loss: 0.1767 - acc: 0.9237 - val_loss: 0.2907 - val_acc: 0.8913\n",
      "Epoch 47/150\n",
      "Epoch 00047: val_loss did not improve\n",
      " - 4s - loss: 0.1737 - acc: 0.9268 - val_loss: 0.2616 - val_acc: 0.8851\n",
      "Epoch 48/150\n",
      "Epoch 00048: val_loss did not improve\n",
      " - 4s - loss: 0.2082 - acc: 0.9045 - val_loss: 0.2712 - val_acc: 0.8820\n",
      "CV1, Done!\n",
      "1282/1282 [==============================] - 1s 1ms/step\n",
      "Train loss: 0.141103041711\n",
      "Train accuracy: 0.949297971919\n",
      "322/322 [==============================] - 0s 1ms/step\n",
      "Valid loss: 0.244628826469\n",
      "Valid accuracy: 0.891304347826\n",
      "\n",
      "===================FOLD= 2\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_18 (InputLayer)           (None, 75, 75, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_69 (Conv2D)              (None, 71, 71, 64)   4864        input_18[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_69 (Activation)      (None, 71, 71, 64)   0           conv2d_69[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_69 (MaxPooling2D) (None, 35, 35, 64)   0           activation_69[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_103 (Dropout)           (None, 35, 35, 64)   0           max_pooling2d_69[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_70 (Conv2D)              (None, 33, 33, 128)  73856       dropout_103[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_70 (Activation)      (None, 33, 33, 128)  0           conv2d_70[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_70 (MaxPooling2D) (None, 16, 16, 128)  0           activation_70[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_104 (Dropout)           (None, 16, 16, 128)  0           max_pooling2d_70[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_71 (Conv2D)              (None, 14, 14, 128)  147584      dropout_104[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_71 (Activation)      (None, 14, 14, 128)  0           conv2d_71[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_71 (MaxPooling2D) (None, 7, 7, 128)    0           activation_71[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_105 (Dropout)           (None, 7, 7, 128)    0           max_pooling2d_71[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_72 (Conv2D)              (None, 5, 5, 64)     73792       dropout_105[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_72 (Activation)      (None, 5, 5, 64)     0           conv2d_72[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_72 (MaxPooling2D) (None, 2, 2, 64)     0           activation_72[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "other (InputLayer)              (None, 13)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_106 (Dropout)           (None, 2, 2, 64)     0           max_pooling2d_72[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_86 (Dense)                (None, 16)           224         other[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_18 (Flatten)            (None, 256)          0           dropout_106[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_87 (Dense)                (None, 16)           272         dense_86[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_18 (Concatenate)    (None, 272)          0           flatten_18[0][0]                 \n",
      "                                                                 dense_87[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_88 (Dense)                (None, 256)          69888       concatenate_18[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_107 (Dropout)           (None, 256)          0           dense_88[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_89 (Dense)                (None, 256)          65792       dropout_107[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_108 (Dropout)           (None, 256)          0           dense_89[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_90 (Dense)                (None, 1)            257         dropout_108[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 436,529\n",
      "Trainable params: 436,529\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/150\n",
      "Epoch 00001: val_loss improved from inf to 0.67776, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 12s - loss: 1.5353 - acc: 0.5196 - val_loss: 0.6778 - val_acc: 0.6262\n",
      "Epoch 2/150\n",
      "Epoch 00002: val_loss improved from 0.67776 to 0.65124, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.7741 - acc: 0.5010 - val_loss: 0.6512 - val_acc: 0.6199\n",
      "Epoch 3/150\n",
      "Epoch 00003: val_loss improved from 0.65124 to 0.61644, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.6853 - acc: 0.5570 - val_loss: 0.6164 - val_acc: 0.5358\n",
      "Epoch 4/150\n",
      "Epoch 00004: val_loss improved from 0.61644 to 0.58250, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.6553 - acc: 0.5730 - val_loss: 0.5825 - val_acc: 0.6542\n",
      "Epoch 5/150\n",
      "Epoch 00005: val_loss improved from 0.58250 to 0.56530, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 6s - loss: 0.6089 - acc: 0.6179 - val_loss: 0.5653 - val_acc: 0.7040\n",
      "Epoch 6/150\n",
      "Epoch 00006: val_loss improved from 0.56530 to 0.55399, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 6s - loss: 0.6119 - acc: 0.6008 - val_loss: 0.5540 - val_acc: 0.6573\n",
      "Epoch 7/150\n",
      "Epoch 00007: val_loss improved from 0.55399 to 0.53416, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.5860 - acc: 0.6394 - val_loss: 0.5342 - val_acc: 0.7165\n",
      "Epoch 8/150\n",
      "Epoch 00008: val_loss improved from 0.53416 to 0.50956, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.5673 - acc: 0.6524 - val_loss: 0.5096 - val_acc: 0.7321\n",
      "Epoch 9/150\n",
      "Epoch 00009: val_loss improved from 0.50956 to 0.49904, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.5579 - acc: 0.6672 - val_loss: 0.4990 - val_acc: 0.7726\n",
      "Epoch 10/150\n",
      "Epoch 00010: val_loss improved from 0.49904 to 0.46665, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 6s - loss: 0.5273 - acc: 0.7106 - val_loss: 0.4667 - val_acc: 0.7913\n",
      "Epoch 11/150\n",
      "Epoch 00011: val_loss improved from 0.46665 to 0.44638, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 6s - loss: 0.5133 - acc: 0.7149 - val_loss: 0.4464 - val_acc: 0.8006\n",
      "Epoch 12/150\n",
      "Epoch 00012: val_loss improved from 0.44638 to 0.39691, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.4801 - acc: 0.7602 - val_loss: 0.3969 - val_acc: 0.8162\n",
      "Epoch 13/150\n",
      "Epoch 00013: val_loss improved from 0.39691 to 0.37557, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.4510 - acc: 0.7659 - val_loss: 0.3756 - val_acc: 0.8131\n",
      "Epoch 14/150\n",
      "Epoch 00014: val_loss did not improve\n",
      " - 5s - loss: 0.4524 - acc: 0.7660 - val_loss: 0.3777 - val_acc: 0.8536\n",
      "Epoch 15/150\n",
      "Epoch 00015: val_loss improved from 0.37557 to 0.33778, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.4206 - acc: 0.7938 - val_loss: 0.3378 - val_acc: 0.8505\n",
      "Epoch 16/150\n",
      "Epoch 00016: val_loss did not improve\n",
      " - 5s - loss: 0.3988 - acc: 0.8055 - val_loss: 0.3431 - val_acc: 0.8567\n",
      "Epoch 17/150\n",
      "Epoch 00017: val_loss improved from 0.33778 to 0.28859, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 6s - loss: 0.3727 - acc: 0.8208 - val_loss: 0.2886 - val_acc: 0.8723\n",
      "Epoch 18/150\n",
      "Epoch 00018: val_loss did not improve\n",
      " - 6s - loss: 0.3827 - acc: 0.8170 - val_loss: 0.3242 - val_acc: 0.8879\n",
      "Epoch 19/150\n",
      "Epoch 00019: val_loss did not improve\n",
      " - 5s - loss: 0.3367 - acc: 0.8433 - val_loss: 0.3088 - val_acc: 0.8816\n",
      "Epoch 20/150\n",
      "Epoch 00020: val_loss improved from 0.28859 to 0.28728, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.3509 - acc: 0.8388 - val_loss: 0.2873 - val_acc: 0.8972\n",
      "Epoch 21/150\n",
      "Epoch 00021: val_loss improved from 0.28728 to 0.25217, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.3452 - acc: 0.8365 - val_loss: 0.2522 - val_acc: 0.8972\n",
      "Epoch 22/150\n",
      "Epoch 00022: val_loss did not improve\n",
      " - 5s - loss: 0.3382 - acc: 0.8391 - val_loss: 0.2788 - val_acc: 0.9065\n",
      "Epoch 23/150\n",
      "Epoch 00023: val_loss did not improve\n",
      " - 5s - loss: 0.3243 - acc: 0.8540 - val_loss: 0.2583 - val_acc: 0.9034\n",
      "Epoch 24/150\n",
      "Epoch 00024: val_loss did not improve\n",
      " - 5s - loss: 0.3295 - acc: 0.8543 - val_loss: 0.3047 - val_acc: 0.8692\n",
      "Epoch 25/150\n",
      "Epoch 00025: val_loss improved from 0.25217 to 0.24275, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.3216 - acc: 0.8476 - val_loss: 0.2428 - val_acc: 0.9128\n",
      "Epoch 26/150\n",
      "Epoch 00026: val_loss did not improve\n",
      " - 5s - loss: 0.3095 - acc: 0.8597 - val_loss: 0.2544 - val_acc: 0.8692\n",
      "Epoch 27/150\n",
      "Epoch 00027: val_loss improved from 0.24275 to 0.24139, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.3307 - acc: 0.8551 - val_loss: 0.2414 - val_acc: 0.9003\n",
      "Epoch 28/150\n",
      "Epoch 00028: val_loss improved from 0.24139 to 0.23670, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.2861 - acc: 0.8749 - val_loss: 0.2367 - val_acc: 0.9065\n",
      "Epoch 29/150\n",
      "Epoch 00029: val_loss improved from 0.23670 to 0.22064, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 6s - loss: 0.3228 - acc: 0.8502 - val_loss: 0.2206 - val_acc: 0.9065\n",
      "Epoch 30/150\n",
      "Epoch 00030: val_loss did not improve\n",
      " - 5s - loss: 0.3058 - acc: 0.8566 - val_loss: 0.2296 - val_acc: 0.9097\n",
      "Epoch 31/150\n",
      "Epoch 00031: val_loss did not improve\n",
      " - 6s - loss: 0.2943 - acc: 0.8685 - val_loss: 0.2421 - val_acc: 0.9034\n",
      "Epoch 32/150\n",
      "Epoch 00032: val_loss did not improve\n",
      " - 5s - loss: 0.2996 - acc: 0.8601 - val_loss: 0.2224 - val_acc: 0.9128\n",
      "Epoch 33/150\n",
      "Epoch 00033: val_loss improved from 0.22064 to 0.21630, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.2746 - acc: 0.8826 - val_loss: 0.2163 - val_acc: 0.9159\n",
      "Epoch 34/150\n",
      "Epoch 00034: val_loss did not improve\n",
      " - 5s - loss: 0.2611 - acc: 0.8864 - val_loss: 0.2247 - val_acc: 0.9128\n",
      "Epoch 35/150\n",
      "Epoch 00035: val_loss did not improve\n",
      " - 5s - loss: 0.2541 - acc: 0.8899 - val_loss: 0.2798 - val_acc: 0.8754\n",
      "Epoch 36/150\n",
      "Epoch 00036: val_loss did not improve\n",
      " - 5s - loss: 0.2989 - acc: 0.8543 - val_loss: 0.2344 - val_acc: 0.9065\n",
      "Epoch 37/150\n",
      "Epoch 00037: val_loss improved from 0.21630 to 0.21115, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.2728 - acc: 0.8749 - val_loss: 0.2112 - val_acc: 0.9128\n",
      "Epoch 38/150\n",
      "Epoch 00038: val_loss did not improve\n",
      " - 5s - loss: 0.2584 - acc: 0.8815 - val_loss: 0.2169 - val_acc: 0.9252\n",
      "Epoch 39/150\n",
      "Epoch 00039: val_loss did not improve\n",
      " - 5s - loss: 0.2806 - acc: 0.8757 - val_loss: 0.2243 - val_acc: 0.9097\n",
      "Epoch 40/150\n",
      "Epoch 00040: val_loss did not improve\n",
      " - 5s - loss: 0.2415 - acc: 0.8970 - val_loss: 0.2287 - val_acc: 0.9003\n",
      "Epoch 41/150\n",
      "Epoch 00041: val_loss improved from 0.21115 to 0.20064, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 6s - loss: 0.2586 - acc: 0.8780 - val_loss: 0.2006 - val_acc: 0.9097\n",
      "Epoch 42/150\n",
      "Epoch 00042: val_loss improved from 0.20064 to 0.20001, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 6s - loss: 0.2286 - acc: 0.8963 - val_loss: 0.2000 - val_acc: 0.9159\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 43/150\n",
      "Epoch 00043: val_loss did not improve\n",
      " - 5s - loss: 0.2344 - acc: 0.8993 - val_loss: 0.2057 - val_acc: 0.9097\n",
      "Epoch 44/150\n",
      "Epoch 00044: val_loss did not improve\n",
      " - 5s - loss: 0.2537 - acc: 0.8853 - val_loss: 0.2096 - val_acc: 0.9097\n",
      "Epoch 45/150\n",
      "Epoch 00045: val_loss did not improve\n",
      " - 5s - loss: 0.2513 - acc: 0.8685 - val_loss: 0.2025 - val_acc: 0.9221\n",
      "Epoch 46/150\n",
      "Epoch 00046: val_loss improved from 0.20001 to 0.19294, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 6s - loss: 0.2296 - acc: 0.9024 - val_loss: 0.1929 - val_acc: 0.9221\n",
      "Epoch 47/150\n",
      "Epoch 00047: val_loss did not improve\n",
      " - 5s - loss: 0.2336 - acc: 0.9013 - val_loss: 0.2122 - val_acc: 0.9097\n",
      "Epoch 48/150\n",
      "Epoch 00048: val_loss did not improve\n",
      " - 5s - loss: 0.2539 - acc: 0.8772 - val_loss: 0.2072 - val_acc: 0.9034\n",
      "Epoch 49/150\n",
      "Epoch 00049: val_loss did not improve\n",
      " - 5s - loss: 0.2257 - acc: 0.9051 - val_loss: 0.2070 - val_acc: 0.9159\n",
      "Epoch 50/150\n",
      "Epoch 00050: val_loss improved from 0.19294 to 0.19292, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 6s - loss: 0.2493 - acc: 0.8848 - val_loss: 0.1929 - val_acc: 0.9283\n",
      "Epoch 51/150\n",
      "Epoch 00051: val_loss did not improve\n",
      " - 5s - loss: 0.2461 - acc: 0.8864 - val_loss: 0.1992 - val_acc: 0.9315\n",
      "Epoch 52/150\n",
      "Epoch 00052: val_loss did not improve\n",
      " - 5s - loss: 0.2116 - acc: 0.9039 - val_loss: 0.2045 - val_acc: 0.9190\n",
      "Epoch 53/150\n",
      "Epoch 00053: val_loss improved from 0.19292 to 0.18761, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.2132 - acc: 0.8986 - val_loss: 0.1876 - val_acc: 0.9252\n",
      "Epoch 54/150\n",
      "Epoch 00054: val_loss did not improve\n",
      " - 5s - loss: 0.2087 - acc: 0.9085 - val_loss: 0.2051 - val_acc: 0.9034\n",
      "Epoch 55/150\n",
      "Epoch 00055: val_loss did not improve\n",
      " - 5s - loss: 0.2152 - acc: 0.9028 - val_loss: 0.1882 - val_acc: 0.9190\n",
      "Epoch 56/150\n",
      "Epoch 00056: val_loss did not improve\n",
      " - 5s - loss: 0.2115 - acc: 0.9146 - val_loss: 0.1980 - val_acc: 0.9252\n",
      "Epoch 57/150\n",
      "Epoch 00057: val_loss did not improve\n",
      " - 5s - loss: 0.2182 - acc: 0.9077 - val_loss: 0.1917 - val_acc: 0.9315\n",
      "Epoch 58/150\n",
      "Epoch 00058: val_loss did not improve\n",
      " - 5s - loss: 0.2112 - acc: 0.9085 - val_loss: 0.1892 - val_acc: 0.9315\n",
      "Epoch 59/150\n",
      "Epoch 00059: val_loss improved from 0.18761 to 0.18375, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 6s - loss: 0.1895 - acc: 0.9161 - val_loss: 0.1837 - val_acc: 0.9283\n",
      "Epoch 60/150\n",
      "Epoch 00060: val_loss did not improve\n",
      " - 5s - loss: 0.1915 - acc: 0.9204 - val_loss: 0.1967 - val_acc: 0.9128\n",
      "Epoch 61/150\n",
      "Epoch 00061: val_loss did not improve\n",
      " - 5s - loss: 0.2213 - acc: 0.9039 - val_loss: 0.1843 - val_acc: 0.9190\n",
      "Epoch 62/150\n",
      "Epoch 00062: val_loss improved from 0.18375 to 0.18345, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 5s - loss: 0.1845 - acc: 0.9207 - val_loss: 0.1834 - val_acc: 0.9159\n",
      "Epoch 63/150\n",
      "Epoch 00063: val_loss did not improve\n",
      " - 5s - loss: 0.1746 - acc: 0.9199 - val_loss: 0.1991 - val_acc: 0.9159\n",
      "Epoch 64/150\n",
      "Epoch 00064: val_loss did not improve\n",
      " - 5s - loss: 0.1931 - acc: 0.9176 - val_loss: 0.2171 - val_acc: 0.9221\n",
      "Epoch 65/150\n",
      "Epoch 00065: val_loss did not improve\n",
      " - 5s - loss: 0.1946 - acc: 0.9199 - val_loss: 0.1897 - val_acc: 0.9283\n",
      "Epoch 66/150\n",
      "Epoch 00066: val_loss improved from 0.18345 to 0.17815, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 6s - loss: 0.1777 - acc: 0.9154 - val_loss: 0.1782 - val_acc: 0.9221\n",
      "Epoch 67/150\n",
      "Epoch 00067: val_loss improved from 0.17815 to 0.17759, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 6s - loss: 0.1797 - acc: 0.9237 - val_loss: 0.1776 - val_acc: 0.9346\n",
      "Epoch 68/150\n",
      "Epoch 00068: val_loss did not improve\n",
      " - 5s - loss: 0.2071 - acc: 0.9123 - val_loss: 0.1797 - val_acc: 0.9190\n",
      "Epoch 69/150\n",
      "Epoch 00069: val_loss did not improve\n",
      " - 5s - loss: 0.1794 - acc: 0.9199 - val_loss: 0.1868 - val_acc: 0.9377\n",
      "Epoch 70/150\n",
      "Epoch 00070: val_loss did not improve\n",
      " - 5s - loss: 0.1615 - acc: 0.9260 - val_loss: 0.1966 - val_acc: 0.9315\n",
      "Epoch 71/150\n",
      "Epoch 00071: val_loss did not improve\n",
      " - 5s - loss: 0.1745 - acc: 0.9245 - val_loss: 0.1886 - val_acc: 0.9065\n",
      "Epoch 72/150\n",
      "Epoch 00072: val_loss did not improve\n",
      " - 6s - loss: 0.1678 - acc: 0.9215 - val_loss: 0.2243 - val_acc: 0.8847\n",
      "Epoch 73/150\n",
      "Epoch 00073: val_loss improved from 0.17759 to 0.17720, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 6s - loss: 0.1839 - acc: 0.9260 - val_loss: 0.1772 - val_acc: 0.9377\n",
      "Epoch 74/150\n",
      "Epoch 00074: val_loss improved from 0.17720 to 0.17435, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage1.hdf5\n",
      " - 6s - loss: 0.1772 - acc: 0.9207 - val_loss: 0.1744 - val_acc: 0.9377\n",
      "Epoch 75/150\n",
      "Epoch 00075: val_loss did not improve\n",
      " - 5s - loss: 0.1666 - acc: 0.9352 - val_loss: 0.2087 - val_acc: 0.9252\n",
      "Epoch 76/150\n",
      "Epoch 00076: val_loss did not improve\n",
      " - 6s - loss: 0.1846 - acc: 0.9138 - val_loss: 0.2137 - val_acc: 0.9003\n",
      "Epoch 77/150\n",
      "Epoch 00077: val_loss did not improve\n",
      " - 5s - loss: 0.1898 - acc: 0.9135 - val_loss: 0.1977 - val_acc: 0.9034\n",
      "Epoch 78/150\n",
      "Epoch 00078: val_loss did not improve\n",
      " - 5s - loss: 0.1862 - acc: 0.9184 - val_loss: 0.1835 - val_acc: 0.9190\n",
      "Epoch 79/150\n",
      "Epoch 00079: val_loss did not improve\n",
      " - 5s - loss: 0.1652 - acc: 0.9268 - val_loss: 0.1876 - val_acc: 0.9221\n",
      "Epoch 80/150\n",
      "Epoch 00080: val_loss did not improve\n",
      " - 5s - loss: 0.1825 - acc: 0.9272 - val_loss: 0.1872 - val_acc: 0.9159\n",
      "Epoch 81/150\n",
      "Epoch 00081: val_loss did not improve\n",
      " - 5s - loss: 0.1593 - acc: 0.9276 - val_loss: 0.2170 - val_acc: 0.8941\n",
      "Epoch 82/150\n",
      "Epoch 00082: val_loss did not improve\n",
      " - 6s - loss: 0.1759 - acc: 0.9135 - val_loss: 0.1984 - val_acc: 0.9034\n",
      "Epoch 83/150\n",
      "Epoch 00083: val_loss did not improve\n",
      " - 6s - loss: 0.1567 - acc: 0.9291 - val_loss: 0.2083 - val_acc: 0.9097\n",
      "Epoch 84/150\n",
      "Epoch 00084: val_loss did not improve\n",
      " - 6s - loss: 0.1808 - acc: 0.9188 - val_loss: 0.1875 - val_acc: 0.9252\n",
      "Epoch 85/150\n",
      "Epoch 00085: val_loss did not improve\n",
      " - 6s - loss: 0.1749 - acc: 0.9199 - val_loss: 0.2020 - val_acc: 0.9159\n",
      "Epoch 86/150\n",
      "Epoch 00086: val_loss did not improve\n",
      " - 6s - loss: 0.1575 - acc: 0.9276 - val_loss: 0.1905 - val_acc: 0.9159\n",
      "Epoch 87/150\n",
      "Epoch 00087: val_loss did not improve\n",
      " - 5s - loss: 0.1571 - acc: 0.9375 - val_loss: 0.1913 - val_acc: 0.9377\n",
      "Epoch 88/150\n",
      "Epoch 00088: val_loss did not improve\n",
      " - 5s - loss: 0.1516 - acc: 0.9257 - val_loss: 0.1863 - val_acc: 0.9283\n",
      "Epoch 89/150\n",
      "Epoch 00089: val_loss did not improve\n",
      " - 5s - loss: 0.1548 - acc: 0.9398 - val_loss: 0.1972 - val_acc: 0.9252\n",
      "Epoch 90/150\n",
      "Epoch 00090: val_loss did not improve\n",
      " - 5s - loss: 0.1558 - acc: 0.9265 - val_loss: 0.2870 - val_acc: 0.9128\n",
      "Epoch 91/150\n",
      "Epoch 00091: val_loss did not improve\n",
      " - 5s - loss: 0.1641 - acc: 0.9260 - val_loss: 0.2106 - val_acc: 0.9252\n",
      "Epoch 92/150\n",
      "Epoch 00092: val_loss did not improve\n",
      " - 5s - loss: 0.1590 - acc: 0.9314 - val_loss: 0.1813 - val_acc: 0.9346\n",
      "Epoch 93/150\n",
      "Epoch 00093: val_loss did not improve\n",
      " - 5s - loss: 0.1597 - acc: 0.9333 - val_loss: 0.2436 - val_acc: 0.8972\n",
      "Epoch 94/150\n",
      "Epoch 00094: val_loss did not improve\n",
      " - 6s - loss: 0.1625 - acc: 0.9268 - val_loss: 0.1973 - val_acc: 0.9065\n",
      "Epoch 95/150\n",
      "Epoch 00095: val_loss did not improve\n",
      " - 6s - loss: 0.1324 - acc: 0.9466 - val_loss: 0.1842 - val_acc: 0.9315\n",
      "Epoch 96/150\n",
      "Epoch 00096: val_loss did not improve\n",
      " - 5s - loss: 0.1251 - acc: 0.9512 - val_loss: 0.1965 - val_acc: 0.9283\n",
      "Epoch 97/150\n",
      "Epoch 00097: val_loss did not improve\n",
      " - 5s - loss: 0.1471 - acc: 0.9364 - val_loss: 0.1947 - val_acc: 0.9252\n",
      "Epoch 98/150\n",
      "Epoch 00098: val_loss did not improve\n",
      " - 5s - loss: 0.1529 - acc: 0.9310 - val_loss: 0.2046 - val_acc: 0.9252\n",
      "Epoch 99/150\n",
      "Epoch 00099: val_loss did not improve\n",
      " - 5s - loss: 0.2042 - acc: 0.9100 - val_loss: 0.2120 - val_acc: 0.9315\n",
      "Epoch 100/150\n",
      "Epoch 00100: val_loss did not improve\n",
      " - 5s - loss: 0.1459 - acc: 0.9367 - val_loss: 0.1859 - val_acc: 0.9315\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 101/150\n",
      "Epoch 00101: val_loss did not improve\n",
      " - 5s - loss: 0.1403 - acc: 0.9455 - val_loss: 0.2032 - val_acc: 0.9221\n",
      "Epoch 102/150\n",
      "Epoch 00102: val_loss did not improve\n",
      " - 5s - loss: 0.1297 - acc: 0.9512 - val_loss: 0.2086 - val_acc: 0.9190\n",
      "Epoch 103/150\n",
      "Epoch 00103: val_loss did not improve\n",
      " - 5s - loss: 0.1275 - acc: 0.9481 - val_loss: 0.2047 - val_acc: 0.9221\n",
      "Epoch 104/150\n",
      "Epoch 00104: val_loss did not improve\n",
      " - 5s - loss: 0.1288 - acc: 0.9451 - val_loss: 0.1870 - val_acc: 0.9283\n",
      "CV2, Done!\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_19 (InputLayer)           (None, 75, 75, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_73 (Conv2D)              (None, 71, 71, 64)   4864        input_19[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_73 (Activation)      (None, 71, 71, 64)   0           conv2d_73[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_73 (MaxPooling2D) (None, 35, 35, 64)   0           activation_73[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_109 (Dropout)           (None, 35, 35, 64)   0           max_pooling2d_73[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_74 (Conv2D)              (None, 33, 33, 128)  73856       dropout_109[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_74 (Activation)      (None, 33, 33, 128)  0           conv2d_74[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_74 (MaxPooling2D) (None, 16, 16, 128)  0           activation_74[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_110 (Dropout)           (None, 16, 16, 128)  0           max_pooling2d_74[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_75 (Conv2D)              (None, 14, 14, 128)  147584      dropout_110[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_75 (Activation)      (None, 14, 14, 128)  0           conv2d_75[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_75 (MaxPooling2D) (None, 7, 7, 128)    0           activation_75[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_111 (Dropout)           (None, 7, 7, 128)    0           max_pooling2d_75[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_76 (Conv2D)              (None, 5, 5, 64)     73792       dropout_111[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_76 (Activation)      (None, 5, 5, 64)     0           conv2d_76[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_76 (MaxPooling2D) (None, 2, 2, 64)     0           activation_76[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "other (InputLayer)              (None, 13)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_112 (Dropout)           (None, 2, 2, 64)     0           max_pooling2d_76[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_91 (Dense)                (None, 16)           224         other[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_19 (Flatten)            (None, 256)          0           dropout_112[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_92 (Dense)                (None, 16)           272         dense_91[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_19 (Concatenate)    (None, 272)          0           flatten_19[0][0]                 \n",
      "                                                                 dense_92[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_93 (Dense)                (None, 256)          69888       concatenate_19[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_113 (Dropout)           (None, 256)          0           dense_93[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_94 (Dense)                (None, 256)          65792       dropout_113[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_114 (Dropout)           (None, 256)          0           dense_94[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_95 (Dense)                (None, 1)            257         dropout_114[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 436,529\n",
      "Trainable params: 431,665\n",
      "Non-trainable params: 4,864\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/150\n",
      "Epoch 00001: val_loss improved from inf to 0.21622, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage2.hdf5\n",
      " - 10s - loss: 0.3304 - acc: 0.8612 - val_loss: 0.2162 - val_acc: 0.9159\n",
      "Epoch 2/150\n",
      "Epoch 00002: val_loss improved from 0.21622 to 0.19297, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage2.hdf5\n",
      " - 4s - loss: 0.2749 - acc: 0.8719 - val_loss: 0.1930 - val_acc: 0.9221\n",
      "Epoch 3/150\n",
      "Epoch 00003: val_loss did not improve\n",
      " - 4s - loss: 0.2734 - acc: 0.8787 - val_loss: 0.2020 - val_acc: 0.9159\n",
      "Epoch 4/150\n",
      "Epoch 00004: val_loss did not improve\n",
      " - 4s - loss: 0.2927 - acc: 0.8705 - val_loss: 0.1973 - val_acc: 0.9283\n",
      "Epoch 5/150\n",
      "Epoch 00005: val_loss did not improve\n",
      " - 4s - loss: 0.2541 - acc: 0.8902 - val_loss: 0.2031 - val_acc: 0.9252\n",
      "Epoch 6/150\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 4s - loss: 0.2661 - acc: 0.8871 - val_loss: 0.2012 - val_acc: 0.9221\n",
      "Epoch 7/150\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 4s - loss: 0.2794 - acc: 0.8731 - val_loss: 0.1952 - val_acc: 0.9252\n",
      "Epoch 8/150\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 4s - loss: 0.2375 - acc: 0.9039 - val_loss: 0.2033 - val_acc: 0.9283\n",
      "Epoch 9/150\n",
      "Epoch 00009: val_loss improved from 0.19297 to 0.18918, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage2.hdf5\n",
      " - 4s - loss: 0.2470 - acc: 0.8887 - val_loss: 0.1892 - val_acc: 0.9252\n",
      "Epoch 10/150\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 4s - loss: 0.2379 - acc: 0.9047 - val_loss: 0.1968 - val_acc: 0.9252\n",
      "Epoch 11/150\n",
      "Epoch 00011: val_loss improved from 0.18918 to 0.18263, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage2.hdf5\n",
      " - 4s - loss: 0.2621 - acc: 0.8902 - val_loss: 0.1826 - val_acc: 0.9283\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12/150\n",
      "Epoch 00012: val_loss did not improve\n",
      " - 4s - loss: 0.2472 - acc: 0.8856 - val_loss: 0.1900 - val_acc: 0.9283\n",
      "Epoch 13/150\n",
      "Epoch 00013: val_loss did not improve\n",
      " - 4s - loss: 0.2311 - acc: 0.9077 - val_loss: 0.1869 - val_acc: 0.9315\n",
      "Epoch 14/150\n",
      "Epoch 00014: val_loss did not improve\n",
      " - 4s - loss: 0.2518 - acc: 0.8975 - val_loss: 0.1864 - val_acc: 0.9283\n",
      "Epoch 15/150\n",
      "Epoch 00015: val_loss did not improve\n",
      " - 4s - loss: 0.2535 - acc: 0.8799 - val_loss: 0.2018 - val_acc: 0.9128\n",
      "Epoch 16/150\n",
      "Epoch 00016: val_loss did not improve\n",
      " - 4s - loss: 0.2353 - acc: 0.8986 - val_loss: 0.1833 - val_acc: 0.9346\n",
      "Epoch 17/150\n",
      "Epoch 00017: val_loss did not improve\n",
      " - 4s - loss: 0.2404 - acc: 0.8990 - val_loss: 0.1855 - val_acc: 0.9252\n",
      "Epoch 18/150\n",
      "Epoch 00018: val_loss did not improve\n",
      " - 4s - loss: 0.2229 - acc: 0.8982 - val_loss: 0.1883 - val_acc: 0.9315\n",
      "Epoch 19/150\n",
      "Epoch 00019: val_loss did not improve\n",
      " - 4s - loss: 0.2019 - acc: 0.9199 - val_loss: 0.1920 - val_acc: 0.9097\n",
      "Epoch 20/150\n",
      "Epoch 00020: val_loss did not improve\n",
      " - 4s - loss: 0.2260 - acc: 0.9054 - val_loss: 0.1886 - val_acc: 0.9252\n",
      "Epoch 21/150\n",
      "Epoch 00021: val_loss did not improve\n",
      " - 4s - loss: 0.2148 - acc: 0.9138 - val_loss: 0.1873 - val_acc: 0.9315\n",
      "Epoch 22/150\n",
      "Epoch 00022: val_loss did not improve\n",
      " - 4s - loss: 0.2089 - acc: 0.9092 - val_loss: 0.1903 - val_acc: 0.9221\n",
      "Epoch 23/150\n",
      "Epoch 00023: val_loss did not improve\n",
      " - 4s - loss: 0.2156 - acc: 0.9108 - val_loss: 0.1887 - val_acc: 0.9252\n",
      "Epoch 24/150\n",
      "Epoch 00024: val_loss did not improve\n",
      " - 4s - loss: 0.1967 - acc: 0.9161 - val_loss: 0.1908 - val_acc: 0.9159\n",
      "Epoch 25/150\n",
      "Epoch 00025: val_loss improved from 0.18263 to 0.18169, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage2.hdf5\n",
      " - 4s - loss: 0.2242 - acc: 0.9013 - val_loss: 0.1817 - val_acc: 0.9315\n",
      "Epoch 26/150\n",
      "Epoch 00026: val_loss did not improve\n",
      " - 4s - loss: 0.2140 - acc: 0.9062 - val_loss: 0.1898 - val_acc: 0.9346\n",
      "Epoch 27/150\n",
      "Epoch 00027: val_loss did not improve\n",
      " - 4s - loss: 0.2113 - acc: 0.9036 - val_loss: 0.1881 - val_acc: 0.9252\n",
      "Epoch 28/150\n",
      "Epoch 00028: val_loss did not improve\n",
      " - 4s - loss: 0.2250 - acc: 0.8970 - val_loss: 0.1981 - val_acc: 0.9097\n",
      "Epoch 29/150\n",
      "Epoch 00029: val_loss did not improve\n",
      " - 4s - loss: 0.2332 - acc: 0.9005 - val_loss: 0.1969 - val_acc: 0.9128\n",
      "Epoch 30/150\n",
      "Epoch 00030: val_loss did not improve\n",
      " - 4s - loss: 0.2092 - acc: 0.9120 - val_loss: 0.1828 - val_acc: 0.9346\n",
      "Epoch 31/150\n",
      "Epoch 00031: val_loss did not improve\n",
      " - 4s - loss: 0.2125 - acc: 0.9176 - val_loss: 0.1866 - val_acc: 0.9221\n",
      "Epoch 32/150\n",
      "Epoch 00032: val_loss did not improve\n",
      " - 4s - loss: 0.2114 - acc: 0.9115 - val_loss: 0.1857 - val_acc: 0.9346\n",
      "Epoch 33/150\n",
      "Epoch 00033: val_loss did not improve\n",
      " - 4s - loss: 0.2012 - acc: 0.9047 - val_loss: 0.1910 - val_acc: 0.9315\n",
      "Epoch 34/150\n",
      "Epoch 00034: val_loss did not improve\n",
      " - 4s - loss: 0.2170 - acc: 0.9100 - val_loss: 0.1960 - val_acc: 0.9065\n",
      "Epoch 35/150\n",
      "Epoch 00035: val_loss did not improve\n",
      " - 4s - loss: 0.2300 - acc: 0.9112 - val_loss: 0.1921 - val_acc: 0.9283\n",
      "Epoch 36/150\n",
      "Epoch 00036: val_loss did not improve\n",
      " - 4s - loss: 0.2139 - acc: 0.9123 - val_loss: 0.1909 - val_acc: 0.9221\n",
      "Epoch 37/150\n",
      "Epoch 00037: val_loss did not improve\n",
      " - 4s - loss: 0.1949 - acc: 0.9199 - val_loss: 0.1866 - val_acc: 0.9315\n",
      "Epoch 38/150\n",
      "Epoch 00038: val_loss did not improve\n",
      " - 4s - loss: 0.1873 - acc: 0.9245 - val_loss: 0.1983 - val_acc: 0.9097\n",
      "Epoch 39/150\n",
      "Epoch 00039: val_loss did not improve\n",
      " - 4s - loss: 0.1899 - acc: 0.9237 - val_loss: 0.1900 - val_acc: 0.9221\n",
      "Epoch 40/150\n",
      "Epoch 00040: val_loss improved from 0.18169 to 0.17940, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage2.hdf5\n",
      " - 4s - loss: 0.2004 - acc: 0.9066 - val_loss: 0.1794 - val_acc: 0.9283\n",
      "Epoch 41/150\n",
      "Epoch 00041: val_loss improved from 0.17940 to 0.17768, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage2.hdf5\n",
      " - 4s - loss: 0.1934 - acc: 0.9253 - val_loss: 0.1777 - val_acc: 0.9283\n",
      "Epoch 42/150\n",
      "Epoch 00042: val_loss did not improve\n",
      " - 4s - loss: 0.1944 - acc: 0.9120 - val_loss: 0.1875 - val_acc: 0.9097\n",
      "Epoch 43/150\n",
      "Epoch 00043: val_loss improved from 0.17768 to 0.17674, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage2.hdf5\n",
      " - 4s - loss: 0.2382 - acc: 0.9059 - val_loss: 0.1767 - val_acc: 0.9190\n",
      "Epoch 44/150\n",
      "Epoch 00044: val_loss improved from 0.17674 to 0.17644, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage2.hdf5\n",
      " - 4s - loss: 0.2368 - acc: 0.8952 - val_loss: 0.1764 - val_acc: 0.9221\n",
      "Epoch 45/150\n",
      "Epoch 00045: val_loss did not improve\n",
      " - 4s - loss: 0.1863 - acc: 0.9314 - val_loss: 0.1865 - val_acc: 0.9252\n",
      "Epoch 46/150\n",
      "Epoch 00046: val_loss improved from 0.17644 to 0.17429, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage2.hdf5\n",
      " - 4s - loss: 0.1751 - acc: 0.9276 - val_loss: 0.1743 - val_acc: 0.9346\n",
      "Epoch 47/150\n",
      "Epoch 00047: val_loss did not improve\n",
      " - 4s - loss: 0.1803 - acc: 0.9165 - val_loss: 0.1839 - val_acc: 0.9346\n",
      "Epoch 48/150\n",
      "Epoch 00048: val_loss did not improve\n",
      " - 4s - loss: 0.1805 - acc: 0.9321 - val_loss: 0.1823 - val_acc: 0.9283\n",
      "Epoch 49/150\n",
      "Epoch 00049: val_loss did not improve\n",
      " - 4s - loss: 0.1994 - acc: 0.9173 - val_loss: 0.1930 - val_acc: 0.9097\n",
      "Epoch 50/150\n",
      "Epoch 00050: val_loss did not improve\n",
      " - 4s - loss: 0.2161 - acc: 0.8956 - val_loss: 0.1872 - val_acc: 0.9190\n",
      "Epoch 51/150\n",
      "Epoch 00051: val_loss did not improve\n",
      " - 4s - loss: 0.1864 - acc: 0.9253 - val_loss: 0.1802 - val_acc: 0.9283\n",
      "Epoch 52/150\n",
      "Epoch 00052: val_loss did not improve\n",
      " - 4s - loss: 0.1834 - acc: 0.9245 - val_loss: 0.1827 - val_acc: 0.9315\n",
      "Epoch 53/150\n",
      "Epoch 00053: val_loss improved from 0.17429 to 0.17086, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage2.hdf5\n",
      " - 4s - loss: 0.1829 - acc: 0.9329 - val_loss: 0.1709 - val_acc: 0.9439\n",
      "Epoch 54/150\n",
      "Epoch 00054: val_loss did not improve\n",
      " - 4s - loss: 0.1679 - acc: 0.9291 - val_loss: 0.1814 - val_acc: 0.9315\n",
      "Epoch 55/150\n",
      "Epoch 00055: val_loss did not improve\n",
      " - 4s - loss: 0.1752 - acc: 0.9234 - val_loss: 0.1803 - val_acc: 0.9252\n",
      "Epoch 56/150\n",
      "Epoch 00056: val_loss did not improve\n",
      " - 4s - loss: 0.1773 - acc: 0.9260 - val_loss: 0.1800 - val_acc: 0.9283\n",
      "Epoch 57/150\n",
      "Epoch 00057: val_loss did not improve\n",
      " - 4s - loss: 0.1765 - acc: 0.9222 - val_loss: 0.1758 - val_acc: 0.9346\n",
      "Epoch 58/150\n",
      "Epoch 00058: val_loss did not improve\n",
      " - 4s - loss: 0.1835 - acc: 0.9268 - val_loss: 0.1783 - val_acc: 0.9408\n",
      "Epoch 59/150\n",
      "Epoch 00059: val_loss did not improve\n",
      " - 4s - loss: 0.1738 - acc: 0.9272 - val_loss: 0.1847 - val_acc: 0.9159\n",
      "Epoch 60/150\n",
      "Epoch 00060: val_loss did not improve\n",
      " - 4s - loss: 0.1923 - acc: 0.9150 - val_loss: 0.1881 - val_acc: 0.9159\n",
      "Epoch 61/150\n",
      "Epoch 00061: val_loss did not improve\n",
      " - 4s - loss: 0.1658 - acc: 0.9237 - val_loss: 0.1766 - val_acc: 0.9283\n",
      "Epoch 62/150\n",
      "Epoch 00062: val_loss did not improve\n",
      " - 4s - loss: 0.1675 - acc: 0.9314 - val_loss: 0.1775 - val_acc: 0.9346\n",
      "Epoch 63/150\n",
      "Epoch 00063: val_loss did not improve\n",
      " - 4s - loss: 0.1663 - acc: 0.9337 - val_loss: 0.1778 - val_acc: 0.9283\n",
      "Epoch 64/150\n",
      "Epoch 00064: val_loss did not improve\n",
      " - 4s - loss: 0.1770 - acc: 0.9265 - val_loss: 0.2002 - val_acc: 0.9065\n",
      "Epoch 65/150\n",
      "Epoch 00065: val_loss improved from 0.17086 to 0.16777, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv2_stage2.hdf5\n",
      " - 4s - loss: 0.1850 - acc: 0.9192 - val_loss: 0.1678 - val_acc: 0.9377\n",
      "Epoch 66/150\n",
      "Epoch 00066: val_loss did not improve\n",
      " - 4s - loss: 0.1700 - acc: 0.9344 - val_loss: 0.1983 - val_acc: 0.9034\n",
      "Epoch 67/150\n",
      "Epoch 00067: val_loss did not improve\n",
      " - 4s - loss: 0.1870 - acc: 0.9199 - val_loss: 0.1764 - val_acc: 0.9346\n",
      "Epoch 68/150\n",
      "Epoch 00068: val_loss did not improve\n",
      " - 4s - loss: 0.1682 - acc: 0.9375 - val_loss: 0.1729 - val_acc: 0.9408\n",
      "Epoch 69/150\n",
      "Epoch 00069: val_loss did not improve\n",
      " - 4s - loss: 0.1561 - acc: 0.9382 - val_loss: 0.1762 - val_acc: 0.9315\n",
      "Epoch 70/150\n",
      "Epoch 00070: val_loss did not improve\n",
      " - 4s - loss: 0.1465 - acc: 0.9443 - val_loss: 0.1768 - val_acc: 0.9346\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 71/150\n",
      "Epoch 00071: val_loss did not improve\n",
      " - 4s - loss: 0.1623 - acc: 0.9298 - val_loss: 0.1752 - val_acc: 0.9315\n",
      "Epoch 72/150\n",
      "Epoch 00072: val_loss did not improve\n",
      " - 4s - loss: 0.1553 - acc: 0.9337 - val_loss: 0.1958 - val_acc: 0.9159\n",
      "Epoch 73/150\n",
      "Epoch 00073: val_loss did not improve\n",
      " - 4s - loss: 0.1627 - acc: 0.9295 - val_loss: 0.1683 - val_acc: 0.9283\n",
      "Epoch 74/150\n",
      "Epoch 00074: val_loss did not improve\n",
      " - 4s - loss: 0.1609 - acc: 0.9337 - val_loss: 0.1755 - val_acc: 0.9283\n",
      "Epoch 75/150\n",
      "Epoch 00075: val_loss did not improve\n",
      " - 4s - loss: 0.1512 - acc: 0.9359 - val_loss: 0.1898 - val_acc: 0.9283\n",
      "Epoch 76/150\n",
      "Epoch 00076: val_loss did not improve\n",
      " - 4s - loss: 0.1581 - acc: 0.9367 - val_loss: 0.1754 - val_acc: 0.9346\n",
      "Epoch 77/150\n",
      "Epoch 00077: val_loss did not improve\n",
      " - 4s - loss: 0.1810 - acc: 0.9326 - val_loss: 0.1968 - val_acc: 0.9221\n",
      "Epoch 78/150\n",
      "Epoch 00078: val_loss did not improve\n",
      " - 4s - loss: 0.1737 - acc: 0.9249 - val_loss: 0.1853 - val_acc: 0.9283\n",
      "Epoch 79/150\n",
      "Epoch 00079: val_loss did not improve\n",
      " - 4s - loss: 0.1517 - acc: 0.9359 - val_loss: 0.1884 - val_acc: 0.9315\n",
      "Epoch 80/150\n",
      "Epoch 00080: val_loss did not improve\n",
      " - 4s - loss: 0.1383 - acc: 0.9420 - val_loss: 0.1877 - val_acc: 0.9283\n",
      "Epoch 81/150\n",
      "Epoch 00081: val_loss did not improve\n",
      " - 4s - loss: 0.1529 - acc: 0.9306 - val_loss: 0.1839 - val_acc: 0.9283\n",
      "Epoch 82/150\n",
      "Epoch 00082: val_loss did not improve\n",
      " - 4s - loss: 0.1629 - acc: 0.9249 - val_loss: 0.1845 - val_acc: 0.9283\n",
      "Epoch 83/150\n",
      "Epoch 00083: val_loss did not improve\n",
      " - 5s - loss: 0.1466 - acc: 0.9443 - val_loss: 0.1965 - val_acc: 0.9159\n",
      "Epoch 84/150\n",
      "Epoch 00084: val_loss did not improve\n",
      " - 4s - loss: 0.1527 - acc: 0.9382 - val_loss: 0.1885 - val_acc: 0.9283\n",
      "Epoch 85/150\n",
      "Epoch 00085: val_loss did not improve\n",
      " - 4s - loss: 0.1483 - acc: 0.9436 - val_loss: 0.1956 - val_acc: 0.9252\n",
      "Epoch 86/150\n",
      "Epoch 00086: val_loss did not improve\n",
      " - 4s - loss: 0.1503 - acc: 0.9398 - val_loss: 0.1965 - val_acc: 0.9128\n",
      "Epoch 87/150\n",
      "Epoch 00087: val_loss did not improve\n",
      " - 4s - loss: 0.1533 - acc: 0.9382 - val_loss: 0.2101 - val_acc: 0.9065\n",
      "Epoch 88/150\n",
      "Epoch 00088: val_loss did not improve\n",
      " - 4s - loss: 0.1572 - acc: 0.9356 - val_loss: 0.1930 - val_acc: 0.9190\n",
      "Epoch 89/150\n",
      "Epoch 00089: val_loss did not improve\n",
      " - 4s - loss: 0.1578 - acc: 0.9337 - val_loss: 0.2043 - val_acc: 0.9097\n",
      "Epoch 90/150\n",
      "Epoch 00090: val_loss did not improve\n",
      " - 4s - loss: 0.1372 - acc: 0.9428 - val_loss: 0.1923 - val_acc: 0.9283\n",
      "Epoch 91/150\n",
      "Epoch 00091: val_loss did not improve\n",
      " - 4s - loss: 0.1610 - acc: 0.9306 - val_loss: 0.1805 - val_acc: 0.9252\n",
      "Epoch 92/150\n",
      "Epoch 00092: val_loss did not improve\n",
      " - 4s - loss: 0.1360 - acc: 0.9497 - val_loss: 0.1957 - val_acc: 0.9190\n",
      "Epoch 93/150\n",
      "Epoch 00093: val_loss did not improve\n",
      " - 4s - loss: 0.1447 - acc: 0.9428 - val_loss: 0.1910 - val_acc: 0.9190\n",
      "Epoch 94/150\n",
      "Epoch 00094: val_loss did not improve\n",
      " - 4s - loss: 0.1418 - acc: 0.9451 - val_loss: 0.2088 - val_acc: 0.9128\n",
      "Epoch 95/150\n",
      "Epoch 00095: val_loss did not improve\n",
      " - 4s - loss: 0.1428 - acc: 0.9428 - val_loss: 0.1907 - val_acc: 0.9190\n",
      "CV2, Done!\n",
      "1283/1283 [==============================] - 2s 1ms/step\n",
      "Train loss: 0.116974927351\n",
      "Train accuracy: 0.957911145752\n",
      "321/321 [==============================] - 0s 1ms/step\n",
      "Valid loss: 0.167773175694\n",
      "Valid accuracy: 0.93769470405\n",
      "\n",
      "===================FOLD= 3\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_20 (InputLayer)           (None, 75, 75, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_77 (Conv2D)              (None, 71, 71, 64)   4864        input_20[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_77 (Activation)      (None, 71, 71, 64)   0           conv2d_77[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_77 (MaxPooling2D) (None, 35, 35, 64)   0           activation_77[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_115 (Dropout)           (None, 35, 35, 64)   0           max_pooling2d_77[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_78 (Conv2D)              (None, 33, 33, 128)  73856       dropout_115[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_78 (Activation)      (None, 33, 33, 128)  0           conv2d_78[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_78 (MaxPooling2D) (None, 16, 16, 128)  0           activation_78[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_116 (Dropout)           (None, 16, 16, 128)  0           max_pooling2d_78[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_79 (Conv2D)              (None, 14, 14, 128)  147584      dropout_116[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_79 (Activation)      (None, 14, 14, 128)  0           conv2d_79[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_79 (MaxPooling2D) (None, 7, 7, 128)    0           activation_79[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_117 (Dropout)           (None, 7, 7, 128)    0           max_pooling2d_79[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_80 (Conv2D)              (None, 5, 5, 64)     73792       dropout_117[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_80 (Activation)      (None, 5, 5, 64)     0           conv2d_80[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_80 (MaxPooling2D) (None, 2, 2, 64)     0           activation_80[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "other (InputLayer)              (None, 13)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_118 (Dropout)           (None, 2, 2, 64)     0           max_pooling2d_80[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_96 (Dense)                (None, 16)           224         other[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_20 (Flatten)            (None, 256)          0           dropout_118[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_97 (Dense)                (None, 16)           272         dense_96[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_20 (Concatenate)    (None, 272)          0           flatten_20[0][0]                 \n",
      "                                                                 dense_97[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_98 (Dense)                (None, 256)          69888       concatenate_20[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_119 (Dropout)           (None, 256)          0           dense_98[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_99 (Dense)                (None, 256)          65792       dropout_119[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_120 (Dropout)           (None, 256)          0           dense_99[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_100 (Dense)               (None, 1)            257         dropout_120[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 436,529\n",
      "Trainable params: 436,529\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/150\n",
      "Epoch 00001: val_loss improved from inf to 0.69140, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 11s - loss: 1.4648 - acc: 0.5105 - val_loss: 0.6914 - val_acc: 0.5078\n",
      "Epoch 2/150\n",
      "Epoch 00002: val_loss improved from 0.69140 to 0.66718, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.7647 - acc: 0.5273 - val_loss: 0.6672 - val_acc: 0.5950\n",
      "Epoch 3/150\n",
      "Epoch 00003: val_loss improved from 0.66718 to 0.63158, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.6808 - acc: 0.5688 - val_loss: 0.6316 - val_acc: 0.6791\n",
      "Epoch 4/150\n",
      "Epoch 00004: val_loss improved from 0.63158 to 0.58179, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.6330 - acc: 0.5924 - val_loss: 0.5818 - val_acc: 0.6854\n",
      "Epoch 5/150\n",
      "Epoch 00005: val_loss improved from 0.58179 to 0.57237, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.5993 - acc: 0.6393 - val_loss: 0.5724 - val_acc: 0.6885\n",
      "Epoch 6/150\n",
      "Epoch 00006: val_loss improved from 0.57237 to 0.56537, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.5993 - acc: 0.6199 - val_loss: 0.5654 - val_acc: 0.7009\n",
      "Epoch 7/150\n",
      "Epoch 00007: val_loss improved from 0.56537 to 0.55508, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.5820 - acc: 0.6177 - val_loss: 0.5551 - val_acc: 0.7072\n",
      "Epoch 8/150\n",
      "Epoch 00008: val_loss improved from 0.55508 to 0.55105, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.5719 - acc: 0.6405 - val_loss: 0.5510 - val_acc: 0.7040\n",
      "Epoch 9/150\n",
      "Epoch 00009: val_loss improved from 0.55105 to 0.52654, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 6s - loss: 0.5579 - acc: 0.6756 - val_loss: 0.5265 - val_acc: 0.7196\n",
      "Epoch 10/150\n",
      "Epoch 00010: val_loss improved from 0.52654 to 0.51553, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 6s - loss: 0.5433 - acc: 0.6855 - val_loss: 0.5155 - val_acc: 0.7477\n",
      "Epoch 11/150\n",
      "Epoch 00011: val_loss improved from 0.51553 to 0.49317, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 6s - loss: 0.5256 - acc: 0.6904 - val_loss: 0.4932 - val_acc: 0.7664\n",
      "Epoch 12/150\n",
      "Epoch 00012: val_loss improved from 0.49317 to 0.48019, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 6s - loss: 0.5049 - acc: 0.7339 - val_loss: 0.4802 - val_acc: 0.7819\n",
      "Epoch 13/150\n",
      "Epoch 00013: val_loss improved from 0.48019 to 0.45036, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.4905 - acc: 0.7549 - val_loss: 0.4504 - val_acc: 0.8255\n",
      "Epoch 14/150\n",
      "Epoch 00014: val_loss improved from 0.45036 to 0.41950, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.4677 - acc: 0.7675 - val_loss: 0.4195 - val_acc: 0.8224\n",
      "Epoch 15/150\n",
      "Epoch 00015: val_loss improved from 0.41950 to 0.40909, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.4328 - acc: 0.7846 - val_loss: 0.4091 - val_acc: 0.8318\n",
      "Epoch 16/150\n",
      "Epoch 00016: val_loss improved from 0.40909 to 0.37534, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.4080 - acc: 0.8048 - val_loss: 0.3753 - val_acc: 0.8411\n",
      "Epoch 17/150\n",
      "Epoch 00017: val_loss improved from 0.37534 to 0.36736, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.4058 - acc: 0.7991 - val_loss: 0.3674 - val_acc: 0.8567\n",
      "Epoch 18/150\n",
      "Epoch 00018: val_loss did not improve\n",
      " - 5s - loss: 0.3721 - acc: 0.8238 - val_loss: 0.3883 - val_acc: 0.8162\n",
      "Epoch 19/150\n",
      "Epoch 00019: val_loss improved from 0.36736 to 0.34417, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 6s - loss: 0.3751 - acc: 0.8231 - val_loss: 0.3442 - val_acc: 0.8754\n",
      "Epoch 20/150\n",
      "Epoch 00020: val_loss did not improve\n",
      " - 5s - loss: 0.3703 - acc: 0.8193 - val_loss: 0.3462 - val_acc: 0.8567\n",
      "Epoch 21/150\n",
      "Epoch 00021: val_loss improved from 0.34417 to 0.33915, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 6s - loss: 0.3643 - acc: 0.8220 - val_loss: 0.3392 - val_acc: 0.8660\n",
      "Epoch 22/150\n",
      "Epoch 00022: val_loss improved from 0.33915 to 0.32506, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 6s - loss: 0.3541 - acc: 0.8372 - val_loss: 0.3251 - val_acc: 0.8692\n",
      "Epoch 23/150\n",
      "Epoch 00023: val_loss improved from 0.32506 to 0.31921, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 6s - loss: 0.3339 - acc: 0.8449 - val_loss: 0.3192 - val_acc: 0.8723\n",
      "Epoch 24/150\n",
      "Epoch 00024: val_loss did not improve\n",
      " - 5s - loss: 0.3251 - acc: 0.8521 - val_loss: 0.3294 - val_acc: 0.8629\n",
      "Epoch 25/150\n",
      "Epoch 00025: val_loss improved from 0.31921 to 0.31176, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.3287 - acc: 0.8506 - val_loss: 0.3118 - val_acc: 0.8505\n",
      "Epoch 26/150\n",
      "Epoch 00026: val_loss improved from 0.31176 to 0.30986, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.3211 - acc: 0.8513 - val_loss: 0.3099 - val_acc: 0.8785\n",
      "Epoch 27/150\n",
      "Epoch 00027: val_loss improved from 0.30986 to 0.30503, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.3079 - acc: 0.8643 - val_loss: 0.3050 - val_acc: 0.8723\n",
      "Epoch 28/150\n",
      "Epoch 00028: val_loss did not improve\n",
      " - 5s - loss: 0.3332 - acc: 0.8548 - val_loss: 0.3344 - val_acc: 0.8255\n",
      "Epoch 29/150\n",
      "Epoch 00029: val_loss improved from 0.30503 to 0.29432, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.3187 - acc: 0.8479 - val_loss: 0.2943 - val_acc: 0.8474\n",
      "Epoch 30/150\n",
      "Epoch 00030: val_loss did not improve\n",
      " - 5s - loss: 0.2990 - acc: 0.8597 - val_loss: 0.3155 - val_acc: 0.8411\n",
      "Epoch 31/150\n",
      "Epoch 00031: val_loss improved from 0.29432 to 0.27485, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 6s - loss: 0.2911 - acc: 0.8772 - val_loss: 0.2749 - val_acc: 0.8816\n",
      "Epoch 32/150\n",
      "Epoch 00032: val_loss did not improve\n",
      " - 6s - loss: 0.2690 - acc: 0.8795 - val_loss: 0.2764 - val_acc: 0.8847\n",
      "Epoch 33/150\n",
      "Epoch 00033: val_loss improved from 0.27485 to 0.26439, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.2628 - acc: 0.8826 - val_loss: 0.2644 - val_acc: 0.9034\n",
      "Epoch 34/150\n",
      "Epoch 00034: val_loss did not improve\n",
      " - 5s - loss: 0.2650 - acc: 0.8868 - val_loss: 0.2800 - val_acc: 0.8941\n",
      "Epoch 35/150\n",
      "Epoch 00035: val_loss improved from 0.26439 to 0.25682, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 6s - loss: 0.2823 - acc: 0.8749 - val_loss: 0.2568 - val_acc: 0.8879\n",
      "Epoch 36/150\n",
      "Epoch 00036: val_loss did not improve\n",
      " - 5s - loss: 0.2683 - acc: 0.8738 - val_loss: 0.2638 - val_acc: 0.8972\n",
      "Epoch 37/150\n",
      "Epoch 00037: val_loss did not improve\n",
      " - 5s - loss: 0.2618 - acc: 0.8780 - val_loss: 0.2589 - val_acc: 0.8910\n",
      "Epoch 38/150\n",
      "Epoch 00038: val_loss improved from 0.25682 to 0.25213, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.2506 - acc: 0.8868 - val_loss: 0.2521 - val_acc: 0.9003\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 39/150\n",
      "Epoch 00039: val_loss did not improve\n",
      " - 5s - loss: 0.2397 - acc: 0.8894 - val_loss: 0.2594 - val_acc: 0.9003\n",
      "Epoch 40/150\n",
      "Epoch 00040: val_loss did not improve\n",
      " - 5s - loss: 0.2462 - acc: 0.8917 - val_loss: 0.2669 - val_acc: 0.9003\n",
      "Epoch 41/150\n",
      "Epoch 00041: val_loss did not improve\n",
      " - 5s - loss: 0.2469 - acc: 0.8891 - val_loss: 0.2571 - val_acc: 0.8910\n",
      "Epoch 42/150\n",
      "Epoch 00042: val_loss improved from 0.25213 to 0.24895, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.2452 - acc: 0.8955 - val_loss: 0.2489 - val_acc: 0.9003\n",
      "Epoch 43/150\n",
      "Epoch 00043: val_loss did not improve\n",
      " - 5s - loss: 0.2317 - acc: 0.8948 - val_loss: 0.2569 - val_acc: 0.8910\n",
      "Epoch 44/150\n",
      "Epoch 00044: val_loss did not improve\n",
      " - 5s - loss: 0.2147 - acc: 0.9016 - val_loss: 0.2526 - val_acc: 0.9003\n",
      "Epoch 45/150\n",
      "Epoch 00045: val_loss improved from 0.24895 to 0.24033, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 6s - loss: 0.2308 - acc: 0.9001 - val_loss: 0.2403 - val_acc: 0.9003\n",
      "Epoch 46/150\n",
      "Epoch 00046: val_loss did not improve\n",
      " - 5s - loss: 0.2329 - acc: 0.8955 - val_loss: 0.2606 - val_acc: 0.8941\n",
      "Epoch 47/150\n",
      "Epoch 00047: val_loss did not improve\n",
      " - 5s - loss: 0.2348 - acc: 0.8952 - val_loss: 0.2522 - val_acc: 0.9003\n",
      "Epoch 48/150\n",
      "Epoch 00048: val_loss did not improve\n",
      " - 5s - loss: 0.2313 - acc: 0.8932 - val_loss: 0.2520 - val_acc: 0.8972\n",
      "Epoch 49/150\n",
      "Epoch 00049: val_loss did not improve\n",
      " - 5s - loss: 0.2701 - acc: 0.8723 - val_loss: 0.2447 - val_acc: 0.8972\n",
      "Epoch 50/150\n",
      "Epoch 00050: val_loss did not improve\n",
      " - 5s - loss: 0.2080 - acc: 0.9085 - val_loss: 0.2662 - val_acc: 0.8847\n",
      "Epoch 51/150\n",
      "Epoch 00051: val_loss did not improve\n",
      " - 5s - loss: 0.2004 - acc: 0.9085 - val_loss: 0.2545 - val_acc: 0.9034\n",
      "Epoch 52/150\n",
      "Epoch 00052: val_loss did not improve\n",
      " - 5s - loss: 0.2122 - acc: 0.9001 - val_loss: 0.2420 - val_acc: 0.9003\n",
      "Epoch 53/150\n",
      "Epoch 00053: val_loss did not improve\n",
      " - 5s - loss: 0.1988 - acc: 0.9169 - val_loss: 0.2728 - val_acc: 0.8816\n",
      "Epoch 54/150\n",
      "Epoch 00054: val_loss did not improve\n",
      " - 5s - loss: 0.2083 - acc: 0.9054 - val_loss: 0.2692 - val_acc: 0.8847\n",
      "Epoch 55/150\n",
      "Epoch 00055: val_loss did not improve\n",
      " - 5s - loss: 0.1909 - acc: 0.9169 - val_loss: 0.2442 - val_acc: 0.9003\n",
      "Epoch 56/150\n",
      "Epoch 00056: val_loss did not improve\n",
      " - 5s - loss: 0.2027 - acc: 0.9092 - val_loss: 0.2462 - val_acc: 0.8972\n",
      "Epoch 57/150\n",
      "Epoch 00057: val_loss did not improve\n",
      " - 5s - loss: 0.1954 - acc: 0.9146 - val_loss: 0.2427 - val_acc: 0.8972\n",
      "Epoch 58/150\n",
      "Epoch 00058: val_loss did not improve\n",
      " - 5s - loss: 0.1907 - acc: 0.9184 - val_loss: 0.2454 - val_acc: 0.8941\n",
      "Epoch 59/150\n",
      "Epoch 00059: val_loss did not improve\n",
      " - 5s - loss: 0.1981 - acc: 0.9051 - val_loss: 0.2564 - val_acc: 0.9065\n",
      "Epoch 60/150\n",
      "Epoch 00060: val_loss did not improve\n",
      " - 5s - loss: 0.1918 - acc: 0.9100 - val_loss: 0.2643 - val_acc: 0.8847\n",
      "Epoch 61/150\n",
      "Epoch 00061: val_loss did not improve\n",
      " - 5s - loss: 0.1784 - acc: 0.9199 - val_loss: 0.2492 - val_acc: 0.8910\n",
      "Epoch 62/150\n",
      "Epoch 00062: val_loss did not improve\n",
      " - 5s - loss: 0.1934 - acc: 0.9074 - val_loss: 0.2548 - val_acc: 0.8941\n",
      "Epoch 63/150\n",
      "Epoch 00063: val_loss improved from 0.24033 to 0.23835, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.1813 - acc: 0.9253 - val_loss: 0.2383 - val_acc: 0.9003\n",
      "Epoch 64/150\n",
      "Epoch 00064: val_loss did not improve\n",
      " - 5s - loss: 0.1747 - acc: 0.9165 - val_loss: 0.2795 - val_acc: 0.8816\n",
      "Epoch 65/150\n",
      "Epoch 00065: val_loss improved from 0.23835 to 0.23710, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 6s - loss: 0.2075 - acc: 0.9123 - val_loss: 0.2371 - val_acc: 0.9097\n",
      "Epoch 66/150\n",
      "Epoch 00066: val_loss did not improve\n",
      " - 5s - loss: 0.1763 - acc: 0.9184 - val_loss: 0.2399 - val_acc: 0.8972\n",
      "Epoch 67/150\n",
      "Epoch 00067: val_loss improved from 0.23710 to 0.23167, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 6s - loss: 0.1759 - acc: 0.9314 - val_loss: 0.2317 - val_acc: 0.8941\n",
      "Epoch 68/150\n",
      "Epoch 00068: val_loss did not improve\n",
      " - 5s - loss: 0.1669 - acc: 0.9260 - val_loss: 0.2725 - val_acc: 0.8785\n",
      "Epoch 69/150\n",
      "Epoch 00069: val_loss did not improve\n",
      " - 5s - loss: 0.1820 - acc: 0.9283 - val_loss: 0.2397 - val_acc: 0.8972\n",
      "Epoch 70/150\n",
      "Epoch 00070: val_loss did not improve\n",
      " - 5s - loss: 0.1710 - acc: 0.9207 - val_loss: 0.2585 - val_acc: 0.8879\n",
      "Epoch 71/150\n",
      "Epoch 00071: val_loss did not improve\n",
      " - 5s - loss: 0.1776 - acc: 0.9173 - val_loss: 0.2479 - val_acc: 0.8972\n",
      "Epoch 72/150\n",
      "Epoch 00072: val_loss did not improve\n",
      " - 5s - loss: 0.1753 - acc: 0.9242 - val_loss: 0.2471 - val_acc: 0.9128\n",
      "Epoch 73/150\n",
      "Epoch 00073: val_loss did not improve\n",
      " - 5s - loss: 0.1854 - acc: 0.9173 - val_loss: 0.2648 - val_acc: 0.9003\n",
      "Epoch 74/150\n",
      "Epoch 00074: val_loss did not improve\n",
      " - 5s - loss: 0.1804 - acc: 0.9237 - val_loss: 0.2540 - val_acc: 0.9065\n",
      "Epoch 75/150\n",
      "Epoch 00075: val_loss did not improve\n",
      " - 5s - loss: 0.1621 - acc: 0.9333 - val_loss: 0.2800 - val_acc: 0.8910\n",
      "Epoch 76/150\n",
      "Epoch 00076: val_loss did not improve\n",
      " - 6s - loss: 0.2089 - acc: 0.9039 - val_loss: 0.2623 - val_acc: 0.8847\n",
      "Epoch 77/150\n",
      "Epoch 00077: val_loss did not improve\n",
      " - 5s - loss: 0.1679 - acc: 0.9219 - val_loss: 0.2676 - val_acc: 0.8941\n",
      "Epoch 78/150\n",
      "Epoch 00078: val_loss did not improve\n",
      " - 5s - loss: 0.1928 - acc: 0.9086 - val_loss: 0.2601 - val_acc: 0.8972\n",
      "Epoch 79/150\n",
      "Epoch 00079: val_loss did not improve\n",
      " - 5s - loss: 0.1796 - acc: 0.9115 - val_loss: 0.2506 - val_acc: 0.9003\n",
      "Epoch 80/150\n",
      "Epoch 00080: val_loss did not improve\n",
      " - 5s - loss: 0.1583 - acc: 0.9306 - val_loss: 0.2719 - val_acc: 0.8847\n",
      "Epoch 81/150\n",
      "Epoch 00081: val_loss did not improve\n",
      " - 5s - loss: 0.1453 - acc: 0.9337 - val_loss: 0.2633 - val_acc: 0.8941\n",
      "Epoch 82/150\n",
      "Epoch 00082: val_loss did not improve\n",
      " - 5s - loss: 0.1484 - acc: 0.9337 - val_loss: 0.2503 - val_acc: 0.9128\n",
      "Epoch 83/150\n",
      "Epoch 00083: val_loss did not improve\n",
      " - 5s - loss: 0.1434 - acc: 0.9390 - val_loss: 0.2529 - val_acc: 0.9003\n",
      "Epoch 84/150\n",
      "Epoch 00084: val_loss did not improve\n",
      " - 5s - loss: 0.1554 - acc: 0.9280 - val_loss: 0.2460 - val_acc: 0.9003\n",
      "Epoch 85/150\n",
      "Epoch 00085: val_loss did not improve\n",
      " - 5s - loss: 0.1847 - acc: 0.9184 - val_loss: 0.2384 - val_acc: 0.8972\n",
      "Epoch 86/150\n",
      "Epoch 00086: val_loss improved from 0.23167 to 0.22940, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 5s - loss: 0.1380 - acc: 0.9459 - val_loss: 0.2294 - val_acc: 0.9128\n",
      "Epoch 87/150\n",
      "Epoch 00087: val_loss did not improve\n",
      " - 5s - loss: 0.1342 - acc: 0.9436 - val_loss: 0.2314 - val_acc: 0.9128\n",
      "Epoch 88/150\n",
      "Epoch 00088: val_loss did not improve\n",
      " - 5s - loss: 0.1562 - acc: 0.9333 - val_loss: 0.2420 - val_acc: 0.9097\n",
      "Epoch 89/150\n",
      "Epoch 00089: val_loss did not improve\n",
      " - 6s - loss: 0.1651 - acc: 0.9321 - val_loss: 0.2366 - val_acc: 0.9065\n",
      "Epoch 90/150\n",
      "Epoch 00090: val_loss did not improve\n",
      " - 6s - loss: 0.1351 - acc: 0.9443 - val_loss: 0.2445 - val_acc: 0.9065\n",
      "Epoch 91/150\n",
      "Epoch 00091: val_loss did not improve\n",
      " - 6s - loss: 0.1251 - acc: 0.9481 - val_loss: 0.2387 - val_acc: 0.9159\n",
      "Epoch 92/150\n",
      "Epoch 00092: val_loss did not improve\n",
      " - 6s - loss: 0.1378 - acc: 0.9405 - val_loss: 0.2353 - val_acc: 0.9003\n",
      "Epoch 93/150\n",
      "Epoch 00093: val_loss did not improve\n",
      " - 5s - loss: 0.1242 - acc: 0.9474 - val_loss: 0.2613 - val_acc: 0.8910\n",
      "Epoch 94/150\n",
      "Epoch 00094: val_loss did not improve\n",
      " - 5s - loss: 0.1386 - acc: 0.9333 - val_loss: 0.2589 - val_acc: 0.9097\n",
      "Epoch 95/150\n",
      "Epoch 00095: val_loss did not improve\n",
      " - 5s - loss: 0.1384 - acc: 0.9413 - val_loss: 0.2798 - val_acc: 0.8910\n",
      "Epoch 96/150\n",
      "Epoch 00096: val_loss did not improve\n",
      " - 5s - loss: 0.1195 - acc: 0.9558 - val_loss: 0.2657 - val_acc: 0.8910\n",
      "Epoch 97/150\n",
      "Epoch 00097: val_loss did not improve\n",
      " - 5s - loss: 0.1471 - acc: 0.9341 - val_loss: 0.2510 - val_acc: 0.8972\n",
      "Epoch 98/150\n",
      "Epoch 00098: val_loss did not improve\n",
      " - 5s - loss: 0.1365 - acc: 0.9382 - val_loss: 0.2674 - val_acc: 0.9034\n",
      "Epoch 99/150\n",
      "Epoch 00099: val_loss did not improve\n",
      " - 5s - loss: 0.1185 - acc: 0.9550 - val_loss: 0.2546 - val_acc: 0.9003\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 100/150\n",
      "Epoch 00100: val_loss did not improve\n",
      " - 5s - loss: 0.1237 - acc: 0.9471 - val_loss: 0.2483 - val_acc: 0.9065\n",
      "Epoch 101/150\n",
      "Epoch 00101: val_loss did not improve\n",
      " - 6s - loss: 0.1234 - acc: 0.9420 - val_loss: 0.2842 - val_acc: 0.8972\n",
      "Epoch 102/150\n",
      "Epoch 00102: val_loss did not improve\n",
      " - 6s - loss: 0.1319 - acc: 0.9527 - val_loss: 0.2527 - val_acc: 0.9065\n",
      "Epoch 103/150\n",
      "Epoch 00103: val_loss did not improve\n",
      " - 5s - loss: 0.1274 - acc: 0.9481 - val_loss: 0.2657 - val_acc: 0.9003\n",
      "Epoch 104/150\n",
      "Epoch 00104: val_loss did not improve\n",
      " - 5s - loss: 0.1334 - acc: 0.9497 - val_loss: 0.2499 - val_acc: 0.9128\n",
      "Epoch 105/150\n",
      "Epoch 00105: val_loss did not improve\n",
      " - 5s - loss: 0.1041 - acc: 0.9611 - val_loss: 0.2717 - val_acc: 0.9128\n",
      "Epoch 106/150\n",
      "Epoch 00106: val_loss did not improve\n",
      " - 5s - loss: 0.1126 - acc: 0.9596 - val_loss: 0.2536 - val_acc: 0.9097\n",
      "Epoch 107/150\n",
      "Epoch 00107: val_loss did not improve\n",
      " - 5s - loss: 0.1109 - acc: 0.9550 - val_loss: 0.2374 - val_acc: 0.9097\n",
      "Epoch 108/150\n",
      "Epoch 00108: val_loss did not improve\n",
      " - 5s - loss: 0.1067 - acc: 0.9619 - val_loss: 0.3003 - val_acc: 0.8972\n",
      "Epoch 109/150\n",
      "Epoch 00109: val_loss improved from 0.22940 to 0.22690, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage1.hdf5\n",
      " - 6s - loss: 0.1378 - acc: 0.9443 - val_loss: 0.2269 - val_acc: 0.9252\n",
      "Epoch 110/150\n",
      "Epoch 00110: val_loss did not improve\n",
      " - 5s - loss: 0.1138 - acc: 0.9535 - val_loss: 0.2477 - val_acc: 0.9128\n",
      "Epoch 111/150\n",
      "Epoch 00111: val_loss did not improve\n",
      " - 5s - loss: 0.1121 - acc: 0.9527 - val_loss: 0.2634 - val_acc: 0.9159\n",
      "Epoch 112/150\n",
      "Epoch 00112: val_loss did not improve\n",
      " - 5s - loss: 0.1134 - acc: 0.9542 - val_loss: 0.2333 - val_acc: 0.9097\n",
      "Epoch 113/150\n",
      "Epoch 00113: val_loss did not improve\n",
      " - 5s - loss: 0.1077 - acc: 0.9581 - val_loss: 0.2392 - val_acc: 0.9065\n",
      "Epoch 114/150\n",
      "Epoch 00114: val_loss did not improve\n",
      " - 5s - loss: 0.1012 - acc: 0.9565 - val_loss: 0.2446 - val_acc: 0.9159\n",
      "Epoch 115/150\n",
      "Epoch 00115: val_loss did not improve\n",
      " - 5s - loss: 0.1118 - acc: 0.9558 - val_loss: 0.2674 - val_acc: 0.8910\n",
      "Epoch 116/150\n",
      "Epoch 00116: val_loss did not improve\n",
      " - 5s - loss: 0.1055 - acc: 0.9550 - val_loss: 0.2562 - val_acc: 0.9065\n",
      "Epoch 117/150\n",
      "Epoch 00117: val_loss did not improve\n",
      " - 5s - loss: 0.1030 - acc: 0.9524 - val_loss: 0.2821 - val_acc: 0.9065\n",
      "Epoch 118/150\n",
      "Epoch 00118: val_loss did not improve\n",
      " - 5s - loss: 0.2678 - acc: 0.8906 - val_loss: 0.2295 - val_acc: 0.8910\n",
      "Epoch 119/150\n",
      "Epoch 00119: val_loss did not improve\n",
      " - 5s - loss: 0.1498 - acc: 0.9314 - val_loss: 0.2393 - val_acc: 0.9097\n",
      "Epoch 120/150\n",
      "Epoch 00120: val_loss did not improve\n",
      " - 5s - loss: 0.1217 - acc: 0.9459 - val_loss: 0.2348 - val_acc: 0.9159\n",
      "Epoch 121/150\n",
      "Epoch 00121: val_loss did not improve\n",
      " - 6s - loss: 0.1101 - acc: 0.9588 - val_loss: 0.2403 - val_acc: 0.9097\n",
      "Epoch 122/150\n",
      "Epoch 00122: val_loss did not improve\n",
      " - 5s - loss: 0.1128 - acc: 0.9489 - val_loss: 0.2701 - val_acc: 0.9065\n",
      "Epoch 123/150\n",
      "Epoch 00123: val_loss did not improve\n",
      " - 5s - loss: 0.1006 - acc: 0.9581 - val_loss: 0.2497 - val_acc: 0.9221\n",
      "Epoch 124/150\n",
      "Epoch 00124: val_loss did not improve\n",
      " - 6s - loss: 0.1103 - acc: 0.9535 - val_loss: 0.2345 - val_acc: 0.9221\n",
      "Epoch 125/150\n",
      "Epoch 00125: val_loss did not improve\n",
      " - 6s - loss: 0.1064 - acc: 0.9615 - val_loss: 0.2842 - val_acc: 0.8941\n",
      "Epoch 126/150\n",
      "Epoch 00126: val_loss did not improve\n",
      " - 5s - loss: 0.1047 - acc: 0.9565 - val_loss: 0.2593 - val_acc: 0.9252\n",
      "Epoch 127/150\n",
      "Epoch 00127: val_loss did not improve\n",
      " - 5s - loss: 0.1149 - acc: 0.9474 - val_loss: 0.2568 - val_acc: 0.9097\n",
      "Epoch 128/150\n",
      "Epoch 00128: val_loss did not improve\n",
      " - 5s - loss: 0.0819 - acc: 0.9741 - val_loss: 0.3002 - val_acc: 0.9065\n",
      "Epoch 129/150\n",
      "Epoch 00129: val_loss did not improve\n",
      " - 5s - loss: 0.0959 - acc: 0.9619 - val_loss: 0.2695 - val_acc: 0.9097\n",
      "Epoch 130/150\n",
      "Epoch 00130: val_loss did not improve\n",
      " - 5s - loss: 0.0875 - acc: 0.9634 - val_loss: 0.2915 - val_acc: 0.9097\n",
      "Epoch 131/150\n",
      "Epoch 00131: val_loss did not improve\n",
      " - 5s - loss: 0.1001 - acc: 0.9596 - val_loss: 0.2840 - val_acc: 0.9034\n",
      "Epoch 132/150\n",
      "Epoch 00132: val_loss did not improve\n",
      " - 5s - loss: 0.0757 - acc: 0.9664 - val_loss: 0.2770 - val_acc: 0.9034\n",
      "Epoch 133/150\n",
      "Epoch 00133: val_loss did not improve\n",
      " - 5s - loss: 0.0890 - acc: 0.9611 - val_loss: 0.2639 - val_acc: 0.9128\n",
      "Epoch 134/150\n",
      "Epoch 00134: val_loss did not improve\n",
      " - 6s - loss: 0.0782 - acc: 0.9687 - val_loss: 0.3050 - val_acc: 0.9003\n",
      "Epoch 135/150\n",
      "Epoch 00135: val_loss did not improve\n",
      " - 6s - loss: 0.0729 - acc: 0.9680 - val_loss: 0.2685 - val_acc: 0.9221\n",
      "Epoch 136/150\n",
      "Epoch 00136: val_loss did not improve\n",
      " - 6s - loss: 0.0755 - acc: 0.9703 - val_loss: 0.3033 - val_acc: 0.9065\n",
      "Epoch 137/150\n",
      "Epoch 00137: val_loss did not improve\n",
      " - 5s - loss: 0.0834 - acc: 0.9672 - val_loss: 0.3156 - val_acc: 0.8972\n",
      "Epoch 138/150\n",
      "Epoch 00138: val_loss did not improve\n",
      " - 5s - loss: 0.0761 - acc: 0.9676 - val_loss: 0.3008 - val_acc: 0.9128\n",
      "Epoch 139/150\n",
      "Epoch 00139: val_loss did not improve\n",
      " - 5s - loss: 0.0907 - acc: 0.9634 - val_loss: 0.2980 - val_acc: 0.9097\n",
      "CV3, Done!\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_21 (InputLayer)           (None, 75, 75, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_81 (Conv2D)              (None, 71, 71, 64)   4864        input_21[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_81 (Activation)      (None, 71, 71, 64)   0           conv2d_81[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_81 (MaxPooling2D) (None, 35, 35, 64)   0           activation_81[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_121 (Dropout)           (None, 35, 35, 64)   0           max_pooling2d_81[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_82 (Conv2D)              (None, 33, 33, 128)  73856       dropout_121[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_82 (Activation)      (None, 33, 33, 128)  0           conv2d_82[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_82 (MaxPooling2D) (None, 16, 16, 128)  0           activation_82[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_122 (Dropout)           (None, 16, 16, 128)  0           max_pooling2d_82[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_83 (Conv2D)              (None, 14, 14, 128)  147584      dropout_122[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_83 (Activation)      (None, 14, 14, 128)  0           conv2d_83[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_83 (MaxPooling2D) (None, 7, 7, 128)    0           activation_83[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_123 (Dropout)           (None, 7, 7, 128)    0           max_pooling2d_83[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_84 (Conv2D)              (None, 5, 5, 64)     73792       dropout_123[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_84 (Activation)      (None, 5, 5, 64)     0           conv2d_84[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_84 (MaxPooling2D) (None, 2, 2, 64)     0           activation_84[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "other (InputLayer)              (None, 13)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_124 (Dropout)           (None, 2, 2, 64)     0           max_pooling2d_84[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_101 (Dense)               (None, 16)           224         other[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_21 (Flatten)            (None, 256)          0           dropout_124[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_102 (Dense)               (None, 16)           272         dense_101[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_21 (Concatenate)    (None, 272)          0           flatten_21[0][0]                 \n",
      "                                                                 dense_102[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_103 (Dense)               (None, 256)          69888       concatenate_21[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_125 (Dropout)           (None, 256)          0           dense_103[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_104 (Dense)               (None, 256)          65792       dropout_125[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_126 (Dropout)           (None, 256)          0           dense_104[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_105 (Dense)               (None, 1)            257         dropout_126[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 436,529\n",
      "Trainable params: 431,665\n",
      "Non-trainable params: 4,864\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/150\n",
      "Epoch 00001: val_loss improved from inf to 0.24218, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage2.hdf5\n",
      " - 10s - loss: 0.2943 - acc: 0.8887 - val_loss: 0.2422 - val_acc: 0.8785\n",
      "Epoch 2/150\n",
      "Epoch 00002: val_loss improved from 0.24218 to 0.22012, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage2.hdf5\n",
      " - 4s - loss: 0.2752 - acc: 0.8940 - val_loss: 0.2201 - val_acc: 0.8910\n",
      "Epoch 3/150\n",
      "Epoch 00003: val_loss improved from 0.22012 to 0.21493, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv3_stage2.hdf5\n",
      " - 4s - loss: 0.2572 - acc: 0.8807 - val_loss: 0.2149 - val_acc: 0.9128\n",
      "Epoch 4/150\n",
      "Epoch 00004: val_loss did not improve\n",
      " - 4s - loss: 0.2448 - acc: 0.8906 - val_loss: 0.2212 - val_acc: 0.9159\n",
      "Epoch 5/150\n",
      "Epoch 00005: val_loss did not improve\n",
      " - 4s - loss: 0.2356 - acc: 0.8982 - val_loss: 0.2326 - val_acc: 0.9034\n",
      "Epoch 6/150\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 4s - loss: 0.2013 - acc: 0.9184 - val_loss: 0.2405 - val_acc: 0.9065\n",
      "Epoch 7/150\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 4s - loss: 0.2356 - acc: 0.9001 - val_loss: 0.2333 - val_acc: 0.9097\n",
      "Epoch 8/150\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 4s - loss: 0.2326 - acc: 0.9051 - val_loss: 0.2336 - val_acc: 0.8972\n",
      "Epoch 9/150\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 4s - loss: 0.2005 - acc: 0.9169 - val_loss: 0.2303 - val_acc: 0.9003\n",
      "Epoch 10/150\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 4s - loss: 0.2027 - acc: 0.9138 - val_loss: 0.2278 - val_acc: 0.9159\n",
      "Epoch 11/150\n",
      "Epoch 00011: val_loss did not improve\n",
      " - 4s - loss: 0.2122 - acc: 0.9104 - val_loss: 0.2227 - val_acc: 0.9097\n",
      "Epoch 12/150\n",
      "Epoch 00012: val_loss did not improve\n",
      " - 4s - loss: 0.1910 - acc: 0.9169 - val_loss: 0.2435 - val_acc: 0.9034\n",
      "Epoch 13/150\n",
      "Epoch 00013: val_loss did not improve\n",
      " - 4s - loss: 0.2016 - acc: 0.9199 - val_loss: 0.2444 - val_acc: 0.9097\n",
      "Epoch 14/150\n",
      "Epoch 00014: val_loss did not improve\n",
      " - 4s - loss: 0.1959 - acc: 0.9207 - val_loss: 0.2268 - val_acc: 0.9190\n",
      "Epoch 15/150\n",
      "Epoch 00015: val_loss did not improve\n",
      " - 4s - loss: 0.1889 - acc: 0.9150 - val_loss: 0.2360 - val_acc: 0.9065\n",
      "Epoch 16/150\n",
      "Epoch 00016: val_loss did not improve\n",
      " - 4s - loss: 0.1886 - acc: 0.9146 - val_loss: 0.2288 - val_acc: 0.9221\n",
      "Epoch 17/150\n",
      "Epoch 00017: val_loss did not improve\n",
      " - 4s - loss: 0.1862 - acc: 0.9215 - val_loss: 0.2239 - val_acc: 0.9128\n",
      "Epoch 18/150\n",
      "Epoch 00018: val_loss did not improve\n",
      " - 4s - loss: 0.1711 - acc: 0.9283 - val_loss: 0.2371 - val_acc: 0.9221\n",
      "Epoch 19/150\n",
      "Epoch 00019: val_loss did not improve\n",
      " - 4s - loss: 0.1844 - acc: 0.9291 - val_loss: 0.2334 - val_acc: 0.9128\n",
      "Epoch 20/150\n",
      "Epoch 00020: val_loss did not improve\n",
      " - 4s - loss: 0.1794 - acc: 0.9173 - val_loss: 0.2355 - val_acc: 0.9190\n",
      "Epoch 21/150\n",
      "Epoch 00021: val_loss did not improve\n",
      " - 4s - loss: 0.1966 - acc: 0.9161 - val_loss: 0.2309 - val_acc: 0.9159\n",
      "Epoch 22/150\n",
      "Epoch 00022: val_loss did not improve\n",
      " - 4s - loss: 0.1737 - acc: 0.9283 - val_loss: 0.2318 - val_acc: 0.9252\n",
      "Epoch 23/150\n",
      "Epoch 00023: val_loss did not improve\n",
      " - 4s - loss: 0.1809 - acc: 0.9219 - val_loss: 0.2592 - val_acc: 0.9065\n",
      "Epoch 24/150\n",
      "Epoch 00024: val_loss did not improve\n",
      " - 4s - loss: 0.1646 - acc: 0.9344 - val_loss: 0.2325 - val_acc: 0.9190\n",
      "Epoch 25/150\n",
      "Epoch 00025: val_loss did not improve\n",
      " - 4s - loss: 0.1732 - acc: 0.9283 - val_loss: 0.2322 - val_acc: 0.9221\n",
      "Epoch 26/150\n",
      "Epoch 00026: val_loss did not improve\n",
      " - 4s - loss: 0.1681 - acc: 0.9298 - val_loss: 0.2332 - val_acc: 0.9252\n",
      "Epoch 27/150\n",
      "Epoch 00027: val_loss did not improve\n",
      " - 4s - loss: 0.1682 - acc: 0.9291 - val_loss: 0.2386 - val_acc: 0.9221\n",
      "Epoch 28/150\n",
      "Epoch 00028: val_loss did not improve\n",
      " - 4s - loss: 0.1760 - acc: 0.9242 - val_loss: 0.2322 - val_acc: 0.9221\n",
      "Epoch 29/150\n",
      "Epoch 00029: val_loss did not improve\n",
      " - 4s - loss: 0.1793 - acc: 0.9268 - val_loss: 0.2272 - val_acc: 0.9221\n",
      "Epoch 30/150\n",
      "Epoch 00030: val_loss did not improve\n",
      " - 4s - loss: 0.1680 - acc: 0.9260 - val_loss: 0.2321 - val_acc: 0.9190\n",
      "Epoch 31/150\n",
      "Epoch 00031: val_loss did not improve\n",
      " - 4s - loss: 0.1644 - acc: 0.9337 - val_loss: 0.2406 - val_acc: 0.9097\n",
      "Epoch 32/150\n",
      "Epoch 00032: val_loss did not improve\n",
      " - 4s - loss: 0.1654 - acc: 0.9329 - val_loss: 0.2328 - val_acc: 0.9097\n",
      "Epoch 33/150\n",
      "Epoch 00033: val_loss did not improve\n",
      " - 4s - loss: 0.1673 - acc: 0.9298 - val_loss: 0.2247 - val_acc: 0.9221\n",
      "CV3, Done!\n",
      "1283/1283 [==============================] - 2s 1ms/step\n",
      "Train loss: 0.103064124973\n",
      "Train accuracy: 0.968823070928\n",
      "321/321 [==============================] - 0s 1ms/step\n",
      "Valid loss: 0.214931747063\n",
      "Valid accuracy: 0.91277258567\n",
      "\n",
      "===================FOLD= 4\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_22 (InputLayer)           (None, 75, 75, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_85 (Conv2D)              (None, 71, 71, 64)   4864        input_22[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_85 (Activation)      (None, 71, 71, 64)   0           conv2d_85[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_85 (MaxPooling2D) (None, 35, 35, 64)   0           activation_85[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_127 (Dropout)           (None, 35, 35, 64)   0           max_pooling2d_85[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_86 (Conv2D)              (None, 33, 33, 128)  73856       dropout_127[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_86 (Activation)      (None, 33, 33, 128)  0           conv2d_86[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_86 (MaxPooling2D) (None, 16, 16, 128)  0           activation_86[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_128 (Dropout)           (None, 16, 16, 128)  0           max_pooling2d_86[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_87 (Conv2D)              (None, 14, 14, 128)  147584      dropout_128[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_87 (Activation)      (None, 14, 14, 128)  0           conv2d_87[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_87 (MaxPooling2D) (None, 7, 7, 128)    0           activation_87[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_129 (Dropout)           (None, 7, 7, 128)    0           max_pooling2d_87[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_88 (Conv2D)              (None, 5, 5, 64)     73792       dropout_129[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_88 (Activation)      (None, 5, 5, 64)     0           conv2d_88[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_88 (MaxPooling2D) (None, 2, 2, 64)     0           activation_88[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "other (InputLayer)              (None, 13)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_130 (Dropout)           (None, 2, 2, 64)     0           max_pooling2d_88[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_106 (Dense)               (None, 16)           224         other[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_22 (Flatten)            (None, 256)          0           dropout_130[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_107 (Dense)               (None, 16)           272         dense_106[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_22 (Concatenate)    (None, 272)          0           flatten_22[0][0]                 \n",
      "                                                                 dense_107[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_108 (Dense)               (None, 256)          69888       concatenate_22[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_131 (Dropout)           (None, 256)          0           dense_108[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_109 (Dense)               (None, 256)          65792       dropout_131[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_132 (Dropout)           (None, 256)          0           dense_109[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_110 (Dense)               (None, 1)            257         dropout_132[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 436,529\n",
      "Trainable params: 436,529\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/150\n",
      "Epoch 00001: val_loss improved from inf to 0.67361, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 11s - loss: 1.1802 - acc: 0.5318 - val_loss: 0.6736 - val_acc: 0.5531\n",
      "Epoch 2/150\n",
      "Epoch 00002: val_loss improved from 0.67361 to 0.64773, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.7182 - acc: 0.5404 - val_loss: 0.6477 - val_acc: 0.5969\n",
      "Epoch 3/150\n",
      "Epoch 00003: val_loss improved from 0.64773 to 0.60446, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.6530 - acc: 0.5785 - val_loss: 0.6045 - val_acc: 0.5625\n",
      "Epoch 4/150\n",
      "Epoch 00004: val_loss improved from 0.60446 to 0.58758, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.6091 - acc: 0.6134 - val_loss: 0.5876 - val_acc: 0.6125\n",
      "Epoch 5/150\n",
      "Epoch 00005: val_loss improved from 0.58758 to 0.57466, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.5834 - acc: 0.6309 - val_loss: 0.5747 - val_acc: 0.6500\n",
      "Epoch 6/150\n",
      "Epoch 00006: val_loss improved from 0.57466 to 0.56806, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.5747 - acc: 0.6313 - val_loss: 0.5681 - val_acc: 0.6531\n",
      "Epoch 7/150\n",
      "Epoch 00007: val_loss improved from 0.56806 to 0.56787, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.5535 - acc: 0.6684 - val_loss: 0.5679 - val_acc: 0.6281\n",
      "Epoch 8/150\n",
      "Epoch 00008: val_loss improved from 0.56787 to 0.54663, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.5390 - acc: 0.6904 - val_loss: 0.5466 - val_acc: 0.7188\n",
      "Epoch 9/150\n",
      "Epoch 00009: val_loss improved from 0.54663 to 0.53227, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.5114 - acc: 0.7195 - val_loss: 0.5323 - val_acc: 0.7281\n",
      "Epoch 10/150\n",
      "Epoch 00010: val_loss improved from 0.53227 to 0.52128, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.5010 - acc: 0.7271 - val_loss: 0.5213 - val_acc: 0.7406\n",
      "Epoch 11/150\n",
      "Epoch 00011: val_loss improved from 0.52128 to 0.48814, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.4927 - acc: 0.7425 - val_loss: 0.4881 - val_acc: 0.7719\n",
      "Epoch 12/150\n",
      "Epoch 00012: val_loss improved from 0.48814 to 0.46091, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.4523 - acc: 0.7735 - val_loss: 0.4609 - val_acc: 0.7844\n",
      "Epoch 13/150\n",
      "Epoch 00013: val_loss improved from 0.46091 to 0.45654, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.4382 - acc: 0.7852 - val_loss: 0.4565 - val_acc: 0.7969\n",
      "Epoch 14/150\n",
      "Epoch 00014: val_loss improved from 0.45654 to 0.43903, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.4084 - acc: 0.7972 - val_loss: 0.4390 - val_acc: 0.8063\n",
      "Epoch 15/150\n",
      "Epoch 00015: val_loss did not improve\n",
      " - 5s - loss: 0.4110 - acc: 0.7997 - val_loss: 0.4558 - val_acc: 0.8063\n",
      "Epoch 16/150\n",
      "Epoch 00016: val_loss improved from 0.43903 to 0.41556, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.4011 - acc: 0.8116 - val_loss: 0.4156 - val_acc: 0.8187\n",
      "Epoch 17/150\n",
      "Epoch 00017: val_loss improved from 0.41556 to 0.38958, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.3878 - acc: 0.8202 - val_loss: 0.3896 - val_acc: 0.8313\n",
      "Epoch 18/150\n",
      "Epoch 00018: val_loss did not improve\n",
      " - 5s - loss: 0.3592 - acc: 0.8240 - val_loss: 0.3972 - val_acc: 0.8531\n",
      "Epoch 19/150\n",
      "Epoch 00019: val_loss improved from 0.38958 to 0.37757, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.3637 - acc: 0.8285 - val_loss: 0.3776 - val_acc: 0.8531\n",
      "Epoch 20/150\n",
      "Epoch 00020: val_loss improved from 0.37757 to 0.35617, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.3557 - acc: 0.8360 - val_loss: 0.3562 - val_acc: 0.8656\n",
      "Epoch 21/150\n",
      "Epoch 00021: val_loss improved from 0.35617 to 0.34657, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.3596 - acc: 0.8383 - val_loss: 0.3466 - val_acc: 0.8656\n",
      "Epoch 22/150\n",
      "Epoch 00022: val_loss improved from 0.34657 to 0.34121, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.3438 - acc: 0.8475 - val_loss: 0.3412 - val_acc: 0.8469\n",
      "Epoch 23/150\n",
      "Epoch 00023: val_loss improved from 0.34121 to 0.32730, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.3202 - acc: 0.8462 - val_loss: 0.3273 - val_acc: 0.8688\n",
      "Epoch 24/150\n",
      "Epoch 00024: val_loss did not improve\n",
      " - 5s - loss: 0.3424 - acc: 0.8438 - val_loss: 0.3319 - val_acc: 0.8750\n",
      "Epoch 25/150\n",
      "Epoch 00025: val_loss did not improve\n",
      " - 5s - loss: 0.2946 - acc: 0.8696 - val_loss: 0.3281 - val_acc: 0.8656\n",
      "Epoch 26/150\n",
      "Epoch 00026: val_loss improved from 0.32730 to 0.31945, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.3153 - acc: 0.8612 - val_loss: 0.3194 - val_acc: 0.8594\n",
      "Epoch 27/150\n",
      "Epoch 00027: val_loss improved from 0.31945 to 0.31675, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.2854 - acc: 0.8795 - val_loss: 0.3168 - val_acc: 0.8719\n",
      "Epoch 28/150\n",
      "Epoch 00028: val_loss improved from 0.31675 to 0.31002, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.3015 - acc: 0.8719 - val_loss: 0.3100 - val_acc: 0.8750\n",
      "Epoch 29/150\n",
      "Epoch 00029: val_loss improved from 0.31002 to 0.29802, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.2999 - acc: 0.8635 - val_loss: 0.2980 - val_acc: 0.8781\n",
      "Epoch 30/150\n",
      "Epoch 00030: val_loss did not improve\n",
      " - 5s - loss: 0.2808 - acc: 0.8667 - val_loss: 0.3118 - val_acc: 0.8719\n",
      "Epoch 31/150\n",
      "Epoch 00031: val_loss did not improve\n",
      " - 5s - loss: 0.2933 - acc: 0.8667 - val_loss: 0.3050 - val_acc: 0.8844\n",
      "Epoch 32/150\n",
      "Epoch 00032: val_loss did not improve\n",
      " - 5s - loss: 0.2991 - acc: 0.8704 - val_loss: 0.3059 - val_acc: 0.8625\n",
      "Epoch 33/150\n",
      "Epoch 00033: val_loss improved from 0.29802 to 0.27825, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.3012 - acc: 0.8624 - val_loss: 0.2783 - val_acc: 0.8781\n",
      "Epoch 34/150\n",
      "Epoch 00034: val_loss did not improve\n",
      " - 5s - loss: 0.2865 - acc: 0.8728 - val_loss: 0.2876 - val_acc: 0.8781\n",
      "Epoch 35/150\n",
      "Epoch 00035: val_loss improved from 0.27825 to 0.27442, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.2968 - acc: 0.8589 - val_loss: 0.2744 - val_acc: 0.8875\n",
      "Epoch 36/150\n",
      "Epoch 00036: val_loss improved from 0.27442 to 0.25170, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.2528 - acc: 0.8955 - val_loss: 0.2517 - val_acc: 0.8875\n",
      "Epoch 37/150\n",
      "Epoch 00037: val_loss did not improve\n",
      " - 5s - loss: 0.2614 - acc: 0.8887 - val_loss: 0.2787 - val_acc: 0.8844\n",
      "Epoch 38/150\n",
      "Epoch 00038: val_loss did not improve\n",
      " - 5s - loss: 0.2419 - acc: 0.8957 - val_loss: 0.2587 - val_acc: 0.9000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 39/150\n",
      "Epoch 00039: val_loss did not improve\n",
      " - 5s - loss: 0.2552 - acc: 0.8828 - val_loss: 0.2559 - val_acc: 0.9031\n",
      "Epoch 40/150\n",
      "Epoch 00040: val_loss did not improve\n",
      " - 5s - loss: 0.2378 - acc: 0.8987 - val_loss: 0.2580 - val_acc: 0.9031\n",
      "Epoch 41/150\n",
      "Epoch 00041: val_loss did not improve\n",
      " - 5s - loss: 0.2548 - acc: 0.8918 - val_loss: 0.2760 - val_acc: 0.8750\n",
      "Epoch 42/150\n",
      "Epoch 00042: val_loss did not improve\n",
      " - 5s - loss: 0.2377 - acc: 0.8873 - val_loss: 0.2612 - val_acc: 0.8875\n",
      "Epoch 43/150\n",
      "Epoch 00043: val_loss did not improve\n",
      " - 5s - loss: 0.2200 - acc: 0.8986 - val_loss: 0.2579 - val_acc: 0.8969\n",
      "Epoch 44/150\n",
      "Epoch 00044: val_loss did not improve\n",
      " - 5s - loss: 0.2241 - acc: 0.8979 - val_loss: 0.2591 - val_acc: 0.9031\n",
      "Epoch 45/150\n",
      "Epoch 00045: val_loss did not improve\n",
      " - 5s - loss: 0.2028 - acc: 0.9192 - val_loss: 0.2570 - val_acc: 0.9031\n",
      "Epoch 46/150\n",
      "Epoch 00046: val_loss did not improve\n",
      " - 5s - loss: 0.2197 - acc: 0.9070 - val_loss: 0.2681 - val_acc: 0.8906\n",
      "Epoch 47/150\n",
      "Epoch 00047: val_loss did not improve\n",
      " - 5s - loss: 0.2097 - acc: 0.9085 - val_loss: 0.2908 - val_acc: 0.8812\n",
      "Epoch 48/150\n",
      "Epoch 00048: val_loss did not improve\n",
      " - 5s - loss: 0.2255 - acc: 0.8926 - val_loss: 0.2530 - val_acc: 0.8938\n",
      "Epoch 49/150\n",
      "Epoch 00049: val_loss improved from 0.25170 to 0.24587, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.2086 - acc: 0.9024 - val_loss: 0.2459 - val_acc: 0.9031\n",
      "Epoch 50/150\n",
      "Epoch 00050: val_loss improved from 0.24587 to 0.24586, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.1965 - acc: 0.9199 - val_loss: 0.2459 - val_acc: 0.9125\n",
      "Epoch 51/150\n",
      "Epoch 00051: val_loss did not improve\n",
      " - 5s - loss: 0.2241 - acc: 0.8971 - val_loss: 0.2895 - val_acc: 0.8500\n",
      "Epoch 52/150\n",
      "Epoch 00052: val_loss did not improve\n",
      " - 5s - loss: 0.2239 - acc: 0.9018 - val_loss: 0.2708 - val_acc: 0.8781\n",
      "Epoch 53/150\n",
      "Epoch 00053: val_loss improved from 0.24586 to 0.23073, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.2005 - acc: 0.9154 - val_loss: 0.2307 - val_acc: 0.9094\n",
      "Epoch 54/150\n",
      "Epoch 00054: val_loss did not improve\n",
      " - 5s - loss: 0.2079 - acc: 0.9115 - val_loss: 0.2645 - val_acc: 0.8812\n",
      "Epoch 55/150\n",
      "Epoch 00055: val_loss did not improve\n",
      " - 5s - loss: 0.1922 - acc: 0.9192 - val_loss: 0.2343 - val_acc: 0.9031\n",
      "Epoch 56/150\n",
      "Epoch 00056: val_loss did not improve\n",
      " - 5s - loss: 0.1856 - acc: 0.9161 - val_loss: 0.2425 - val_acc: 0.9062\n",
      "Epoch 57/150\n",
      "Epoch 00057: val_loss did not improve\n",
      " - 5s - loss: 0.2010 - acc: 0.9085 - val_loss: 0.2512 - val_acc: 0.9000\n",
      "Epoch 58/150\n",
      "Epoch 00058: val_loss did not improve\n",
      " - 5s - loss: 0.1992 - acc: 0.9100 - val_loss: 0.2362 - val_acc: 0.8938\n",
      "Epoch 59/150\n",
      "Epoch 00059: val_loss did not improve\n",
      " - 5s - loss: 0.2034 - acc: 0.9100 - val_loss: 0.2321 - val_acc: 0.9031\n",
      "Epoch 60/150\n",
      "Epoch 00060: val_loss did not improve\n",
      " - 5s - loss: 0.1939 - acc: 0.9178 - val_loss: 0.2446 - val_acc: 0.9062\n",
      "Epoch 61/150\n",
      "Epoch 00061: val_loss improved from 0.23073 to 0.22994, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.2190 - acc: 0.8940 - val_loss: 0.2299 - val_acc: 0.9062\n",
      "Epoch 62/150\n",
      "Epoch 00062: val_loss did not improve\n",
      " - 5s - loss: 0.1612 - acc: 0.9283 - val_loss: 0.2413 - val_acc: 0.8938\n",
      "Epoch 63/150\n",
      "Epoch 00063: val_loss did not improve\n",
      " - 5s - loss: 0.1834 - acc: 0.9138 - val_loss: 0.2720 - val_acc: 0.8719\n",
      "Epoch 64/150\n",
      "Epoch 00064: val_loss did not improve\n",
      " - 5s - loss: 0.1742 - acc: 0.9222 - val_loss: 0.3521 - val_acc: 0.8625\n",
      "Epoch 65/150\n",
      "Epoch 00065: val_loss did not improve\n",
      " - 5s - loss: 0.2165 - acc: 0.9009 - val_loss: 0.2416 - val_acc: 0.8812\n",
      "Epoch 66/150\n",
      "Epoch 00066: val_loss did not improve\n",
      " - 5s - loss: 0.1698 - acc: 0.9260 - val_loss: 0.2354 - val_acc: 0.9125\n",
      "Epoch 67/150\n",
      "Epoch 00067: val_loss did not improve\n",
      " - 5s - loss: 0.1703 - acc: 0.9237 - val_loss: 0.2327 - val_acc: 0.9094\n",
      "Epoch 68/150\n",
      "Epoch 00068: val_loss improved from 0.22994 to 0.22065, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage1.hdf5\n",
      " - 5s - loss: 0.1743 - acc: 0.9170 - val_loss: 0.2207 - val_acc: 0.9062\n",
      "Epoch 69/150\n",
      "Epoch 00069: val_loss did not improve\n",
      " - 5s - loss: 0.2284 - acc: 0.8821 - val_loss: 0.2356 - val_acc: 0.8938\n",
      "Epoch 70/150\n",
      "Epoch 00070: val_loss did not improve\n",
      " - 5s - loss: 0.1663 - acc: 0.9199 - val_loss: 0.2317 - val_acc: 0.9125\n",
      "Epoch 71/150\n",
      "Epoch 00071: val_loss did not improve\n",
      " - 5s - loss: 0.1599 - acc: 0.9306 - val_loss: 0.2308 - val_acc: 0.9250\n",
      "Epoch 72/150\n",
      "Epoch 00072: val_loss did not improve\n",
      " - 5s - loss: 0.1485 - acc: 0.9283 - val_loss: 0.2431 - val_acc: 0.9125\n",
      "Epoch 73/150\n",
      "Epoch 00073: val_loss did not improve\n",
      " - 5s - loss: 0.1707 - acc: 0.9321 - val_loss: 0.2580 - val_acc: 0.9031\n",
      "Epoch 74/150\n",
      "Epoch 00074: val_loss did not improve\n",
      " - 5s - loss: 0.1493 - acc: 0.9367 - val_loss: 0.2333 - val_acc: 0.9156\n",
      "Epoch 75/150\n",
      "Epoch 00075: val_loss did not improve\n",
      " - 5s - loss: 0.1730 - acc: 0.9276 - val_loss: 0.2391 - val_acc: 0.8969\n",
      "Epoch 76/150\n",
      "Epoch 00076: val_loss did not improve\n",
      " - 5s - loss: 0.1582 - acc: 0.9291 - val_loss: 0.2280 - val_acc: 0.9156\n",
      "Epoch 77/150\n",
      "Epoch 00077: val_loss did not improve\n",
      " - 5s - loss: 0.1582 - acc: 0.9306 - val_loss: 0.2405 - val_acc: 0.9156\n",
      "Epoch 78/150\n",
      "Epoch 00078: val_loss did not improve\n",
      " - 5s - loss: 0.1460 - acc: 0.9367 - val_loss: 0.2561 - val_acc: 0.9000\n",
      "Epoch 79/150\n",
      "Epoch 00079: val_loss did not improve\n",
      " - 5s - loss: 0.1624 - acc: 0.9245 - val_loss: 0.2623 - val_acc: 0.8938\n",
      "Epoch 80/150\n",
      "Epoch 00080: val_loss did not improve\n",
      " - 5s - loss: 0.1648 - acc: 0.9176 - val_loss: 0.2498 - val_acc: 0.9219\n",
      "Epoch 81/150\n",
      "Epoch 00081: val_loss did not improve\n",
      " - 5s - loss: 0.1422 - acc: 0.9382 - val_loss: 0.2796 - val_acc: 0.8875\n",
      "Epoch 82/150\n",
      "Epoch 00082: val_loss did not improve\n",
      " - 5s - loss: 0.1311 - acc: 0.9375 - val_loss: 0.2432 - val_acc: 0.9187\n",
      "Epoch 83/150\n",
      "Epoch 00083: val_loss did not improve\n",
      " - 5s - loss: 0.1574 - acc: 0.9321 - val_loss: 0.2472 - val_acc: 0.9187\n",
      "Epoch 84/150\n",
      "Epoch 00084: val_loss did not improve\n",
      " - 5s - loss: 0.1525 - acc: 0.9329 - val_loss: 0.2705 - val_acc: 0.8844\n",
      "Epoch 85/150\n",
      "Epoch 00085: val_loss did not improve\n",
      " - 5s - loss: 0.1423 - acc: 0.9382 - val_loss: 0.2589 - val_acc: 0.9125\n",
      "Epoch 86/150\n",
      "Epoch 00086: val_loss did not improve\n",
      " - 5s - loss: 0.1358 - acc: 0.9436 - val_loss: 0.2627 - val_acc: 0.9094\n",
      "Epoch 87/150\n",
      "Epoch 00087: val_loss did not improve\n",
      " - 5s - loss: 0.1336 - acc: 0.9420 - val_loss: 0.2588 - val_acc: 0.9031\n",
      "Epoch 88/150\n",
      "Epoch 00088: val_loss did not improve\n",
      " - 5s - loss: 0.1186 - acc: 0.9451 - val_loss: 0.3041 - val_acc: 0.8969\n",
      "Epoch 89/150\n",
      "Epoch 00089: val_loss did not improve\n",
      " - 5s - loss: 0.1565 - acc: 0.9314 - val_loss: 0.2375 - val_acc: 0.9187\n",
      "Epoch 90/150\n",
      "Epoch 00090: val_loss did not improve\n",
      " - 5s - loss: 0.1418 - acc: 0.9443 - val_loss: 0.2447 - val_acc: 0.9156\n",
      "Epoch 91/150\n",
      "Epoch 00091: val_loss did not improve\n",
      " - 5s - loss: 0.1293 - acc: 0.9443 - val_loss: 0.2622 - val_acc: 0.9187\n",
      "Epoch 92/150\n",
      "Epoch 00092: val_loss did not improve\n",
      " - 5s - loss: 0.1236 - acc: 0.9451 - val_loss: 0.2576 - val_acc: 0.8969\n",
      "Epoch 93/150\n",
      "Epoch 00093: val_loss did not improve\n",
      " - 5s - loss: 0.1232 - acc: 0.9542 - val_loss: 0.2476 - val_acc: 0.9156\n",
      "Epoch 94/150\n",
      "Epoch 00094: val_loss did not improve\n",
      " - 5s - loss: 0.1428 - acc: 0.9382 - val_loss: 0.2606 - val_acc: 0.8938\n",
      "Epoch 95/150\n",
      "Epoch 00095: val_loss did not improve\n",
      " - 5s - loss: 0.1093 - acc: 0.9481 - val_loss: 0.3577 - val_acc: 0.8781\n",
      "Epoch 96/150\n",
      "Epoch 00096: val_loss did not improve\n",
      " - 5s - loss: 0.1395 - acc: 0.9390 - val_loss: 0.3623 - val_acc: 0.8656\n",
      "Epoch 97/150\n",
      "Epoch 00097: val_loss did not improve\n",
      " - 5s - loss: 0.1165 - acc: 0.9573 - val_loss: 0.2519 - val_acc: 0.9125\n",
      "Epoch 98/150\n",
      "Epoch 00098: val_loss did not improve\n",
      " - 5s - loss: 0.1301 - acc: 0.9481 - val_loss: 0.2628 - val_acc: 0.9062\n",
      "CV4, Done!\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_23 (InputLayer)           (None, 75, 75, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_89 (Conv2D)              (None, 71, 71, 64)   4864        input_23[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_89 (Activation)      (None, 71, 71, 64)   0           conv2d_89[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_89 (MaxPooling2D) (None, 35, 35, 64)   0           activation_89[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_133 (Dropout)           (None, 35, 35, 64)   0           max_pooling2d_89[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_90 (Conv2D)              (None, 33, 33, 128)  73856       dropout_133[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_90 (Activation)      (None, 33, 33, 128)  0           conv2d_90[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_90 (MaxPooling2D) (None, 16, 16, 128)  0           activation_90[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_134 (Dropout)           (None, 16, 16, 128)  0           max_pooling2d_90[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_91 (Conv2D)              (None, 14, 14, 128)  147584      dropout_134[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_91 (Activation)      (None, 14, 14, 128)  0           conv2d_91[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_91 (MaxPooling2D) (None, 7, 7, 128)    0           activation_91[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_135 (Dropout)           (None, 7, 7, 128)    0           max_pooling2d_91[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_92 (Conv2D)              (None, 5, 5, 64)     73792       dropout_135[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_92 (Activation)      (None, 5, 5, 64)     0           conv2d_92[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_92 (MaxPooling2D) (None, 2, 2, 64)     0           activation_92[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "other (InputLayer)              (None, 13)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_136 (Dropout)           (None, 2, 2, 64)     0           max_pooling2d_92[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_111 (Dense)               (None, 16)           224         other[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_23 (Flatten)            (None, 256)          0           dropout_136[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_112 (Dense)               (None, 16)           272         dense_111[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_23 (Concatenate)    (None, 272)          0           flatten_23[0][0]                 \n",
      "                                                                 dense_112[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_113 (Dense)               (None, 256)          69888       concatenate_23[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_137 (Dropout)           (None, 256)          0           dense_113[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_114 (Dense)               (None, 256)          65792       dropout_137[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_138 (Dropout)           (None, 256)          0           dense_114[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_115 (Dense)               (None, 1)            257         dropout_138[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 436,529\n",
      "Trainable params: 431,665\n",
      "Non-trainable params: 4,864\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/150\n",
      "Epoch 00001: val_loss improved from inf to 0.22001, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage2.hdf5\n",
      " - 10s - loss: 0.3082 - acc: 0.8566 - val_loss: 0.2200 - val_acc: 0.8906\n",
      "Epoch 2/150\n",
      "Epoch 00002: val_loss did not improve\n",
      " - 4s - loss: 0.2647 - acc: 0.8810 - val_loss: 0.2241 - val_acc: 0.8969\n",
      "Epoch 3/150\n",
      "Epoch 00003: val_loss did not improve\n",
      " - 4s - loss: 0.2953 - acc: 0.8690 - val_loss: 0.2271 - val_acc: 0.8875\n",
      "Epoch 4/150\n",
      "Epoch 00004: val_loss did not improve\n",
      " - 4s - loss: 0.2689 - acc: 0.8810 - val_loss: 0.2236 - val_acc: 0.8844\n",
      "Epoch 5/150\n",
      "Epoch 00005: val_loss improved from 0.22001 to 0.21905, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage2.hdf5\n",
      " - 4s - loss: 0.2583 - acc: 0.8796 - val_loss: 0.2190 - val_acc: 0.8906\n",
      "Epoch 6/150\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 4s - loss: 0.2676 - acc: 0.8856 - val_loss: 0.2249 - val_acc: 0.8875\n",
      "Epoch 7/150\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 4s - loss: 0.2522 - acc: 0.8963 - val_loss: 0.2219 - val_acc: 0.8875\n",
      "Epoch 8/150\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 4s - loss: 0.2392 - acc: 0.8978 - val_loss: 0.2208 - val_acc: 0.8906\n",
      "Epoch 9/150\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 4s - loss: 0.2426 - acc: 0.8848 - val_loss: 0.2253 - val_acc: 0.8938\n",
      "Epoch 10/150\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 4s - loss: 0.2401 - acc: 0.8932 - val_loss: 0.2282 - val_acc: 0.8969\n",
      "Epoch 11/150\n",
      "Epoch 00011: val_loss did not improve\n",
      " - 4s - loss: 0.2308 - acc: 0.8964 - val_loss: 0.2192 - val_acc: 0.8938\n",
      "Epoch 12/150\n",
      "Epoch 00012: val_loss did not improve\n",
      " - 4s - loss: 0.2339 - acc: 0.8955 - val_loss: 0.2191 - val_acc: 0.8875\n",
      "Epoch 13/150\n",
      "Epoch 00013: val_loss did not improve\n",
      " - 4s - loss: 0.2491 - acc: 0.8934 - val_loss: 0.2260 - val_acc: 0.8938\n",
      "Epoch 14/150\n",
      "Epoch 00014: val_loss did not improve\n",
      " - 4s - loss: 0.2425 - acc: 0.9032 - val_loss: 0.2336 - val_acc: 0.8844\n",
      "Epoch 15/150\n",
      "Epoch 00015: val_loss did not improve\n",
      " - 4s - loss: 0.2553 - acc: 0.8904 - val_loss: 0.2202 - val_acc: 0.8906\n",
      "Epoch 16/150\n",
      "Epoch 00016: val_loss did not improve\n",
      " - 4s - loss: 0.2403 - acc: 0.8887 - val_loss: 0.2262 - val_acc: 0.8938\n",
      "Epoch 17/150\n",
      "Epoch 00017: val_loss did not improve\n",
      " - 4s - loss: 0.2181 - acc: 0.9032 - val_loss: 0.2333 - val_acc: 0.8875\n",
      "Epoch 18/150\n",
      "Epoch 00018: val_loss improved from 0.21905 to 0.21775, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage2.hdf5\n",
      " - 4s - loss: 0.2157 - acc: 0.9054 - val_loss: 0.2178 - val_acc: 0.9000\n",
      "Epoch 19/150\n",
      "Epoch 00019: val_loss did not improve\n",
      " - 4s - loss: 0.2370 - acc: 0.8973 - val_loss: 0.2320 - val_acc: 0.8844\n",
      "Epoch 20/150\n",
      "Epoch 00020: val_loss did not improve\n",
      " - 4s - loss: 0.2357 - acc: 0.8963 - val_loss: 0.2183 - val_acc: 0.9031\n",
      "Epoch 21/150\n",
      "Epoch 00021: val_loss did not improve\n",
      " - 4s - loss: 0.2069 - acc: 0.9010 - val_loss: 0.2337 - val_acc: 0.8938\n",
      "Epoch 22/150\n",
      "Epoch 00022: val_loss did not improve\n",
      " - 4s - loss: 0.2179 - acc: 0.9062 - val_loss: 0.2264 - val_acc: 0.8906\n",
      "Epoch 23/150\n",
      "Epoch 00023: val_loss did not improve\n",
      " - 4s - loss: 0.2121 - acc: 0.9093 - val_loss: 0.2255 - val_acc: 0.8969\n",
      "Epoch 24/150\n",
      "Epoch 00024: val_loss did not improve\n",
      " - 4s - loss: 0.2088 - acc: 0.9123 - val_loss: 0.2267 - val_acc: 0.9031\n",
      "Epoch 25/150\n",
      "Epoch 00025: val_loss did not improve\n",
      " - 4s - loss: 0.2115 - acc: 0.8993 - val_loss: 0.2235 - val_acc: 0.8969\n",
      "Epoch 26/150\n",
      "Epoch 00026: val_loss did not improve\n",
      " - 4s - loss: 0.2193 - acc: 0.9085 - val_loss: 0.2317 - val_acc: 0.8938\n",
      "Epoch 27/150\n",
      "Epoch 00027: val_loss did not improve\n",
      " - 4s - loss: 0.2028 - acc: 0.9215 - val_loss: 0.2214 - val_acc: 0.8906\n",
      "Epoch 28/150\n",
      "Epoch 00028: val_loss did not improve\n",
      " - 4s - loss: 0.2029 - acc: 0.9085 - val_loss: 0.2393 - val_acc: 0.8844\n",
      "Epoch 29/150\n",
      "Epoch 00029: val_loss improved from 0.21775 to 0.21498, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage2.hdf5\n",
      " - 4s - loss: 0.2243 - acc: 0.8940 - val_loss: 0.2150 - val_acc: 0.8938\n",
      "Epoch 30/150\n",
      "Epoch 00030: val_loss did not improve\n",
      " - 4s - loss: 0.2154 - acc: 0.9117 - val_loss: 0.2209 - val_acc: 0.8969\n",
      "Epoch 31/150\n",
      "Epoch 00031: val_loss did not improve\n",
      " - 4s - loss: 0.2038 - acc: 0.9123 - val_loss: 0.2253 - val_acc: 0.9000\n",
      "Epoch 32/150\n",
      "Epoch 00032: val_loss did not improve\n",
      " - 4s - loss: 0.2087 - acc: 0.9010 - val_loss: 0.2243 - val_acc: 0.9000\n",
      "Epoch 33/150\n",
      "Epoch 00033: val_loss did not improve\n",
      " - 4s - loss: 0.2153 - acc: 0.9048 - val_loss: 0.2219 - val_acc: 0.9031\n",
      "Epoch 34/150\n",
      "Epoch 00034: val_loss did not improve\n",
      " - 4s - loss: 0.1913 - acc: 0.9245 - val_loss: 0.2275 - val_acc: 0.9000\n",
      "Epoch 35/150\n",
      "Epoch 00035: val_loss did not improve\n",
      " - 4s - loss: 0.2093 - acc: 0.9108 - val_loss: 0.2159 - val_acc: 0.8969\n",
      "Epoch 36/150\n",
      "Epoch 00036: val_loss improved from 0.21498 to 0.21326, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage2.hdf5\n",
      " - 4s - loss: 0.1886 - acc: 0.9253 - val_loss: 0.2133 - val_acc: 0.9094\n",
      "Epoch 37/150\n",
      "Epoch 00037: val_loss did not improve\n",
      " - 4s - loss: 0.2031 - acc: 0.8993 - val_loss: 0.2163 - val_acc: 0.9031\n",
      "Epoch 38/150\n",
      "Epoch 00038: val_loss did not improve\n",
      " - 4s - loss: 0.2047 - acc: 0.9079 - val_loss: 0.2312 - val_acc: 0.8969\n",
      "Epoch 39/150\n",
      "Epoch 00039: val_loss improved from 0.21326 to 0.21236, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage2.hdf5\n",
      " - 4s - loss: 0.1878 - acc: 0.9222 - val_loss: 0.2124 - val_acc: 0.9000\n",
      "Epoch 40/150\n",
      "Epoch 00040: val_loss did not improve\n",
      " - 4s - loss: 0.1972 - acc: 0.9208 - val_loss: 0.2197 - val_acc: 0.9000\n",
      "Epoch 41/150\n",
      "Epoch 00041: val_loss did not improve\n",
      " - 4s - loss: 0.2044 - acc: 0.9079 - val_loss: 0.2489 - val_acc: 0.8906\n",
      "Epoch 42/150\n",
      "Epoch 00042: val_loss improved from 0.21236 to 0.20621, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage2.hdf5\n",
      " - 4s - loss: 0.2108 - acc: 0.9108 - val_loss: 0.2062 - val_acc: 0.9031\n",
      "Epoch 43/150\n",
      "Epoch 00043: val_loss did not improve\n",
      " - 4s - loss: 0.1888 - acc: 0.9222 - val_loss: 0.2265 - val_acc: 0.9062\n",
      "Epoch 44/150\n",
      "Epoch 00044: val_loss did not improve\n",
      " - 4s - loss: 0.1873 - acc: 0.9162 - val_loss: 0.2137 - val_acc: 0.8969\n",
      "Epoch 45/150\n",
      "Epoch 00045: val_loss did not improve\n",
      " - 4s - loss: 0.2084 - acc: 0.9048 - val_loss: 0.2182 - val_acc: 0.8938\n",
      "Epoch 46/150\n",
      "Epoch 00046: val_loss did not improve\n",
      " - 4s - loss: 0.1770 - acc: 0.9245 - val_loss: 0.2083 - val_acc: 0.9094\n",
      "Epoch 47/150\n",
      "Epoch 00047: val_loss did not improve\n",
      " - 4s - loss: 0.1741 - acc: 0.9321 - val_loss: 0.2152 - val_acc: 0.9094\n",
      "Epoch 48/150\n",
      "Epoch 00048: val_loss did not improve\n",
      " - 4s - loss: 0.1918 - acc: 0.9178 - val_loss: 0.2565 - val_acc: 0.8875\n",
      "Epoch 49/150\n",
      "Epoch 00049: val_loss did not improve\n",
      " - 4s - loss: 0.1981 - acc: 0.9154 - val_loss: 0.2275 - val_acc: 0.8969\n",
      "Epoch 50/150\n",
      "Epoch 00050: val_loss did not improve\n",
      " - 4s - loss: 0.2000 - acc: 0.9131 - val_loss: 0.2143 - val_acc: 0.9062\n",
      "Epoch 51/150\n",
      "Epoch 00051: val_loss did not improve\n",
      " - 4s - loss: 0.1854 - acc: 0.9147 - val_loss: 0.2173 - val_acc: 0.9000\n",
      "Epoch 52/150\n",
      "Epoch 00052: val_loss did not improve\n",
      " - 4s - loss: 0.1898 - acc: 0.9161 - val_loss: 0.2125 - val_acc: 0.9094\n",
      "Epoch 53/150\n",
      "Epoch 00053: val_loss did not improve\n",
      " - 4s - loss: 0.1785 - acc: 0.9307 - val_loss: 0.2168 - val_acc: 0.9062\n",
      "Epoch 54/150\n",
      "Epoch 00054: val_loss did not improve\n",
      " - 4s - loss: 0.1756 - acc: 0.9237 - val_loss: 0.2324 - val_acc: 0.8938\n",
      "Epoch 55/150\n",
      "Epoch 00055: val_loss did not improve\n",
      " - 4s - loss: 0.2005 - acc: 0.9138 - val_loss: 0.2094 - val_acc: 0.9062\n",
      "Epoch 56/150\n",
      "Epoch 00056: val_loss did not improve\n",
      " - 4s - loss: 0.1847 - acc: 0.9314 - val_loss: 0.2097 - val_acc: 0.9031\n",
      "Epoch 57/150\n",
      "Epoch 00057: val_loss did not improve\n",
      " - 4s - loss: 0.1848 - acc: 0.9192 - val_loss: 0.2125 - val_acc: 0.9000\n",
      "Epoch 58/150\n",
      "Epoch 00058: val_loss did not improve\n",
      " - 4s - loss: 0.1783 - acc: 0.9306 - val_loss: 0.2136 - val_acc: 0.9031\n",
      "Epoch 59/150\n",
      "Epoch 00059: val_loss did not improve\n",
      " - 4s - loss: 0.1605 - acc: 0.9352 - val_loss: 0.2130 - val_acc: 0.9031\n",
      "Epoch 60/150\n",
      "Epoch 00060: val_loss improved from 0.20621 to 0.20173, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage2.hdf5\n",
      " - 4s - loss: 0.1819 - acc: 0.9300 - val_loss: 0.2017 - val_acc: 0.9125\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 61/150\n",
      "Epoch 00061: val_loss did not improve\n",
      " - 4s - loss: 0.1899 - acc: 0.9176 - val_loss: 0.2100 - val_acc: 0.8969\n",
      "Epoch 62/150\n",
      "Epoch 00062: val_loss did not improve\n",
      " - 4s - loss: 0.1757 - acc: 0.9199 - val_loss: 0.2138 - val_acc: 0.9094\n",
      "Epoch 63/150\n",
      "Epoch 00063: val_loss did not improve\n",
      " - 4s - loss: 0.1780 - acc: 0.9237 - val_loss: 0.2087 - val_acc: 0.9000\n",
      "Epoch 64/150\n",
      "Epoch 00064: val_loss did not improve\n",
      " - 4s - loss: 0.1788 - acc: 0.9268 - val_loss: 0.2130 - val_acc: 0.9094\n",
      "Epoch 65/150\n",
      "Epoch 00065: val_loss did not improve\n",
      " - 4s - loss: 0.1958 - acc: 0.9199 - val_loss: 0.2025 - val_acc: 0.9094\n",
      "Epoch 66/150\n",
      "Epoch 00066: val_loss did not improve\n",
      " - 4s - loss: 0.1623 - acc: 0.9337 - val_loss: 0.2113 - val_acc: 0.9031\n",
      "Epoch 67/150\n",
      "Epoch 00067: val_loss did not improve\n",
      " - 4s - loss: 0.1597 - acc: 0.9367 - val_loss: 0.2208 - val_acc: 0.9094\n",
      "Epoch 68/150\n",
      "Epoch 00068: val_loss did not improve\n",
      " - 4s - loss: 0.1708 - acc: 0.9276 - val_loss: 0.2061 - val_acc: 0.9156\n",
      "Epoch 69/150\n",
      "Epoch 00069: val_loss did not improve\n",
      " - 4s - loss: 0.1745 - acc: 0.9384 - val_loss: 0.2085 - val_acc: 0.9062\n",
      "Epoch 70/150\n",
      "Epoch 00070: val_loss did not improve\n",
      " - 4s - loss: 0.1581 - acc: 0.9413 - val_loss: 0.2166 - val_acc: 0.9062\n",
      "Epoch 71/150\n",
      "Epoch 00071: val_loss did not improve\n",
      " - 4s - loss: 0.1530 - acc: 0.9344 - val_loss: 0.2055 - val_acc: 0.9094\n",
      "Epoch 72/150\n",
      "Epoch 00072: val_loss did not improve\n",
      " - 4s - loss: 0.1643 - acc: 0.9291 - val_loss: 0.2044 - val_acc: 0.9125\n",
      "Epoch 73/150\n",
      "Epoch 00073: val_loss did not improve\n",
      " - 4s - loss: 0.1587 - acc: 0.9428 - val_loss: 0.2168 - val_acc: 0.9125\n",
      "Epoch 74/150\n",
      "Epoch 00074: val_loss did not improve\n",
      " - 4s - loss: 0.1648 - acc: 0.9298 - val_loss: 0.2130 - val_acc: 0.9125\n",
      "Epoch 75/150\n",
      "Epoch 00075: val_loss did not improve\n",
      " - 4s - loss: 0.1433 - acc: 0.9428 - val_loss: 0.2184 - val_acc: 0.9125\n",
      "Epoch 76/150\n",
      "Epoch 00076: val_loss did not improve\n",
      " - 4s - loss: 0.1584 - acc: 0.9321 - val_loss: 0.2245 - val_acc: 0.8906\n",
      "Epoch 77/150\n",
      "Epoch 00077: val_loss did not improve\n",
      " - 4s - loss: 0.1411 - acc: 0.9459 - val_loss: 0.2104 - val_acc: 0.9094\n",
      "Epoch 78/150\n",
      "Epoch 00078: val_loss improved from 0.20173 to 0.19833, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv4_stage2.hdf5\n",
      " - 4s - loss: 0.1582 - acc: 0.9359 - val_loss: 0.1983 - val_acc: 0.9094\n",
      "Epoch 79/150\n",
      "Epoch 00079: val_loss did not improve\n",
      " - 4s - loss: 0.1572 - acc: 0.9405 - val_loss: 0.2090 - val_acc: 0.9094\n",
      "Epoch 80/150\n",
      "Epoch 00080: val_loss did not improve\n",
      " - 4s - loss: 0.1458 - acc: 0.9436 - val_loss: 0.2413 - val_acc: 0.8969\n",
      "Epoch 81/150\n",
      "Epoch 00081: val_loss did not improve\n",
      " - 4s - loss: 0.1581 - acc: 0.9352 - val_loss: 0.2175 - val_acc: 0.9000\n",
      "Epoch 82/150\n",
      "Epoch 00082: val_loss did not improve\n",
      " - 4s - loss: 0.1578 - acc: 0.9375 - val_loss: 0.2020 - val_acc: 0.9187\n",
      "Epoch 83/150\n",
      "Epoch 00083: val_loss did not improve\n",
      " - 4s - loss: 0.1514 - acc: 0.9420 - val_loss: 0.2074 - val_acc: 0.9031\n",
      "Epoch 84/150\n",
      "Epoch 00084: val_loss did not improve\n",
      " - 4s - loss: 0.1373 - acc: 0.9398 - val_loss: 0.2081 - val_acc: 0.9187\n",
      "Epoch 85/150\n",
      "Epoch 00085: val_loss did not improve\n",
      " - 4s - loss: 0.1402 - acc: 0.9436 - val_loss: 0.2028 - val_acc: 0.9250\n",
      "Epoch 86/150\n",
      "Epoch 00086: val_loss did not improve\n",
      " - 4s - loss: 0.1404 - acc: 0.9428 - val_loss: 0.2193 - val_acc: 0.9219\n",
      "Epoch 87/150\n",
      "Epoch 00087: val_loss did not improve\n",
      " - 4s - loss: 0.1354 - acc: 0.9420 - val_loss: 0.2136 - val_acc: 0.9125\n",
      "Epoch 88/150\n",
      "Epoch 00088: val_loss did not improve\n",
      " - 4s - loss: 0.1449 - acc: 0.9459 - val_loss: 0.2082 - val_acc: 0.9125\n",
      "Epoch 89/150\n",
      "Epoch 00089: val_loss did not improve\n",
      " - 4s - loss: 0.1440 - acc: 0.9382 - val_loss: 0.2216 - val_acc: 0.9125\n",
      "Epoch 90/150\n",
      "Epoch 00090: val_loss did not improve\n",
      " - 4s - loss: 0.1480 - acc: 0.9405 - val_loss: 0.2091 - val_acc: 0.9125\n",
      "Epoch 91/150\n",
      "Epoch 00091: val_loss did not improve\n",
      " - 4s - loss: 0.1339 - acc: 0.9443 - val_loss: 0.2231 - val_acc: 0.9187\n",
      "Epoch 92/150\n",
      "Epoch 00092: val_loss did not improve\n",
      " - 4s - loss: 0.1457 - acc: 0.9398 - val_loss: 0.2080 - val_acc: 0.9156\n",
      "Epoch 93/150\n",
      "Epoch 00093: val_loss did not improve\n",
      " - 4s - loss: 0.1297 - acc: 0.9504 - val_loss: 0.2467 - val_acc: 0.9000\n",
      "Epoch 94/150\n",
      "Epoch 00094: val_loss did not improve\n",
      " - 4s - loss: 0.1645 - acc: 0.9314 - val_loss: 0.2117 - val_acc: 0.9156\n",
      "Epoch 95/150\n",
      "Epoch 00095: val_loss did not improve\n",
      " - 4s - loss: 0.1257 - acc: 0.9504 - val_loss: 0.2129 - val_acc: 0.9187\n",
      "Epoch 96/150\n",
      "Epoch 00096: val_loss did not improve\n",
      " - 4s - loss: 0.1351 - acc: 0.9422 - val_loss: 0.2221 - val_acc: 0.9156\n",
      "Epoch 97/150\n",
      "Epoch 00097: val_loss did not improve\n",
      " - 4s - loss: 0.1468 - acc: 0.9504 - val_loss: 0.2475 - val_acc: 0.9187\n",
      "Epoch 98/150\n",
      "Epoch 00098: val_loss did not improve\n",
      " - 4s - loss: 0.1647 - acc: 0.9216 - val_loss: 0.2260 - val_acc: 0.9062\n",
      "Epoch 99/150\n",
      "Epoch 00099: val_loss did not improve\n",
      " - 4s - loss: 0.1805 - acc: 0.9291 - val_loss: 0.2233 - val_acc: 0.9094\n",
      "Epoch 100/150\n",
      "Epoch 00100: val_loss did not improve\n",
      " - 4s - loss: 0.1330 - acc: 0.9504 - val_loss: 0.2338 - val_acc: 0.8969\n",
      "Epoch 101/150\n",
      "Epoch 00101: val_loss did not improve\n",
      " - 4s - loss: 0.1380 - acc: 0.9420 - val_loss: 0.2309 - val_acc: 0.9094\n",
      "Epoch 102/150\n",
      "Epoch 00102: val_loss did not improve\n",
      " - 4s - loss: 0.1294 - acc: 0.9428 - val_loss: 0.2260 - val_acc: 0.9094\n",
      "Epoch 103/150\n",
      "Epoch 00103: val_loss did not improve\n",
      " - 4s - loss: 0.1134 - acc: 0.9542 - val_loss: 0.2316 - val_acc: 0.9125\n",
      "Epoch 104/150\n",
      "Epoch 00104: val_loss did not improve\n",
      " - 4s - loss: 0.1371 - acc: 0.9459 - val_loss: 0.2212 - val_acc: 0.9094\n",
      "Epoch 105/150\n",
      "Epoch 00105: val_loss did not improve\n",
      " - 4s - loss: 0.1293 - acc: 0.9535 - val_loss: 0.2280 - val_acc: 0.9156\n",
      "Epoch 106/150\n",
      "Epoch 00106: val_loss did not improve\n",
      " - 4s - loss: 0.1144 - acc: 0.9565 - val_loss: 0.2246 - val_acc: 0.9062\n",
      "Epoch 107/150\n",
      "Epoch 00107: val_loss did not improve\n",
      " - 4s - loss: 0.1333 - acc: 0.9391 - val_loss: 0.2245 - val_acc: 0.9156\n",
      "Epoch 108/150\n",
      "Epoch 00108: val_loss did not improve\n",
      " - 4s - loss: 0.1318 - acc: 0.9498 - val_loss: 0.2296 - val_acc: 0.9156\n",
      "CV4, Done!\n",
      "1284/1284 [==============================] - 2s 1ms/step\n",
      "Train loss: 0.0902021350078\n",
      "Train accuracy: 0.968068535826\n",
      "320/320 [==============================] - 0s 1ms/step\n",
      "Valid loss: 0.198332033306\n",
      "Valid accuracy: 0.909375\n",
      "\n",
      "===================FOLD= 5\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_24 (InputLayer)           (None, 75, 75, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_93 (Conv2D)              (None, 71, 71, 64)   4864        input_24[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_93 (Activation)      (None, 71, 71, 64)   0           conv2d_93[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_93 (MaxPooling2D) (None, 35, 35, 64)   0           activation_93[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_139 (Dropout)           (None, 35, 35, 64)   0           max_pooling2d_93[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_94 (Conv2D)              (None, 33, 33, 128)  73856       dropout_139[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_94 (Activation)      (None, 33, 33, 128)  0           conv2d_94[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_94 (MaxPooling2D) (None, 16, 16, 128)  0           activation_94[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_140 (Dropout)           (None, 16, 16, 128)  0           max_pooling2d_94[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_95 (Conv2D)              (None, 14, 14, 128)  147584      dropout_140[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_95 (Activation)      (None, 14, 14, 128)  0           conv2d_95[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_95 (MaxPooling2D) (None, 7, 7, 128)    0           activation_95[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_141 (Dropout)           (None, 7, 7, 128)    0           max_pooling2d_95[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_96 (Conv2D)              (None, 5, 5, 64)     73792       dropout_141[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_96 (Activation)      (None, 5, 5, 64)     0           conv2d_96[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_96 (MaxPooling2D) (None, 2, 2, 64)     0           activation_96[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "other (InputLayer)              (None, 13)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_142 (Dropout)           (None, 2, 2, 64)     0           max_pooling2d_96[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_116 (Dense)               (None, 16)           224         other[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_24 (Flatten)            (None, 256)          0           dropout_142[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_117 (Dense)               (None, 16)           272         dense_116[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_24 (Concatenate)    (None, 272)          0           flatten_24[0][0]                 \n",
      "                                                                 dense_117[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_118 (Dense)               (None, 256)          69888       concatenate_24[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_143 (Dropout)           (None, 256)          0           dense_118[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_119 (Dense)               (None, 256)          65792       dropout_143[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_144 (Dropout)           (None, 256)          0           dense_119[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_120 (Dense)               (None, 1)            257         dropout_144[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 436,529\n",
      "Trainable params: 436,529\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/150\n",
      "Epoch 00001: val_loss improved from inf to 0.68168, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 12s - loss: 1.3316 - acc: 0.5229 - val_loss: 0.6817 - val_acc: 0.5969\n",
      "Epoch 2/150\n",
      "Epoch 00002: val_loss improved from 0.68168 to 0.65134, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.7251 - acc: 0.5334 - val_loss: 0.6513 - val_acc: 0.6031\n",
      "Epoch 3/150\n",
      "Epoch 00003: val_loss improved from 0.65134 to 0.62043, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.6710 - acc: 0.5596 - val_loss: 0.6204 - val_acc: 0.6344\n",
      "Epoch 4/150\n",
      "Epoch 00004: val_loss improved from 0.62043 to 0.59054, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.6343 - acc: 0.5905 - val_loss: 0.5905 - val_acc: 0.6469\n",
      "Epoch 5/150\n",
      "Epoch 00005: val_loss improved from 0.59054 to 0.57596, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.5934 - acc: 0.6158 - val_loss: 0.5760 - val_acc: 0.6625\n",
      "Epoch 6/150\n",
      "Epoch 00006: val_loss improved from 0.57596 to 0.56610, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.5789 - acc: 0.6409 - val_loss: 0.5661 - val_acc: 0.6594\n",
      "Epoch 7/150\n",
      "Epoch 00007: val_loss improved from 0.56610 to 0.55018, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.5581 - acc: 0.6593 - val_loss: 0.5502 - val_acc: 0.6937\n",
      "Epoch 8/150\n",
      "Epoch 00008: val_loss improved from 0.55018 to 0.53800, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.5358 - acc: 0.7018 - val_loss: 0.5380 - val_acc: 0.7312\n",
      "Epoch 9/150\n",
      "Epoch 00009: val_loss improved from 0.53800 to 0.50366, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.5169 - acc: 0.7218 - val_loss: 0.5037 - val_acc: 0.7469\n",
      "Epoch 10/150\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 5s - loss: 0.4841 - acc: 0.7485 - val_loss: 0.5235 - val_acc: 0.6875\n",
      "Epoch 11/150\n",
      "Epoch 00011: val_loss improved from 0.50366 to 0.45327, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 6s - loss: 0.4975 - acc: 0.7401 - val_loss: 0.4533 - val_acc: 0.7750\n",
      "Epoch 12/150\n",
      "Epoch 00012: val_loss improved from 0.45327 to 0.41967, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.4443 - acc: 0.7827 - val_loss: 0.4197 - val_acc: 0.8000\n",
      "Epoch 13/150\n",
      "Epoch 00013: val_loss improved from 0.41967 to 0.41302, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.4462 - acc: 0.7753 - val_loss: 0.4130 - val_acc: 0.7937\n",
      "Epoch 14/150\n",
      "Epoch 00014: val_loss improved from 0.41302 to 0.39266, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.4074 - acc: 0.8057 - val_loss: 0.3927 - val_acc: 0.8031\n",
      "Epoch 15/150\n",
      "Epoch 00015: val_loss improved from 0.39266 to 0.39204, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.4080 - acc: 0.8193 - val_loss: 0.3920 - val_acc: 0.8000\n",
      "Epoch 16/150\n",
      "Epoch 00016: val_loss improved from 0.39204 to 0.37572, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.3865 - acc: 0.8216 - val_loss: 0.3757 - val_acc: 0.8187\n",
      "Epoch 17/150\n",
      "Epoch 00017: val_loss improved from 0.37572 to 0.37078, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.3782 - acc: 0.8224 - val_loss: 0.3708 - val_acc: 0.8156\n",
      "Epoch 18/150\n",
      "Epoch 00018: val_loss improved from 0.37078 to 0.36504, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.3845 - acc: 0.8209 - val_loss: 0.3650 - val_acc: 0.8094\n",
      "Epoch 19/150\n",
      "Epoch 00019: val_loss improved from 0.36504 to 0.34447, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.3869 - acc: 0.8111 - val_loss: 0.3445 - val_acc: 0.8219\n",
      "Epoch 20/150\n",
      "Epoch 00020: val_loss did not improve\n",
      " - 5s - loss: 0.3680 - acc: 0.8353 - val_loss: 0.3447 - val_acc: 0.8219\n",
      "Epoch 21/150\n",
      "Epoch 00021: val_loss improved from 0.34447 to 0.34382, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.3513 - acc: 0.8345 - val_loss: 0.3438 - val_acc: 0.8187\n",
      "Epoch 22/150\n",
      "Epoch 00022: val_loss improved from 0.34382 to 0.32664, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.3321 - acc: 0.8467 - val_loss: 0.3266 - val_acc: 0.8500\n",
      "Epoch 23/150\n",
      "Epoch 00023: val_loss did not improve\n",
      " - 5s - loss: 0.3241 - acc: 0.8627 - val_loss: 0.3324 - val_acc: 0.8313\n",
      "Epoch 24/150\n",
      "Epoch 00024: val_loss improved from 0.32664 to 0.32170, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.3340 - acc: 0.8461 - val_loss: 0.3217 - val_acc: 0.8531\n",
      "Epoch 25/150\n",
      "Epoch 00025: val_loss improved from 0.32170 to 0.30981, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.3055 - acc: 0.8696 - val_loss: 0.3098 - val_acc: 0.8594\n",
      "Epoch 26/150\n",
      "Epoch 00026: val_loss did not improve\n",
      " - 5s - loss: 0.3170 - acc: 0.8521 - val_loss: 0.3107 - val_acc: 0.8469\n",
      "Epoch 27/150\n",
      "Epoch 00027: val_loss improved from 0.30981 to 0.29586, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.2929 - acc: 0.8644 - val_loss: 0.2959 - val_acc: 0.8594\n",
      "Epoch 28/150\n",
      "Epoch 00028: val_loss did not improve\n",
      " - 5s - loss: 0.3018 - acc: 0.8582 - val_loss: 0.3406 - val_acc: 0.8375\n",
      "Epoch 29/150\n",
      "Epoch 00029: val_loss did not improve\n",
      " - 5s - loss: 0.2916 - acc: 0.8645 - val_loss: 0.3110 - val_acc: 0.8531\n",
      "Epoch 30/150\n",
      "Epoch 00030: val_loss improved from 0.29586 to 0.28600, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 6s - loss: 0.2981 - acc: 0.8621 - val_loss: 0.2860 - val_acc: 0.8812\n",
      "Epoch 31/150\n",
      "Epoch 00031: val_loss did not improve\n",
      " - 5s - loss: 0.2797 - acc: 0.8713 - val_loss: 0.3129 - val_acc: 0.8594\n",
      "Epoch 32/150\n",
      "Epoch 00032: val_loss did not improve\n",
      " - 5s - loss: 0.2869 - acc: 0.8604 - val_loss: 0.3374 - val_acc: 0.8281\n",
      "Epoch 33/150\n",
      "Epoch 00033: val_loss improved from 0.28600 to 0.27395, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.2730 - acc: 0.8713 - val_loss: 0.2740 - val_acc: 0.8719\n",
      "Epoch 34/150\n",
      "Epoch 00034: val_loss improved from 0.27395 to 0.27103, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.2527 - acc: 0.8880 - val_loss: 0.2710 - val_acc: 0.8781\n",
      "Epoch 35/150\n",
      "Epoch 00035: val_loss improved from 0.27103 to 0.26636, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.2708 - acc: 0.8726 - val_loss: 0.2664 - val_acc: 0.8906\n",
      "Epoch 36/150\n",
      "Epoch 00036: val_loss did not improve\n",
      " - 5s - loss: 0.2344 - acc: 0.8887 - val_loss: 0.2699 - val_acc: 0.8656\n",
      "Epoch 37/150\n",
      "Epoch 00037: val_loss did not improve\n",
      " - 5s - loss: 0.2521 - acc: 0.8894 - val_loss: 0.3032 - val_acc: 0.8406\n",
      "Epoch 38/150\n",
      "Epoch 00038: val_loss improved from 0.26636 to 0.25584, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.2327 - acc: 0.8993 - val_loss: 0.2558 - val_acc: 0.8875\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 39/150\n",
      "Epoch 00039: val_loss did not improve\n",
      " - 5s - loss: 0.2279 - acc: 0.9032 - val_loss: 0.2639 - val_acc: 0.8781\n",
      "Epoch 40/150\n",
      "Epoch 00040: val_loss did not improve\n",
      " - 5s - loss: 0.2249 - acc: 0.9077 - val_loss: 0.3077 - val_acc: 0.8500\n",
      "Epoch 41/150\n",
      "Epoch 00041: val_loss did not improve\n",
      " - 5s - loss: 0.2393 - acc: 0.8911 - val_loss: 0.3289 - val_acc: 0.8375\n",
      "Epoch 42/150\n",
      "Epoch 00042: val_loss did not improve\n",
      " - 5s - loss: 0.2298 - acc: 0.8955 - val_loss: 0.3282 - val_acc: 0.8438\n",
      "Epoch 43/150\n",
      "Epoch 00043: val_loss did not improve\n",
      " - 5s - loss: 0.2208 - acc: 0.9032 - val_loss: 0.3358 - val_acc: 0.8375\n",
      "Epoch 44/150\n",
      "Epoch 00044: val_loss did not improve\n",
      " - 5s - loss: 0.2533 - acc: 0.8917 - val_loss: 0.2891 - val_acc: 0.8656\n",
      "Epoch 45/150\n",
      "Epoch 00045: val_loss did not improve\n",
      " - 5s - loss: 0.2094 - acc: 0.9054 - val_loss: 0.2672 - val_acc: 0.8812\n",
      "Epoch 46/150\n",
      "Epoch 00046: val_loss improved from 0.25584 to 0.23998, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.1917 - acc: 0.9146 - val_loss: 0.2400 - val_acc: 0.9000\n",
      "Epoch 47/150\n",
      "Epoch 00047: val_loss did not improve\n",
      " - 5s - loss: 0.2103 - acc: 0.9009 - val_loss: 0.2429 - val_acc: 0.9031\n",
      "Epoch 48/150\n",
      "Epoch 00048: val_loss did not improve\n",
      " - 5s - loss: 0.2276 - acc: 0.8958 - val_loss: 0.2444 - val_acc: 0.8938\n",
      "Epoch 49/150\n",
      "Epoch 00049: val_loss did not improve\n",
      " - 5s - loss: 0.2055 - acc: 0.9115 - val_loss: 0.2508 - val_acc: 0.9031\n",
      "Epoch 50/150\n",
      "Epoch 00050: val_loss improved from 0.23998 to 0.23912, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.1903 - acc: 0.9161 - val_loss: 0.2391 - val_acc: 0.9094\n",
      "Epoch 51/150\n",
      "Epoch 00051: val_loss improved from 0.23912 to 0.23004, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.1904 - acc: 0.9207 - val_loss: 0.2300 - val_acc: 0.8938\n",
      "Epoch 52/150\n",
      "Epoch 00052: val_loss did not improve\n",
      " - 5s - loss: 0.2074 - acc: 0.9070 - val_loss: 0.2734 - val_acc: 0.8906\n",
      "Epoch 53/150\n",
      "Epoch 00053: val_loss improved from 0.23004 to 0.22888, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage1.hdf5\n",
      " - 5s - loss: 0.1908 - acc: 0.9146 - val_loss: 0.2289 - val_acc: 0.9156\n",
      "Epoch 54/150\n",
      "Epoch 00054: val_loss did not improve\n",
      " - 5s - loss: 0.1845 - acc: 0.9147 - val_loss: 0.3344 - val_acc: 0.8719\n",
      "Epoch 55/150\n",
      "Epoch 00055: val_loss did not improve\n",
      " - 5s - loss: 0.2157 - acc: 0.9033 - val_loss: 0.2661 - val_acc: 0.8938\n",
      "Epoch 56/150\n",
      "Epoch 00056: val_loss did not improve\n",
      " - 5s - loss: 0.1961 - acc: 0.9131 - val_loss: 0.2295 - val_acc: 0.9031\n",
      "Epoch 57/150\n",
      "Epoch 00057: val_loss did not improve\n",
      " - 5s - loss: 0.1839 - acc: 0.9169 - val_loss: 0.2447 - val_acc: 0.9156\n",
      "Epoch 58/150\n",
      "Epoch 00058: val_loss did not improve\n",
      " - 5s - loss: 0.1753 - acc: 0.9222 - val_loss: 0.2798 - val_acc: 0.9000\n",
      "Epoch 59/150\n",
      "Epoch 00059: val_loss did not improve\n",
      " - 5s - loss: 0.2322 - acc: 0.9016 - val_loss: 0.2563 - val_acc: 0.8969\n",
      "Epoch 60/150\n",
      "Epoch 00060: val_loss did not improve\n",
      " - 5s - loss: 0.1813 - acc: 0.9222 - val_loss: 0.2558 - val_acc: 0.9125\n",
      "Epoch 61/150\n",
      "Epoch 00061: val_loss did not improve\n",
      " - 5s - loss: 0.1743 - acc: 0.9216 - val_loss: 0.2432 - val_acc: 0.9031\n",
      "Epoch 62/150\n",
      "Epoch 00062: val_loss did not improve\n",
      " - 5s - loss: 0.1899 - acc: 0.9133 - val_loss: 0.2511 - val_acc: 0.9000\n",
      "Epoch 63/150\n",
      "Epoch 00063: val_loss did not improve\n",
      " - 5s - loss: 0.1785 - acc: 0.9291 - val_loss: 0.2453 - val_acc: 0.9062\n",
      "Epoch 64/150\n",
      "Epoch 00064: val_loss did not improve\n",
      " - 5s - loss: 0.1730 - acc: 0.9268 - val_loss: 0.2479 - val_acc: 0.8906\n",
      "Epoch 65/150\n",
      "Epoch 00065: val_loss did not improve\n",
      " - 5s - loss: 0.1699 - acc: 0.9231 - val_loss: 0.2559 - val_acc: 0.9094\n",
      "Epoch 66/150\n",
      "Epoch 00066: val_loss did not improve\n",
      " - 5s - loss: 0.1641 - acc: 0.9300 - val_loss: 0.2406 - val_acc: 0.9125\n",
      "Epoch 67/150\n",
      "Epoch 00067: val_loss did not improve\n",
      " - 5s - loss: 0.1737 - acc: 0.9254 - val_loss: 0.2678 - val_acc: 0.9000\n",
      "Epoch 68/150\n",
      "Epoch 00068: val_loss did not improve\n",
      " - 5s - loss: 0.1978 - acc: 0.9115 - val_loss: 0.2382 - val_acc: 0.9031\n",
      "Epoch 69/150\n",
      "Epoch 00069: val_loss did not improve\n",
      " - 5s - loss: 0.1531 - acc: 0.9352 - val_loss: 0.2452 - val_acc: 0.9094\n",
      "Epoch 70/150\n",
      "Epoch 00070: val_loss did not improve\n",
      " - 5s - loss: 0.1539 - acc: 0.9321 - val_loss: 0.2734 - val_acc: 0.9125\n",
      "Epoch 71/150\n",
      "Epoch 00071: val_loss did not improve\n",
      " - 5s - loss: 0.1710 - acc: 0.9246 - val_loss: 0.2499 - val_acc: 0.9125\n",
      "Epoch 72/150\n",
      "Epoch 00072: val_loss did not improve\n",
      " - 5s - loss: 0.1624 - acc: 0.9254 - val_loss: 0.2320 - val_acc: 0.9187\n",
      "Epoch 73/150\n",
      "Epoch 00073: val_loss did not improve\n",
      " - 5s - loss: 0.1602 - acc: 0.9283 - val_loss: 0.2695 - val_acc: 0.9125\n",
      "Epoch 74/150\n",
      "Epoch 00074: val_loss did not improve\n",
      " - 5s - loss: 0.1529 - acc: 0.9337 - val_loss: 0.3728 - val_acc: 0.8562\n",
      "Epoch 75/150\n",
      "Epoch 00075: val_loss did not improve\n",
      " - 5s - loss: 0.1554 - acc: 0.9352 - val_loss: 0.2693 - val_acc: 0.9031\n",
      "Epoch 76/150\n",
      "Epoch 00076: val_loss did not improve\n",
      " - 5s - loss: 0.1574 - acc: 0.9268 - val_loss: 0.4722 - val_acc: 0.8625\n",
      "Epoch 77/150\n",
      "Epoch 00077: val_loss did not improve\n",
      " - 5s - loss: 0.1482 - acc: 0.9398 - val_loss: 0.2588 - val_acc: 0.9062\n",
      "Epoch 78/150\n",
      "Epoch 00078: val_loss did not improve\n",
      " - 5s - loss: 0.1381 - acc: 0.9466 - val_loss: 0.2513 - val_acc: 0.9156\n",
      "Epoch 79/150\n",
      "Epoch 00079: val_loss did not improve\n",
      " - 5s - loss: 0.1495 - acc: 0.9436 - val_loss: 0.2346 - val_acc: 0.9219\n",
      "Epoch 80/150\n",
      "Epoch 00080: val_loss did not improve\n",
      " - 5s - loss: 0.1427 - acc: 0.9375 - val_loss: 0.3053 - val_acc: 0.8688\n",
      "Epoch 81/150\n",
      "Epoch 00081: val_loss did not improve\n",
      " - 5s - loss: 0.1632 - acc: 0.9307 - val_loss: 0.2393 - val_acc: 0.9031\n",
      "Epoch 82/150\n",
      "Epoch 00082: val_loss did not improve\n",
      " - 5s - loss: 0.1567 - acc: 0.9361 - val_loss: 0.2391 - val_acc: 0.9125\n",
      "Epoch 83/150\n",
      "Epoch 00083: val_loss did not improve\n",
      " - 5s - loss: 0.1329 - acc: 0.9405 - val_loss: 0.2671 - val_acc: 0.9094\n",
      "CV5, Done!\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_25 (InputLayer)           (None, 75, 75, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_97 (Conv2D)              (None, 71, 71, 64)   4864        input_25[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "activation_97 (Activation)      (None, 71, 71, 64)   0           conv2d_97[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_97 (MaxPooling2D) (None, 35, 35, 64)   0           activation_97[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_145 (Dropout)           (None, 35, 35, 64)   0           max_pooling2d_97[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_98 (Conv2D)              (None, 33, 33, 128)  73856       dropout_145[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_98 (Activation)      (None, 33, 33, 128)  0           conv2d_98[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_98 (MaxPooling2D) (None, 16, 16, 128)  0           activation_98[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_146 (Dropout)           (None, 16, 16, 128)  0           max_pooling2d_98[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_99 (Conv2D)              (None, 14, 14, 128)  147584      dropout_146[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_99 (Activation)      (None, 14, 14, 128)  0           conv2d_99[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_99 (MaxPooling2D) (None, 7, 7, 128)    0           activation_99[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_147 (Dropout)           (None, 7, 7, 128)    0           max_pooling2d_99[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_100 (Conv2D)             (None, 5, 5, 64)     73792       dropout_147[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "activation_100 (Activation)     (None, 5, 5, 64)     0           conv2d_100[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_100 (MaxPooling2D (None, 2, 2, 64)     0           activation_100[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "other (InputLayer)              (None, 13)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_148 (Dropout)           (None, 2, 2, 64)     0           max_pooling2d_100[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "dense_121 (Dense)               (None, 16)           224         other[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_25 (Flatten)            (None, 256)          0           dropout_148[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_122 (Dense)               (None, 16)           272         dense_121[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_25 (Concatenate)    (None, 272)          0           flatten_25[0][0]                 \n",
      "                                                                 dense_122[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_123 (Dense)               (None, 256)          69888       concatenate_25[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_149 (Dropout)           (None, 256)          0           dense_123[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_124 (Dense)               (None, 256)          65792       dropout_149[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dropout_150 (Dropout)           (None, 256)          0           dense_124[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_125 (Dense)               (None, 1)            257         dropout_150[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 436,529\n",
      "Trainable params: 431,665\n",
      "Non-trainable params: 4,864\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/150\n",
      "Epoch 00001: val_loss improved from inf to 0.25216, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage2.hdf5\n",
      " - 10s - loss: 0.3359 - acc: 0.8545 - val_loss: 0.2522 - val_acc: 0.9031\n",
      "Epoch 2/150\n",
      "Epoch 00002: val_loss improved from 0.25216 to 0.24290, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage2.hdf5\n",
      " - 4s - loss: 0.3180 - acc: 0.8665 - val_loss: 0.2429 - val_acc: 0.9156\n",
      "Epoch 3/150\n",
      "Epoch 00003: val_loss did not improve\n",
      " - 4s - loss: 0.2871 - acc: 0.8673 - val_loss: 0.2568 - val_acc: 0.9062\n",
      "Epoch 4/150\n",
      "Epoch 00004: val_loss improved from 0.24290 to 0.22334, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage2.hdf5\n",
      " - 4s - loss: 0.2641 - acc: 0.8818 - val_loss: 0.2233 - val_acc: 0.9156\n",
      "Epoch 5/150\n",
      "Epoch 00005: val_loss did not improve\n",
      " - 4s - loss: 0.2652 - acc: 0.8864 - val_loss: 0.2270 - val_acc: 0.9156\n",
      "Epoch 6/150\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 4s - loss: 0.2619 - acc: 0.8925 - val_loss: 0.2269 - val_acc: 0.9125\n",
      "Epoch 7/150\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 4s - loss: 0.2975 - acc: 0.8760 - val_loss: 0.2368 - val_acc: 0.9187\n",
      "Epoch 8/150\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 4s - loss: 0.2702 - acc: 0.8871 - val_loss: 0.2383 - val_acc: 0.9125\n",
      "Epoch 9/150\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 4s - loss: 0.2733 - acc: 0.8798 - val_loss: 0.2309 - val_acc: 0.9187\n",
      "Epoch 10/150\n",
      "Epoch 00010: val_loss improved from 0.22334 to 0.22190, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage2.hdf5\n",
      " - 4s - loss: 0.2484 - acc: 0.8926 - val_loss: 0.2219 - val_acc: 0.9156\n",
      "Epoch 11/150\n",
      "Epoch 00011: val_loss did not improve\n",
      " - 4s - loss: 0.2648 - acc: 0.8909 - val_loss: 0.2372 - val_acc: 0.9219\n",
      "Epoch 12/150\n",
      "Epoch 00012: val_loss did not improve\n",
      " - 4s - loss: 0.2574 - acc: 0.8896 - val_loss: 0.2227 - val_acc: 0.9313\n",
      "Epoch 13/150\n",
      "Epoch 00013: val_loss improved from 0.22190 to 0.21533, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage2.hdf5\n",
      " - 4s - loss: 0.2300 - acc: 0.9009 - val_loss: 0.2153 - val_acc: 0.9313\n",
      "Epoch 14/150\n",
      "Epoch 00014: val_loss did not improve\n",
      " - 4s - loss: 0.2345 - acc: 0.9009 - val_loss: 0.2703 - val_acc: 0.8875\n",
      "Epoch 15/150\n",
      "Epoch 00015: val_loss did not improve\n",
      " - 4s - loss: 0.2313 - acc: 0.8909 - val_loss: 0.2318 - val_acc: 0.9219\n",
      "Epoch 16/150\n",
      "Epoch 00016: val_loss improved from 0.21533 to 0.21414, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage2.hdf5\n",
      " - 4s - loss: 0.2434 - acc: 0.9018 - val_loss: 0.2141 - val_acc: 0.9250\n",
      "Epoch 17/150\n",
      "Epoch 00017: val_loss did not improve\n",
      " - 4s - loss: 0.2331 - acc: 0.9002 - val_loss: 0.2790 - val_acc: 0.8844\n",
      "Epoch 18/150\n",
      "Epoch 00018: val_loss did not improve\n",
      " - 4s - loss: 0.2365 - acc: 0.8950 - val_loss: 0.2341 - val_acc: 0.9219\n",
      "Epoch 19/150\n",
      "Epoch 00019: val_loss did not improve\n",
      " - 4s - loss: 0.2490 - acc: 0.8918 - val_loss: 0.2294 - val_acc: 0.9125\n",
      "Epoch 20/150\n",
      "Epoch 00020: val_loss did not improve\n",
      " - 4s - loss: 0.2388 - acc: 0.9039 - val_loss: 0.2155 - val_acc: 0.9219\n",
      "Epoch 21/150\n",
      "Epoch 00021: val_loss did not improve\n",
      " - 4s - loss: 0.2227 - acc: 0.9123 - val_loss: 0.2397 - val_acc: 0.9094\n",
      "Epoch 22/150\n",
      "Epoch 00022: val_loss did not improve\n",
      " - 4s - loss: 0.2232 - acc: 0.9094 - val_loss: 0.2179 - val_acc: 0.9250\n",
      "Epoch 23/150\n",
      "Epoch 00023: val_loss improved from 0.21414 to 0.21122, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage2.hdf5\n",
      " - 4s - loss: 0.2179 - acc: 0.9062 - val_loss: 0.2112 - val_acc: 0.9344\n",
      "Epoch 24/150\n",
      "Epoch 00024: val_loss did not improve\n",
      " - 4s - loss: 0.2207 - acc: 0.9024 - val_loss: 0.2124 - val_acc: 0.9281\n",
      "Epoch 25/150\n",
      "Epoch 00025: val_loss did not improve\n",
      " - 4s - loss: 0.1986 - acc: 0.9161 - val_loss: 0.2665 - val_acc: 0.9031\n",
      "Epoch 26/150\n",
      "Epoch 00026: val_loss did not improve\n",
      " - 4s - loss: 0.2330 - acc: 0.8949 - val_loss: 0.2192 - val_acc: 0.9250\n",
      "Epoch 27/150\n",
      "Epoch 00027: val_loss did not improve\n",
      " - 4s - loss: 0.2279 - acc: 0.8981 - val_loss: 0.2361 - val_acc: 0.9187\n",
      "Epoch 28/150\n",
      "Epoch 00028: val_loss did not improve\n",
      " - 4s - loss: 0.2190 - acc: 0.9108 - val_loss: 0.2578 - val_acc: 0.9062\n",
      "Epoch 29/150\n",
      "Epoch 00029: val_loss did not improve\n",
      " - 4s - loss: 0.2094 - acc: 0.9161 - val_loss: 0.2267 - val_acc: 0.9281\n",
      "Epoch 30/150\n",
      "Epoch 00030: val_loss improved from 0.21122 to 0.20892, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage2.hdf5\n",
      " - 4s - loss: 0.2142 - acc: 0.9071 - val_loss: 0.2089 - val_acc: 0.9313\n",
      "Epoch 31/150\n",
      "Epoch 00031: val_loss improved from 0.20892 to 0.20117, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage2.hdf5\n",
      " - 4s - loss: 0.2139 - acc: 0.9063 - val_loss: 0.2012 - val_acc: 0.9313\n",
      "Epoch 32/150\n",
      "Epoch 00032: val_loss did not improve\n",
      " - 4s - loss: 0.2044 - acc: 0.9199 - val_loss: 0.2445 - val_acc: 0.9094\n",
      "Epoch 33/150\n",
      "Epoch 00033: val_loss did not improve\n",
      " - 4s - loss: 0.2025 - acc: 0.9184 - val_loss: 0.2243 - val_acc: 0.9219\n",
      "Epoch 34/150\n",
      "Epoch 00034: val_loss improved from 0.20117 to 0.19031, saving model to model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv5_stage2.hdf5\n",
      " - 4s - loss: 0.2066 - acc: 0.9192 - val_loss: 0.1903 - val_acc: 0.9406\n",
      "Epoch 35/150\n",
      "Epoch 00035: val_loss did not improve\n",
      " - 4s - loss: 0.2131 - acc: 0.9056 - val_loss: 0.2086 - val_acc: 0.9281\n",
      "Epoch 36/150\n",
      "Epoch 00036: val_loss did not improve\n",
      " - 4s - loss: 0.2300 - acc: 0.9063 - val_loss: 0.1973 - val_acc: 0.9281\n",
      "Epoch 37/150\n",
      "Epoch 00037: val_loss did not improve\n",
      " - 4s - loss: 0.2366 - acc: 0.8972 - val_loss: 0.2383 - val_acc: 0.9094\n",
      "Epoch 38/150\n",
      "Epoch 00038: val_loss did not improve\n",
      " - 4s - loss: 0.2124 - acc: 0.8972 - val_loss: 0.2359 - val_acc: 0.9219\n",
      "Epoch 39/150\n",
      "Epoch 00039: val_loss did not improve\n",
      " - 4s - loss: 0.1837 - acc: 0.9306 - val_loss: 0.2787 - val_acc: 0.9062\n",
      "Epoch 40/150\n",
      "Epoch 00040: val_loss did not improve\n",
      " - 4s - loss: 0.2086 - acc: 0.9169 - val_loss: 0.2260 - val_acc: 0.9156\n",
      "Epoch 41/150\n",
      "Epoch 00041: val_loss did not improve\n",
      " - 4s - loss: 0.1947 - acc: 0.9260 - val_loss: 0.2149 - val_acc: 0.9313\n",
      "Epoch 42/150\n",
      "Epoch 00042: val_loss did not improve\n",
      " - 4s - loss: 0.1962 - acc: 0.9161 - val_loss: 0.2277 - val_acc: 0.9156\n",
      "Epoch 43/150\n",
      "Epoch 00043: val_loss did not improve\n",
      " - 4s - loss: 0.1939 - acc: 0.9176 - val_loss: 0.2473 - val_acc: 0.9156\n",
      "Epoch 44/150\n",
      "Epoch 00044: val_loss did not improve\n",
      " - 4s - loss: 0.2075 - acc: 0.9155 - val_loss: 0.2496 - val_acc: 0.9094\n",
      "Epoch 45/150\n",
      "Epoch 00045: val_loss did not improve\n",
      " - 4s - loss: 0.1957 - acc: 0.9222 - val_loss: 0.2352 - val_acc: 0.9125\n",
      "Epoch 46/150\n",
      "Epoch 00046: val_loss did not improve\n",
      " - 4s - loss: 0.1987 - acc: 0.9131 - val_loss: 0.1950 - val_acc: 0.9313\n",
      "Epoch 47/150\n",
      "Epoch 00047: val_loss did not improve\n",
      " - 4s - loss: 0.1953 - acc: 0.9237 - val_loss: 0.2555 - val_acc: 0.9000\n",
      "Epoch 48/150\n",
      "Epoch 00048: val_loss did not improve\n",
      " - 4s - loss: 0.2024 - acc: 0.9126 - val_loss: 0.2216 - val_acc: 0.9219\n",
      "Epoch 49/150\n",
      "Epoch 00049: val_loss did not improve\n",
      " - 4s - loss: 0.1987 - acc: 0.9207 - val_loss: 0.2254 - val_acc: 0.9219\n",
      "Epoch 50/150\n",
      "Epoch 00050: val_loss did not improve\n",
      " - 4s - loss: 0.1919 - acc: 0.9207 - val_loss: 0.2075 - val_acc: 0.9250\n",
      "Epoch 51/150\n",
      "Epoch 00051: val_loss did not improve\n",
      " - 4s - loss: 0.1905 - acc: 0.9223 - val_loss: 0.2244 - val_acc: 0.9281\n",
      "Epoch 52/150\n",
      "Epoch 00052: val_loss did not improve\n",
      " - 4s - loss: 0.1914 - acc: 0.9184 - val_loss: 0.2126 - val_acc: 0.9344\n",
      "Epoch 53/150\n",
      "Epoch 00053: val_loss did not improve\n",
      " - 4s - loss: 0.1901 - acc: 0.9161 - val_loss: 0.1976 - val_acc: 0.9344\n",
      "Epoch 54/150\n",
      "Epoch 00054: val_loss did not improve\n",
      " - 4s - loss: 0.1796 - acc: 0.9268 - val_loss: 0.2220 - val_acc: 0.9313\n",
      "Epoch 55/150\n",
      "Epoch 00055: val_loss did not improve\n",
      " - 4s - loss: 0.1891 - acc: 0.9298 - val_loss: 0.1957 - val_acc: 0.9313\n",
      "Epoch 56/150\n",
      "Epoch 00056: val_loss did not improve\n",
      " - 4s - loss: 0.1822 - acc: 0.9207 - val_loss: 0.2184 - val_acc: 0.9094\n",
      "Epoch 57/150\n",
      "Epoch 00057: val_loss did not improve\n",
      " - 4s - loss: 0.2020 - acc: 0.9169 - val_loss: 0.2024 - val_acc: 0.9344\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 58/150\n",
      "Epoch 00058: val_loss did not improve\n",
      " - 4s - loss: 0.1652 - acc: 0.9352 - val_loss: 0.2025 - val_acc: 0.9313\n",
      "Epoch 59/150\n",
      "Epoch 00059: val_loss did not improve\n",
      " - 4s - loss: 0.1878 - acc: 0.9215 - val_loss: 0.2115 - val_acc: 0.9281\n",
      "Epoch 60/150\n",
      "Epoch 00060: val_loss did not improve\n",
      " - 4s - loss: 0.1772 - acc: 0.9199 - val_loss: 0.2300 - val_acc: 0.9219\n",
      "Epoch 61/150\n",
      "Epoch 00061: val_loss did not improve\n",
      " - 4s - loss: 0.1826 - acc: 0.9216 - val_loss: 0.2142 - val_acc: 0.9281\n",
      "Epoch 62/150\n",
      "Epoch 00062: val_loss did not improve\n",
      " - 4s - loss: 0.1939 - acc: 0.9269 - val_loss: 0.2163 - val_acc: 0.9250\n",
      "Epoch 63/150\n",
      "Epoch 00063: val_loss did not improve\n",
      " - 4s - loss: 0.1773 - acc: 0.9283 - val_loss: 0.2286 - val_acc: 0.9187\n",
      "Epoch 64/150\n",
      "Epoch 00064: val_loss did not improve\n",
      " - 4s - loss: 0.1686 - acc: 0.9321 - val_loss: 0.2162 - val_acc: 0.9219\n",
      "CV5, Done!\n",
      "1284/1284 [==============================] - 2s 1ms/step\n",
      "Train loss: 0.153598635135\n",
      "Train accuracy: 0.939252336449\n",
      "320/320 [==============================] - 0s 1ms/step\n",
      "Valid loss: 0.190312217921\n",
      "Valid accuracy: 0.940625\n",
      " Test Log Loss Validation=  0.203232492341\n"
     ]
    }
   ],
   "source": [
    "whole_valid = Model11_CV(K = 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Evaluation_CV(X_train,K=3):\n",
    "    folds = list(StratifiedKFold(n_splits=K, shuffle=True,random_state = 3).split(X_train, target_train))\n",
    "    y_test_pred_log=0\n",
    "    y_train_pred_log=0\n",
    "    y_valid_pred_log = 0.0*target_train\n",
    "    for j, (train_idx, valid_idx) in enumerate(folds):\n",
    "        print('\\n===================FOLD=',j+1)\n",
    "        X_train_cv = X_train[train_idx]\n",
    "        y_train_cv = target_train[train_idx]\n",
    "        X_valid = X_train[valid_idx]\n",
    "        y_valid= target_train[valid_idx]\n",
    "        \n",
    "        #Angle\n",
    "#         X_angle_cv=X_angle[train_idx]\n",
    "#         X_angle_hold=X_angle[test_idx]\n",
    "\n",
    "        #define file path and get callbacks\n",
    "        file_path = 'model save/Model 6-Advanced CNN with DA and self TL(cv3)/Model-6 cv'+str(j+1)+'.hdf5'\n",
    "        Cnnmodel=cnnmodel(input_shape = (75,75,3),lr = 0.0001)\n",
    "        Cnnmodel.load_weights(file_path)\n",
    "\n",
    "\n",
    "#         callbacks = get_callbacks(filepath=file_path, patience=5)\n",
    "#         gen_flow = gen_flow_for_two_inputs(X_train_cv, X_angle_cv, y_train_cv)\n",
    "#         galaxyModel= getVggAngleModel()\n",
    "#         galaxyModel.fit_generator(\n",
    "#                 gen_flow,\n",
    "#                 steps_per_epoch=24,\n",
    "#                 epochs=100,\n",
    "#                 shuffle=True,\n",
    "#                 verbose=1,\n",
    "#                 validation_data=([X_holdout,X_angle_hold], Y_holdout),\n",
    "#                 callbacks=callbacks)\n",
    "\n",
    "        #Getting the Best Model\n",
    "        #Getting Training Score\n",
    "        print('cv'+str(j+1))\n",
    "        score = Cnnmodel.evaluate(X_train_cv, y_train_cv)\n",
    "        print('Train loss:', score[0])\n",
    "        print('Train accuracy:', score[1])\n",
    "        #Getting Test Score\n",
    "        score = Cnnmodel.evaluate(X_valid, y_valid)\n",
    "        print('Valid loss:', score[0])\n",
    "        print('Valid accuracy:', score[1])\n",
    "        print('\\n')\n",
    "\n",
    "        #Getting validation Score.\n",
    "        pred_valid=Cnnmodel.predict(X_valid)\n",
    "        y_valid_pred_log[valid_idx] = pred_valid.reshape(pred_valid.shape[0])\n",
    "\n",
    "        #Getting Test Scores\n",
    "        temp_test=Cnnmodel.predict(X_test)\n",
    "        y_test_pred_log+=temp_test.reshape(temp_test.shape[0])\n",
    "\n",
    "        #Getting Train Scores\n",
    "        temp_train=Cnnmodel.predict(X_train)\n",
    "        y_train_pred_log+=temp_train.reshape(temp_train.shape[0])\n",
    "\n",
    "    y_valid_pred_log=y_valid_pred_log/K\n",
    "    y_train_pred_log=y_train_pred_log/K\n",
    "\n",
    "    print('\\n Train Log Loss Validation= ',log_loss(target_train, y_train_pred_log))\n",
    "    print(' Test Log Loss Validation= ',log_loss(target_train, y_valid_pred_log))\n",
    "    return y_test_pred_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "preds = Evaluation_CV(X_train,K=5)\n",
    "sub = test[['id']]\n",
    "sub['is_iceberg'] = preds\n",
    "sub.to_csv('5cv.csv',index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submission(min max median stacking)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test_scores(K=5):\n",
    "    y_test_pred_log=0\n",
    "    test_df = test_data[['id']]\n",
    "    for j in range(K):\n",
    "        print('\\n===================FOLD=',j+1)\n",
    "        file_path = 'model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv'+str(j+1)+'.hdf5'\n",
    "        Cnnmodel = model_compile(cnnmodel(),lr = 0.0001,decay = 0, freezing_layers = None,weights_path = file_path)\n",
    "\n",
    "        #Getting Test Scores\n",
    "        temp_test=Cnnmodel.predict(X_test)\n",
    "        test_df['cv'+str(j+1)]=temp_test.reshape(temp_test.shape[0])\n",
    "    return test_df\n",
    "\n",
    "def test_scores_scaled_others(X_test_others,K = 5):\n",
    "    folds = list(StratifiedKFold(n_splits=K, shuffle=True,random_state = 3).split(X, target))\n",
    "    test_df = test_data[['id']]\n",
    "    for j, (train_idx, valid_idx) in enumerate(folds):\n",
    "        print('\\n===================FOLD=',j+1)\n",
    "        \n",
    "        global X_train_others\n",
    "        X_train_others = np.array(X_others)[train_idx]\n",
    "\n",
    "        \n",
    "        scaler = MinMaxScaler().fit(X_train_others)\n",
    "        X_test_others = scaler.transform(X_test_others)\n",
    "\n",
    "        file_path = 'model save/Model 11-Advanced CNN with DA, self TL and max layer(cv5)/Model-11 cv'+str(j+1)+'_stage2.hdf5'\n",
    "        Cnnmodel = model_compile(cnnmodel_with_others(),lr = 0.0001,decay = 0, freezing_layers = 5,weights_path = file_path)\n",
    "        temp_test=Cnnmodel.predict([X_test,X_test_others])\n",
    "        test_df['cv'+str(j+1)]=temp_test.reshape(temp_test.shape[0])\n",
    "    return test_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================FOLD= 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\sklearn\\utils\\validation.py:475: DataConversionWarning: Data with input dtype object was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 75, 75, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 71, 71, 64)   4864        input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 71, 71, 64)   0           conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)  (None, 35, 35, 64)   0           activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 35, 35, 64)   0           max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 33, 33, 128)  73856       dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 33, 33, 128)  0           conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2D)  (None, 16, 16, 128)  0           activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 16, 16, 128)  0           max_pooling2d_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 14, 14, 128)  147584      dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 14, 14, 128)  0           conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2D)  (None, 7, 7, 128)    0           activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 7, 7, 128)    0           max_pooling2d_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 5, 5, 64)     73792       dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 5, 5, 64)     0           conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2D)  (None, 2, 2, 64)     0           activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "other (InputLayer)              (None, 13)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 2, 2, 64)     0           max_pooling2d_4[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 16)           224         other[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 256)          0           dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 16)           272         dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 272)          0           flatten_1[0][0]                  \n",
      "                                                                 dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 256)          69888       concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 256)          0           dense_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 256)          65792       dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)             (None, 256)          0           dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 1)            257         dropout_6[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 436,529\n",
      "Trainable params: 431,665\n",
      "Non-trainable params: 4,864\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\ipykernel_launcher.py:30: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================FOLD= 2\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            (None, 75, 75, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_5 (Conv2D)               (None, 71, 71, 64)   4864        input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 71, 71, 64)   0           conv2d_5[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_5 (MaxPooling2D)  (None, 35, 35, 64)   0           activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dropout_7 (Dropout)             (None, 35, 35, 64)   0           max_pooling2d_5[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 33, 33, 128)  73856       dropout_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 33, 33, 128)  0           conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_6 (MaxPooling2D)  (None, 16, 16, 128)  0           activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dropout_8 (Dropout)             (None, 16, 16, 128)  0           max_pooling2d_6[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_7 (Conv2D)               (None, 14, 14, 128)  147584      dropout_8[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 14, 14, 128)  0           conv2d_7[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_7 (MaxPooling2D)  (None, 7, 7, 128)    0           activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dropout_9 (Dropout)             (None, 7, 7, 128)    0           max_pooling2d_7[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_8 (Conv2D)               (None, 5, 5, 64)     73792       dropout_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 5, 5, 64)     0           conv2d_8[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_8 (MaxPooling2D)  (None, 2, 2, 64)     0           activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "other (InputLayer)              (None, 13)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_10 (Dropout)            (None, 2, 2, 64)     0           max_pooling2d_8[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 16)           224         other[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_2 (Flatten)             (None, 256)          0           dropout_10[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 16)           272         dense_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 272)          0           flatten_2[0][0]                  \n",
      "                                                                 dense_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 256)          69888       concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_11 (Dropout)            (None, 256)          0           dense_8[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_9 (Dense)                 (None, 256)          65792       dropout_11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_12 (Dropout)            (None, 256)          0           dense_9[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_10 (Dense)                (None, 1)            257         dropout_12[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 436,529\n",
      "Trainable params: 431,665\n",
      "Non-trainable params: 4,864\n",
      "__________________________________________________________________________________________________\n",
      "\n",
      "===================FOLD= 3\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_3 (InputLayer)            (None, 75, 75, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_9 (Conv2D)               (None, 71, 71, 64)   4864        input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 71, 71, 64)   0           conv2d_9[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_9 (MaxPooling2D)  (None, 35, 35, 64)   0           activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dropout_13 (Dropout)            (None, 35, 35, 64)   0           max_pooling2d_9[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_10 (Conv2D)              (None, 33, 33, 128)  73856       dropout_13[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_10 (Activation)      (None, 33, 33, 128)  0           conv2d_10[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_10 (MaxPooling2D) (None, 16, 16, 128)  0           activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_14 (Dropout)            (None, 16, 16, 128)  0           max_pooling2d_10[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_11 (Conv2D)              (None, 14, 14, 128)  147584      dropout_14[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_11 (Activation)      (None, 14, 14, 128)  0           conv2d_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_11 (MaxPooling2D) (None, 7, 7, 128)    0           activation_11[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_15 (Dropout)            (None, 7, 7, 128)    0           max_pooling2d_11[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_12 (Conv2D)              (None, 5, 5, 64)     73792       dropout_15[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_12 (Activation)      (None, 5, 5, 64)     0           conv2d_12[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_12 (MaxPooling2D) (None, 2, 2, 64)     0           activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "other (InputLayer)              (None, 13)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_16 (Dropout)            (None, 2, 2, 64)     0           max_pooling2d_12[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_11 (Dense)                (None, 16)           224         other[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_3 (Flatten)             (None, 256)          0           dropout_16[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_12 (Dense)                (None, 16)           272         dense_11[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 272)          0           flatten_3[0][0]                  \n",
      "                                                                 dense_12[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_13 (Dense)                (None, 256)          69888       concatenate_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_17 (Dropout)            (None, 256)          0           dense_13[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_14 (Dense)                (None, 256)          65792       dropout_17[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_18 (Dropout)            (None, 256)          0           dense_14[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_15 (Dense)                (None, 1)            257         dropout_18[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 436,529\n",
      "Trainable params: 431,665\n",
      "Non-trainable params: 4,864\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================FOLD= 4\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_4 (InputLayer)            (None, 75, 75, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_13 (Conv2D)              (None, 71, 71, 64)   4864        input_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_13 (Activation)      (None, 71, 71, 64)   0           conv2d_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_13 (MaxPooling2D) (None, 35, 35, 64)   0           activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_19 (Dropout)            (None, 35, 35, 64)   0           max_pooling2d_13[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_14 (Conv2D)              (None, 33, 33, 128)  73856       dropout_19[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_14 (Activation)      (None, 33, 33, 128)  0           conv2d_14[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_14 (MaxPooling2D) (None, 16, 16, 128)  0           activation_14[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_20 (Dropout)            (None, 16, 16, 128)  0           max_pooling2d_14[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_15 (Conv2D)              (None, 14, 14, 128)  147584      dropout_20[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_15 (Activation)      (None, 14, 14, 128)  0           conv2d_15[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_15 (MaxPooling2D) (None, 7, 7, 128)    0           activation_15[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_21 (Dropout)            (None, 7, 7, 128)    0           max_pooling2d_15[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_16 (Conv2D)              (None, 5, 5, 64)     73792       dropout_21[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_16 (Activation)      (None, 5, 5, 64)     0           conv2d_16[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_16 (MaxPooling2D) (None, 2, 2, 64)     0           activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "other (InputLayer)              (None, 13)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_22 (Dropout)            (None, 2, 2, 64)     0           max_pooling2d_16[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_16 (Dense)                (None, 16)           224         other[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_4 (Flatten)             (None, 256)          0           dropout_22[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_17 (Dense)                (None, 16)           272         dense_16[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_4 (Concatenate)     (None, 272)          0           flatten_4[0][0]                  \n",
      "                                                                 dense_17[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_18 (Dense)                (None, 256)          69888       concatenate_4[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_23 (Dropout)            (None, 256)          0           dense_18[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_19 (Dense)                (None, 256)          65792       dropout_23[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_24 (Dropout)            (None, 256)          0           dense_19[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_20 (Dense)                (None, 1)            257         dropout_24[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 436,529\n",
      "Trainable params: 431,665\n",
      "Non-trainable params: 4,864\n",
      "__________________________________________________________________________________________________\n",
      "\n",
      "===================FOLD= 5\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_5 (InputLayer)            (None, 75, 75, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_17 (Conv2D)              (None, 71, 71, 64)   4864        input_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "activation_17 (Activation)      (None, 71, 71, 64)   0           conv2d_17[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_17 (MaxPooling2D) (None, 35, 35, 64)   0           activation_17[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_25 (Dropout)            (None, 35, 35, 64)   0           max_pooling2d_17[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_18 (Conv2D)              (None, 33, 33, 128)  73856       dropout_25[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_18 (Activation)      (None, 33, 33, 128)  0           conv2d_18[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_18 (MaxPooling2D) (None, 16, 16, 128)  0           activation_18[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_26 (Dropout)            (None, 16, 16, 128)  0           max_pooling2d_18[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_19 (Conv2D)              (None, 14, 14, 128)  147584      dropout_26[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_19 (Activation)      (None, 14, 14, 128)  0           conv2d_19[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_19 (MaxPooling2D) (None, 7, 7, 128)    0           activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_27 (Dropout)            (None, 7, 7, 128)    0           max_pooling2d_19[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_20 (Conv2D)              (None, 5, 5, 64)     73792       dropout_27[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "activation_20 (Activation)      (None, 5, 5, 64)     0           conv2d_20[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_20 (MaxPooling2D) (None, 2, 2, 64)     0           activation_20[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "other (InputLayer)              (None, 13)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_28 (Dropout)            (None, 2, 2, 64)     0           max_pooling2d_20[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_21 (Dense)                (None, 16)           224         other[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "flatten_5 (Flatten)             (None, 256)          0           dropout_28[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_22 (Dense)                (None, 16)           272         dense_21[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_5 (Concatenate)     (None, 272)          0           flatten_5[0][0]                  \n",
      "                                                                 dense_22[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_23 (Dense)                (None, 256)          69888       concatenate_5[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_29 (Dropout)            (None, 256)          0           dense_23[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_24 (Dense)                (None, 256)          65792       dropout_29[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_30 (Dropout)            (None, 256)          0           dense_24[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_25 (Dense)                (None, 1)            257         dropout_30[0][0]                 \n",
      "==================================================================================================\n",
      "Total params: 436,529\n",
      "Trainable params: 431,665\n",
      "Non-trainable params: 4,864\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "test_df = test_scores_scaled_others(X_test_others = X_test_others)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>cv1</th>\n",
       "      <th>cv2</th>\n",
       "      <th>cv3</th>\n",
       "      <th>cv4</th>\n",
       "      <th>cv5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5941774d</td>\n",
       "      <td>5.667688e-02</td>\n",
       "      <td>5.530631e-02</td>\n",
       "      <td>8.635153e-03</td>\n",
       "      <td>1.127893e-03</td>\n",
       "      <td>4.170072e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4023181e</td>\n",
       "      <td>5.026714e-01</td>\n",
       "      <td>8.580565e-01</td>\n",
       "      <td>3.927149e-01</td>\n",
       "      <td>3.979664e-01</td>\n",
       "      <td>2.290074e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b20200e4</td>\n",
       "      <td>2.789924e-06</td>\n",
       "      <td>3.951226e-10</td>\n",
       "      <td>1.032073e-10</td>\n",
       "      <td>5.433600e-08</td>\n",
       "      <td>8.329140e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e7f018bb</td>\n",
       "      <td>9.994588e-01</td>\n",
       "      <td>9.999734e-01</td>\n",
       "      <td>9.998581e-01</td>\n",
       "      <td>9.999951e-01</td>\n",
       "      <td>9.999849e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4371c8c3</td>\n",
       "      <td>1.669144e-02</td>\n",
       "      <td>6.438503e-02</td>\n",
       "      <td>4.461252e-02</td>\n",
       "      <td>3.018984e-03</td>\n",
       "      <td>1.088171e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>a8d9b1fd</td>\n",
       "      <td>1.406482e-01</td>\n",
       "      <td>1.800535e-02</td>\n",
       "      <td>2.911703e-03</td>\n",
       "      <td>8.031805e-03</td>\n",
       "      <td>2.462742e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>29e7727e</td>\n",
       "      <td>1.855465e-01</td>\n",
       "      <td>1.123179e-01</td>\n",
       "      <td>7.316975e-02</td>\n",
       "      <td>5.049995e-03</td>\n",
       "      <td>2.020663e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>92a51ffb</td>\n",
       "      <td>9.979691e-01</td>\n",
       "      <td>9.999943e-01</td>\n",
       "      <td>9.998738e-01</td>\n",
       "      <td>9.999973e-01</td>\n",
       "      <td>9.999726e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>c769ac97</td>\n",
       "      <td>8.177361e-05</td>\n",
       "      <td>5.616964e-05</td>\n",
       "      <td>2.393075e-06</td>\n",
       "      <td>2.250952e-06</td>\n",
       "      <td>6.520145e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>aee0547d</td>\n",
       "      <td>4.983280e-07</td>\n",
       "      <td>2.763404e-07</td>\n",
       "      <td>2.184263e-08</td>\n",
       "      <td>1.156752e-09</td>\n",
       "      <td>2.064336e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>565b28ac</td>\n",
       "      <td>1.511666e-05</td>\n",
       "      <td>8.481084e-06</td>\n",
       "      <td>1.915825e-06</td>\n",
       "      <td>1.947710e-08</td>\n",
       "      <td>5.455509e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>e04e9775</td>\n",
       "      <td>2.145108e-01</td>\n",
       "      <td>6.484185e-01</td>\n",
       "      <td>3.081630e-01</td>\n",
       "      <td>9.101298e-02</td>\n",
       "      <td>2.580765e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>8e8161d1</td>\n",
       "      <td>2.706455e-03</td>\n",
       "      <td>7.825905e-03</td>\n",
       "      <td>6.196049e-03</td>\n",
       "      <td>2.258463e-05</td>\n",
       "      <td>1.751517e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>4cf4d256</td>\n",
       "      <td>3.776796e-01</td>\n",
       "      <td>8.680509e-01</td>\n",
       "      <td>3.707848e-01</td>\n",
       "      <td>3.931115e-01</td>\n",
       "      <td>2.610141e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>139e5324</td>\n",
       "      <td>1.657643e-02</td>\n",
       "      <td>2.716942e-04</td>\n",
       "      <td>3.593864e-05</td>\n",
       "      <td>4.265339e-03</td>\n",
       "      <td>8.636447e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>f156976f</td>\n",
       "      <td>1.608710e-03</td>\n",
       "      <td>7.008111e-03</td>\n",
       "      <td>1.274463e-03</td>\n",
       "      <td>5.250456e-04</td>\n",
       "      <td>8.246358e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>68a117cc</td>\n",
       "      <td>8.594736e-02</td>\n",
       "      <td>2.305674e-03</td>\n",
       "      <td>1.826357e-03</td>\n",
       "      <td>9.703810e-03</td>\n",
       "      <td>4.464464e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>d9aa7a56</td>\n",
       "      <td>2.259670e-01</td>\n",
       "      <td>2.138297e-02</td>\n",
       "      <td>4.971057e-03</td>\n",
       "      <td>1.381533e-02</td>\n",
       "      <td>6.850597e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>9005b143</td>\n",
       "      <td>5.395933e-01</td>\n",
       "      <td>2.903121e-01</td>\n",
       "      <td>2.651281e-02</td>\n",
       "      <td>1.873317e-02</td>\n",
       "      <td>4.544104e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>5f6d3988</td>\n",
       "      <td>9.959655e-01</td>\n",
       "      <td>9.992558e-01</td>\n",
       "      <td>9.740520e-01</td>\n",
       "      <td>9.994076e-01</td>\n",
       "      <td>9.604470e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>9ad70954</td>\n",
       "      <td>1.401283e-02</td>\n",
       "      <td>7.709266e-03</td>\n",
       "      <td>2.763549e-03</td>\n",
       "      <td>1.011760e-04</td>\n",
       "      <td>7.388195e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>b9087b9e</td>\n",
       "      <td>2.441433e-01</td>\n",
       "      <td>4.942774e-01</td>\n",
       "      <td>5.023513e-01</td>\n",
       "      <td>2.631663e-01</td>\n",
       "      <td>1.896616e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>a39a1427</td>\n",
       "      <td>2.748383e-01</td>\n",
       "      <td>6.085083e-01</td>\n",
       "      <td>7.936671e-02</td>\n",
       "      <td>1.877789e-01</td>\n",
       "      <td>2.060031e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>82fbe8ed</td>\n",
       "      <td>1.093100e-01</td>\n",
       "      <td>1.750146e-01</td>\n",
       "      <td>4.207040e-02</td>\n",
       "      <td>2.376729e-03</td>\n",
       "      <td>3.546486e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1fae4879</td>\n",
       "      <td>3.575499e-08</td>\n",
       "      <td>1.084889e-08</td>\n",
       "      <td>1.098709e-06</td>\n",
       "      <td>2.299321e-09</td>\n",
       "      <td>1.099892e-13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>6dd8f13d</td>\n",
       "      <td>3.685710e-05</td>\n",
       "      <td>3.017996e-08</td>\n",
       "      <td>1.945800e-09</td>\n",
       "      <td>2.980999e-07</td>\n",
       "      <td>2.638232e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>bbad5958</td>\n",
       "      <td>5.359598e-02</td>\n",
       "      <td>8.420833e-02</td>\n",
       "      <td>5.728618e-02</td>\n",
       "      <td>2.809570e-03</td>\n",
       "      <td>1.495250e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>54527583</td>\n",
       "      <td>3.359738e-02</td>\n",
       "      <td>3.805183e-02</td>\n",
       "      <td>2.905826e-03</td>\n",
       "      <td>8.412496e-04</td>\n",
       "      <td>1.929508e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>be8fa29c</td>\n",
       "      <td>4.603783e-02</td>\n",
       "      <td>5.425699e-02</td>\n",
       "      <td>2.093825e-02</td>\n",
       "      <td>5.805754e-04</td>\n",
       "      <td>3.134798e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>81a3328f</td>\n",
       "      <td>9.999920e-01</td>\n",
       "      <td>9.999932e-01</td>\n",
       "      <td>9.998558e-01</td>\n",
       "      <td>9.999987e-01</td>\n",
       "      <td>9.999995e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8394</th>\n",
       "      <td>8ae30ce6</td>\n",
       "      <td>1.027925e-03</td>\n",
       "      <td>6.909480e-04</td>\n",
       "      <td>5.888595e-05</td>\n",
       "      <td>5.139901e-06</td>\n",
       "      <td>6.562841e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8395</th>\n",
       "      <td>de27ed88</td>\n",
       "      <td>1.377063e-06</td>\n",
       "      <td>1.180363e-05</td>\n",
       "      <td>2.015048e-05</td>\n",
       "      <td>2.182222e-08</td>\n",
       "      <td>4.638314e-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8396</th>\n",
       "      <td>66d5196f</td>\n",
       "      <td>1.108589e-01</td>\n",
       "      <td>3.955138e-02</td>\n",
       "      <td>1.815870e-02</td>\n",
       "      <td>1.669010e-01</td>\n",
       "      <td>3.089899e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8397</th>\n",
       "      <td>d85f1858</td>\n",
       "      <td>6.098050e-01</td>\n",
       "      <td>8.873850e-01</td>\n",
       "      <td>4.649777e-01</td>\n",
       "      <td>6.202719e-01</td>\n",
       "      <td>2.729808e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8398</th>\n",
       "      <td>16dcb33a</td>\n",
       "      <td>4.826720e-01</td>\n",
       "      <td>8.960060e-01</td>\n",
       "      <td>1.729266e-01</td>\n",
       "      <td>3.625170e-01</td>\n",
       "      <td>9.194563e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8399</th>\n",
       "      <td>eca3158e</td>\n",
       "      <td>4.961205e-05</td>\n",
       "      <td>2.655229e-06</td>\n",
       "      <td>6.830871e-07</td>\n",
       "      <td>1.162538e-05</td>\n",
       "      <td>1.005283e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8400</th>\n",
       "      <td>08daeee6</td>\n",
       "      <td>1.555613e-03</td>\n",
       "      <td>2.683922e-03</td>\n",
       "      <td>4.425508e-04</td>\n",
       "      <td>1.137437e-05</td>\n",
       "      <td>3.475475e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8401</th>\n",
       "      <td>e9c513ee</td>\n",
       "      <td>1.044979e-04</td>\n",
       "      <td>6.859004e-05</td>\n",
       "      <td>1.440048e-03</td>\n",
       "      <td>7.341595e-06</td>\n",
       "      <td>1.804747e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8402</th>\n",
       "      <td>b1519fa6</td>\n",
       "      <td>5.034913e-01</td>\n",
       "      <td>8.893497e-01</td>\n",
       "      <td>3.615991e-01</td>\n",
       "      <td>5.334579e-01</td>\n",
       "      <td>1.632764e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8403</th>\n",
       "      <td>dfc89540</td>\n",
       "      <td>3.166263e-01</td>\n",
       "      <td>5.819805e-01</td>\n",
       "      <td>1.975608e-01</td>\n",
       "      <td>8.605079e-02</td>\n",
       "      <td>1.889828e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8404</th>\n",
       "      <td>8fd8c0e9</td>\n",
       "      <td>7.018314e-01</td>\n",
       "      <td>8.961312e-01</td>\n",
       "      <td>5.426914e-01</td>\n",
       "      <td>4.041732e-01</td>\n",
       "      <td>3.649027e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8405</th>\n",
       "      <td>45df6347</td>\n",
       "      <td>6.745182e-04</td>\n",
       "      <td>7.321003e-04</td>\n",
       "      <td>2.210055e-04</td>\n",
       "      <td>4.798997e-06</td>\n",
       "      <td>2.060182e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8406</th>\n",
       "      <td>bf7928d7</td>\n",
       "      <td>6.432744e-04</td>\n",
       "      <td>1.223835e-04</td>\n",
       "      <td>1.277349e-06</td>\n",
       "      <td>5.546077e-06</td>\n",
       "      <td>5.530911e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8407</th>\n",
       "      <td>7b587c05</td>\n",
       "      <td>3.508109e-02</td>\n",
       "      <td>2.629731e-02</td>\n",
       "      <td>2.727933e-03</td>\n",
       "      <td>1.717930e-05</td>\n",
       "      <td>6.469895e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8408</th>\n",
       "      <td>c2834388</td>\n",
       "      <td>2.572496e-01</td>\n",
       "      <td>4.950851e-01</td>\n",
       "      <td>7.370229e-02</td>\n",
       "      <td>1.362856e-01</td>\n",
       "      <td>3.132047e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8409</th>\n",
       "      <td>146143c3</td>\n",
       "      <td>9.993097e-01</td>\n",
       "      <td>9.999967e-01</td>\n",
       "      <td>9.998068e-01</td>\n",
       "      <td>9.999962e-01</td>\n",
       "      <td>9.999985e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8410</th>\n",
       "      <td>d59aee00</td>\n",
       "      <td>6.161645e-08</td>\n",
       "      <td>7.447319e-08</td>\n",
       "      <td>2.159056e-06</td>\n",
       "      <td>8.249420e-09</td>\n",
       "      <td>1.164222e-13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8411</th>\n",
       "      <td>cbc0b93b</td>\n",
       "      <td>3.506733e-01</td>\n",
       "      <td>7.191308e-01</td>\n",
       "      <td>5.206221e-02</td>\n",
       "      <td>3.252431e-02</td>\n",
       "      <td>4.744930e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8412</th>\n",
       "      <td>088e2ff7</td>\n",
       "      <td>9.493098e-01</td>\n",
       "      <td>9.884309e-01</td>\n",
       "      <td>9.704044e-01</td>\n",
       "      <td>9.935458e-01</td>\n",
       "      <td>6.229601e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8413</th>\n",
       "      <td>673d33cd</td>\n",
       "      <td>5.702388e-02</td>\n",
       "      <td>9.616089e-02</td>\n",
       "      <td>6.940368e-02</td>\n",
       "      <td>4.169415e-03</td>\n",
       "      <td>4.285263e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8414</th>\n",
       "      <td>674b031e</td>\n",
       "      <td>1.830288e-04</td>\n",
       "      <td>2.247378e-04</td>\n",
       "      <td>1.896236e-05</td>\n",
       "      <td>1.177426e-06</td>\n",
       "      <td>3.736832e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8415</th>\n",
       "      <td>43db4207</td>\n",
       "      <td>8.535291e-01</td>\n",
       "      <td>6.895116e-01</td>\n",
       "      <td>3.483818e-01</td>\n",
       "      <td>2.983214e-01</td>\n",
       "      <td>2.023785e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8416</th>\n",
       "      <td>156855e1</td>\n",
       "      <td>6.737168e-09</td>\n",
       "      <td>2.469757e-11</td>\n",
       "      <td>4.007767e-09</td>\n",
       "      <td>1.721056e-11</td>\n",
       "      <td>3.046618e-16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8417</th>\n",
       "      <td>ac96cfb0</td>\n",
       "      <td>3.358772e-02</td>\n",
       "      <td>1.665146e-02</td>\n",
       "      <td>2.161247e-02</td>\n",
       "      <td>4.287829e-02</td>\n",
       "      <td>5.285869e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8418</th>\n",
       "      <td>fe45aef5</td>\n",
       "      <td>8.980864e-01</td>\n",
       "      <td>9.959552e-01</td>\n",
       "      <td>9.452092e-01</td>\n",
       "      <td>9.577305e-01</td>\n",
       "      <td>3.437249e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8419</th>\n",
       "      <td>16ee9b50</td>\n",
       "      <td>4.760905e-07</td>\n",
       "      <td>1.263890e-07</td>\n",
       "      <td>1.116841e-08</td>\n",
       "      <td>9.440033e-10</td>\n",
       "      <td>2.793488e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8420</th>\n",
       "      <td>5a599eb7</td>\n",
       "      <td>3.634332e-01</td>\n",
       "      <td>8.072237e-01</td>\n",
       "      <td>1.210266e-01</td>\n",
       "      <td>1.517795e-01</td>\n",
       "      <td>1.508340e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8421</th>\n",
       "      <td>df30d6dd</td>\n",
       "      <td>1.465780e-02</td>\n",
       "      <td>4.289179e-03</td>\n",
       "      <td>4.484839e-03</td>\n",
       "      <td>6.736985e-05</td>\n",
       "      <td>4.927859e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8422</th>\n",
       "      <td>18af95b1</td>\n",
       "      <td>9.993871e-01</td>\n",
       "      <td>9.997384e-01</td>\n",
       "      <td>9.989774e-01</td>\n",
       "      <td>9.999465e-01</td>\n",
       "      <td>9.997872e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8423</th>\n",
       "      <td>27d788c8</td>\n",
       "      <td>3.238954e-01</td>\n",
       "      <td>1.725587e-01</td>\n",
       "      <td>7.514057e-02</td>\n",
       "      <td>6.552050e-02</td>\n",
       "      <td>8.111687e-02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8424 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            id           cv1           cv2           cv3           cv4  \\\n",
       "0     5941774d  5.667688e-02  5.530631e-02  8.635153e-03  1.127893e-03   \n",
       "1     4023181e  5.026714e-01  8.580565e-01  3.927149e-01  3.979664e-01   \n",
       "2     b20200e4  2.789924e-06  3.951226e-10  1.032073e-10  5.433600e-08   \n",
       "3     e7f018bb  9.994588e-01  9.999734e-01  9.998581e-01  9.999951e-01   \n",
       "4     4371c8c3  1.669144e-02  6.438503e-02  4.461252e-02  3.018984e-03   \n",
       "5     a8d9b1fd  1.406482e-01  1.800535e-02  2.911703e-03  8.031805e-03   \n",
       "6     29e7727e  1.855465e-01  1.123179e-01  7.316975e-02  5.049995e-03   \n",
       "7     92a51ffb  9.979691e-01  9.999943e-01  9.998738e-01  9.999973e-01   \n",
       "8     c769ac97  8.177361e-05  5.616964e-05  2.393075e-06  2.250952e-06   \n",
       "9     aee0547d  4.983280e-07  2.763404e-07  2.184263e-08  1.156752e-09   \n",
       "10    565b28ac  1.511666e-05  8.481084e-06  1.915825e-06  1.947710e-08   \n",
       "11    e04e9775  2.145108e-01  6.484185e-01  3.081630e-01  9.101298e-02   \n",
       "12    8e8161d1  2.706455e-03  7.825905e-03  6.196049e-03  2.258463e-05   \n",
       "13    4cf4d256  3.776796e-01  8.680509e-01  3.707848e-01  3.931115e-01   \n",
       "14    139e5324  1.657643e-02  2.716942e-04  3.593864e-05  4.265339e-03   \n",
       "15    f156976f  1.608710e-03  7.008111e-03  1.274463e-03  5.250456e-04   \n",
       "16    68a117cc  8.594736e-02  2.305674e-03  1.826357e-03  9.703810e-03   \n",
       "17    d9aa7a56  2.259670e-01  2.138297e-02  4.971057e-03  1.381533e-02   \n",
       "18    9005b143  5.395933e-01  2.903121e-01  2.651281e-02  1.873317e-02   \n",
       "19    5f6d3988  9.959655e-01  9.992558e-01  9.740520e-01  9.994076e-01   \n",
       "20    9ad70954  1.401283e-02  7.709266e-03  2.763549e-03  1.011760e-04   \n",
       "21    b9087b9e  2.441433e-01  4.942774e-01  5.023513e-01  2.631663e-01   \n",
       "22    a39a1427  2.748383e-01  6.085083e-01  7.936671e-02  1.877789e-01   \n",
       "23    82fbe8ed  1.093100e-01  1.750146e-01  4.207040e-02  2.376729e-03   \n",
       "24    1fae4879  3.575499e-08  1.084889e-08  1.098709e-06  2.299321e-09   \n",
       "25    6dd8f13d  3.685710e-05  3.017996e-08  1.945800e-09  2.980999e-07   \n",
       "26    bbad5958  5.359598e-02  8.420833e-02  5.728618e-02  2.809570e-03   \n",
       "27    54527583  3.359738e-02  3.805183e-02  2.905826e-03  8.412496e-04   \n",
       "28    be8fa29c  4.603783e-02  5.425699e-02  2.093825e-02  5.805754e-04   \n",
       "29    81a3328f  9.999920e-01  9.999932e-01  9.998558e-01  9.999987e-01   \n",
       "...        ...           ...           ...           ...           ...   \n",
       "8394  8ae30ce6  1.027925e-03  6.909480e-04  5.888595e-05  5.139901e-06   \n",
       "8395  de27ed88  1.377063e-06  1.180363e-05  2.015048e-05  2.182222e-08   \n",
       "8396  66d5196f  1.108589e-01  3.955138e-02  1.815870e-02  1.669010e-01   \n",
       "8397  d85f1858  6.098050e-01  8.873850e-01  4.649777e-01  6.202719e-01   \n",
       "8398  16dcb33a  4.826720e-01  8.960060e-01  1.729266e-01  3.625170e-01   \n",
       "8399  eca3158e  4.961205e-05  2.655229e-06  6.830871e-07  1.162538e-05   \n",
       "8400  08daeee6  1.555613e-03  2.683922e-03  4.425508e-04  1.137437e-05   \n",
       "8401  e9c513ee  1.044979e-04  6.859004e-05  1.440048e-03  7.341595e-06   \n",
       "8402  b1519fa6  5.034913e-01  8.893497e-01  3.615991e-01  5.334579e-01   \n",
       "8403  dfc89540  3.166263e-01  5.819805e-01  1.975608e-01  8.605079e-02   \n",
       "8404  8fd8c0e9  7.018314e-01  8.961312e-01  5.426914e-01  4.041732e-01   \n",
       "8405  45df6347  6.745182e-04  7.321003e-04  2.210055e-04  4.798997e-06   \n",
       "8406  bf7928d7  6.432744e-04  1.223835e-04  1.277349e-06  5.546077e-06   \n",
       "8407  7b587c05  3.508109e-02  2.629731e-02  2.727933e-03  1.717930e-05   \n",
       "8408  c2834388  2.572496e-01  4.950851e-01  7.370229e-02  1.362856e-01   \n",
       "8409  146143c3  9.993097e-01  9.999967e-01  9.998068e-01  9.999962e-01   \n",
       "8410  d59aee00  6.161645e-08  7.447319e-08  2.159056e-06  8.249420e-09   \n",
       "8411  cbc0b93b  3.506733e-01  7.191308e-01  5.206221e-02  3.252431e-02   \n",
       "8412  088e2ff7  9.493098e-01  9.884309e-01  9.704044e-01  9.935458e-01   \n",
       "8413  673d33cd  5.702388e-02  9.616089e-02  6.940368e-02  4.169415e-03   \n",
       "8414  674b031e  1.830288e-04  2.247378e-04  1.896236e-05  1.177426e-06   \n",
       "8415  43db4207  8.535291e-01  6.895116e-01  3.483818e-01  2.983214e-01   \n",
       "8416  156855e1  6.737168e-09  2.469757e-11  4.007767e-09  1.721056e-11   \n",
       "8417  ac96cfb0  3.358772e-02  1.665146e-02  2.161247e-02  4.287829e-02   \n",
       "8418  fe45aef5  8.980864e-01  9.959552e-01  9.452092e-01  9.577305e-01   \n",
       "8419  16ee9b50  4.760905e-07  1.263890e-07  1.116841e-08  9.440033e-10   \n",
       "8420  5a599eb7  3.634332e-01  8.072237e-01  1.210266e-01  1.517795e-01   \n",
       "8421  df30d6dd  1.465780e-02  4.289179e-03  4.484839e-03  6.736985e-05   \n",
       "8422  18af95b1  9.993871e-01  9.997384e-01  9.989774e-01  9.999465e-01   \n",
       "8423  27d788c8  3.238954e-01  1.725587e-01  7.514057e-02  6.552050e-02   \n",
       "\n",
       "               cv5  \n",
       "0     4.170072e-03  \n",
       "1     2.290074e-02  \n",
       "2     8.329140e-08  \n",
       "3     9.999849e-01  \n",
       "4     1.088171e-04  \n",
       "5     2.462742e-03  \n",
       "6     2.020663e-02  \n",
       "7     9.999726e-01  \n",
       "8     6.520145e-07  \n",
       "9     2.064336e-09  \n",
       "10    5.455509e-09  \n",
       "11    2.580765e-03  \n",
       "12    1.751517e-05  \n",
       "13    2.610141e-02  \n",
       "14    8.636447e-06  \n",
       "15    8.246358e-05  \n",
       "16    4.464464e-06  \n",
       "17    6.850597e-04  \n",
       "18    4.544104e-03  \n",
       "19    9.604470e-01  \n",
       "20    7.388195e-05  \n",
       "21    1.896616e-02  \n",
       "22    2.060031e-03  \n",
       "23    3.546486e-04  \n",
       "24    1.099892e-13  \n",
       "25    2.638232e-07  \n",
       "26    1.495250e-04  \n",
       "27    1.929508e-04  \n",
       "28    3.134798e-04  \n",
       "29    9.999995e-01  \n",
       "...            ...  \n",
       "8394  6.562841e-07  \n",
       "8395  4.638314e-12  \n",
       "8396  3.089899e-04  \n",
       "8397  2.729808e-02  \n",
       "8398  9.194563e-03  \n",
       "8399  1.005283e-09  \n",
       "8400  3.475475e-05  \n",
       "8401  1.804747e-06  \n",
       "8402  1.632764e-02  \n",
       "8403  1.889828e-03  \n",
       "8404  3.649027e-02  \n",
       "8405  2.060182e-06  \n",
       "8406  5.530911e-07  \n",
       "8407  6.469895e-06  \n",
       "8408  3.132047e-04  \n",
       "8409  9.999985e-01  \n",
       "8410  1.164222e-13  \n",
       "8411  4.744930e-03  \n",
       "8412  6.229601e-01  \n",
       "8413  4.285263e-04  \n",
       "8414  3.736832e-07  \n",
       "8415  2.023785e-03  \n",
       "8416  3.046618e-16  \n",
       "8417  5.285869e-05  \n",
       "8418  3.437249e-01  \n",
       "8419  2.793488e-10  \n",
       "8420  1.508340e-03  \n",
       "8421  4.927859e-05  \n",
       "8422  9.997872e-01  \n",
       "8423  8.111687e-02  \n",
       "\n",
       "[8424 rows x 6 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_sub_mmm(test_df):\n",
    "    prediction = test_df\n",
    "    prediction['is_iceberg'] = np.where(np.all(prediction[prediction.columns[1:]]>0.9,axis = 1),\n",
    "                                    np.max(prediction[prediction.columns[1:]],axis = 1),\n",
    "                                     np.where(np.all(prediction[prediction.columns[1:]]<0.1,axis = 1),np.min(prediction[prediction.columns[1:]],axis = 1),np.median(prediction[prediction.columns[1:]],axis = 1)))\n",
    "    sub = prediction[['id','is_iceberg']]\n",
    "    sub.to_csv('sub.csv',index = False)\n",
    "    return sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "sub = get_sub_mmm(test_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Multiple submission stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path = 'prediction/stacking 1/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>lb0.1661.csv</th>\n",
       "      <th>lb0.1717.csv</th>\n",
       "      <th>lb0.1728.csv</th>\n",
       "      <th>lb0.1978.csv</th>\n",
       "      <th>lb0.2168.csv</th>\n",
       "      <th>mean-(2)(4)lb0.1717.csv</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5941774d</td>\n",
       "      <td>4.883857e-02</td>\n",
       "      <td>3.457386e-02</td>\n",
       "      <td>7.377190e-03</td>\n",
       "      <td>3.609517e-03</td>\n",
       "      <td>2.721015e-03</td>\n",
       "      <td>8.815090e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4023181e</td>\n",
       "      <td>3.974485e-01</td>\n",
       "      <td>4.705284e-01</td>\n",
       "      <td>4.416436e-01</td>\n",
       "      <td>3.767341e-01</td>\n",
       "      <td>4.024012e-01</td>\n",
       "      <td>4.446322e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b20200e4</td>\n",
       "      <td>4.541493e-12</td>\n",
       "      <td>2.426515e-14</td>\n",
       "      <td>5.958568e-11</td>\n",
       "      <td>7.925581e-10</td>\n",
       "      <td>6.370236e-12</td>\n",
       "      <td>3.539300e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e7f018bb</td>\n",
       "      <td>9.999995e-01</td>\n",
       "      <td>9.999987e-01</td>\n",
       "      <td>9.999954e-01</td>\n",
       "      <td>9.999983e-01</td>\n",
       "      <td>9.999939e-01</td>\n",
       "      <td>9.999878e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4371c8c3</td>\n",
       "      <td>1.377228e-04</td>\n",
       "      <td>7.653040e-05</td>\n",
       "      <td>4.422425e-04</td>\n",
       "      <td>3.091512e-03</td>\n",
       "      <td>2.120113e-04</td>\n",
       "      <td>3.209088e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>a8d9b1fd</td>\n",
       "      <td>4.990788e-02</td>\n",
       "      <td>8.502229e-02</td>\n",
       "      <td>5.834405e-02</td>\n",
       "      <td>6.308465e-02</td>\n",
       "      <td>7.421116e-04</td>\n",
       "      <td>2.790584e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>29e7727e</td>\n",
       "      <td>1.211617e-01</td>\n",
       "      <td>1.527639e-01</td>\n",
       "      <td>1.079694e-01</td>\n",
       "      <td>1.907021e-01</td>\n",
       "      <td>1.918441e-02</td>\n",
       "      <td>1.444582e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>92a51ffb</td>\n",
       "      <td>9.999950e-01</td>\n",
       "      <td>9.999959e-01</td>\n",
       "      <td>9.999992e-01</td>\n",
       "      <td>9.999897e-01</td>\n",
       "      <td>9.999989e-01</td>\n",
       "      <td>9.999560e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>c769ac97</td>\n",
       "      <td>2.814740e-12</td>\n",
       "      <td>1.046281e-08</td>\n",
       "      <td>2.635662e-09</td>\n",
       "      <td>2.700044e-09</td>\n",
       "      <td>3.130832e-06</td>\n",
       "      <td>3.906417e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>aee0547d</td>\n",
       "      <td>2.542710e-13</td>\n",
       "      <td>6.446605e-12</td>\n",
       "      <td>1.078889e-14</td>\n",
       "      <td>3.189674e-13</td>\n",
       "      <td>1.297212e-08</td>\n",
       "      <td>4.328927e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>565b28ac</td>\n",
       "      <td>9.971104e-13</td>\n",
       "      <td>5.888663e-12</td>\n",
       "      <td>6.722855e-13</td>\n",
       "      <td>1.076952e-11</td>\n",
       "      <td>1.552837e-09</td>\n",
       "      <td>5.762643e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>e04e9775</td>\n",
       "      <td>1.819807e-01</td>\n",
       "      <td>1.019719e-01</td>\n",
       "      <td>2.525990e-01</td>\n",
       "      <td>1.698858e-01</td>\n",
       "      <td>9.596787e-02</td>\n",
       "      <td>7.891672e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>8e8161d1</td>\n",
       "      <td>3.857386e-05</td>\n",
       "      <td>2.738997e-04</td>\n",
       "      <td>1.202334e-04</td>\n",
       "      <td>1.461222e-04</td>\n",
       "      <td>6.403032e-05</td>\n",
       "      <td>3.526576e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>4cf4d256</td>\n",
       "      <td>3.473218e-01</td>\n",
       "      <td>2.508824e-01</td>\n",
       "      <td>3.674485e-01</td>\n",
       "      <td>4.239623e-01</td>\n",
       "      <td>1.974677e-01</td>\n",
       "      <td>3.408813e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>139e5324</td>\n",
       "      <td>1.150640e-07</td>\n",
       "      <td>5.397478e-08</td>\n",
       "      <td>1.083773e-04</td>\n",
       "      <td>2.080786e-05</td>\n",
       "      <td>8.291981e-06</td>\n",
       "      <td>1.039578e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>f156976f</td>\n",
       "      <td>1.041737e-03</td>\n",
       "      <td>2.046762e-05</td>\n",
       "      <td>2.432792e-04</td>\n",
       "      <td>3.732556e-04</td>\n",
       "      <td>9.986849e-05</td>\n",
       "      <td>1.645820e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>68a117cc</td>\n",
       "      <td>2.734506e-05</td>\n",
       "      <td>1.148563e-05</td>\n",
       "      <td>1.051761e-03</td>\n",
       "      <td>4.755504e-05</td>\n",
       "      <td>5.155202e-05</td>\n",
       "      <td>4.690950e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>d9aa7a56</td>\n",
       "      <td>2.632210e-02</td>\n",
       "      <td>2.818458e-02</td>\n",
       "      <td>6.553245e-02</td>\n",
       "      <td>3.636210e-03</td>\n",
       "      <td>1.591020e-03</td>\n",
       "      <td>7.785125e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>9005b143</td>\n",
       "      <td>3.735686e-02</td>\n",
       "      <td>1.252761e-01</td>\n",
       "      <td>2.325613e-01</td>\n",
       "      <td>2.137174e-01</td>\n",
       "      <td>3.841120e-02</td>\n",
       "      <td>2.930468e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>5f6d3988</td>\n",
       "      <td>9.995091e-01</td>\n",
       "      <td>9.992446e-01</td>\n",
       "      <td>9.995149e-01</td>\n",
       "      <td>9.965498e-01</td>\n",
       "      <td>9.808260e-01</td>\n",
       "      <td>9.902107e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>9ad70954</td>\n",
       "      <td>2.501507e-05</td>\n",
       "      <td>4.546368e-05</td>\n",
       "      <td>9.811139e-04</td>\n",
       "      <td>9.323658e-05</td>\n",
       "      <td>2.770799e-05</td>\n",
       "      <td>7.274421e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>b9087b9e</td>\n",
       "      <td>4.356706e-01</td>\n",
       "      <td>2.627963e-01</td>\n",
       "      <td>3.184613e-01</td>\n",
       "      <td>4.025350e-01</td>\n",
       "      <td>2.274429e-01</td>\n",
       "      <td>3.658111e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>a39a1427</td>\n",
       "      <td>9.563221e-02</td>\n",
       "      <td>1.002567e-01</td>\n",
       "      <td>1.961334e-01</td>\n",
       "      <td>1.131404e-01</td>\n",
       "      <td>6.434636e-02</td>\n",
       "      <td>6.154930e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>82fbe8ed</td>\n",
       "      <td>5.137844e-02</td>\n",
       "      <td>2.904787e-03</td>\n",
       "      <td>6.970322e-02</td>\n",
       "      <td>5.754833e-02</td>\n",
       "      <td>2.952904e-03</td>\n",
       "      <td>1.790067e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1fae4879</td>\n",
       "      <td>1.721667e-15</td>\n",
       "      <td>2.377082e-18</td>\n",
       "      <td>2.513818e-22</td>\n",
       "      <td>1.065264e-20</td>\n",
       "      <td>2.361504e-11</td>\n",
       "      <td>1.654060e-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>6dd8f13d</td>\n",
       "      <td>4.883254e-11</td>\n",
       "      <td>1.074130e-13</td>\n",
       "      <td>2.447664e-11</td>\n",
       "      <td>1.598836e-09</td>\n",
       "      <td>1.752715e-09</td>\n",
       "      <td>9.518749e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>bbad5958</td>\n",
       "      <td>1.642404e-03</td>\n",
       "      <td>2.905246e-04</td>\n",
       "      <td>2.006807e-02</td>\n",
       "      <td>7.325834e-02</td>\n",
       "      <td>1.867912e-03</td>\n",
       "      <td>4.206313e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>54527583</td>\n",
       "      <td>1.738595e-03</td>\n",
       "      <td>1.223689e-05</td>\n",
       "      <td>1.989683e-03</td>\n",
       "      <td>2.310304e-04</td>\n",
       "      <td>3.261114e-05</td>\n",
       "      <td>1.315730e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>be8fa29c</td>\n",
       "      <td>3.743509e-02</td>\n",
       "      <td>6.152839e-04</td>\n",
       "      <td>1.880052e-03</td>\n",
       "      <td>1.088336e-04</td>\n",
       "      <td>3.825938e-04</td>\n",
       "      <td>3.722554e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>81a3328f</td>\n",
       "      <td>9.999998e-01</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>9.999980e-01</td>\n",
       "      <td>9.999989e-01</td>\n",
       "      <td>9.999965e-01</td>\n",
       "      <td>9.999979e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8394</th>\n",
       "      <td>8ae30ce6</td>\n",
       "      <td>2.933512e-07</td>\n",
       "      <td>1.970363e-07</td>\n",
       "      <td>6.423959e-07</td>\n",
       "      <td>7.008944e-07</td>\n",
       "      <td>2.530175e-07</td>\n",
       "      <td>5.571086e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8395</th>\n",
       "      <td>de27ed88</td>\n",
       "      <td>5.462461e-14</td>\n",
       "      <td>4.662100e-13</td>\n",
       "      <td>3.010890e-19</td>\n",
       "      <td>2.105800e-12</td>\n",
       "      <td>2.433944e-08</td>\n",
       "      <td>2.389352e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8396</th>\n",
       "      <td>66d5196f</td>\n",
       "      <td>5.470167e-02</td>\n",
       "      <td>3.276580e-02</td>\n",
       "      <td>8.763818e-02</td>\n",
       "      <td>3.883383e-02</td>\n",
       "      <td>3.911817e-03</td>\n",
       "      <td>2.362077e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8397</th>\n",
       "      <td>d85f1858</td>\n",
       "      <td>4.493008e-01</td>\n",
       "      <td>4.202433e-01</td>\n",
       "      <td>4.632542e-01</td>\n",
       "      <td>4.503037e-01</td>\n",
       "      <td>4.659027e-01</td>\n",
       "      <td>5.016388e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8398</th>\n",
       "      <td>16dcb33a</td>\n",
       "      <td>2.501919e-01</td>\n",
       "      <td>3.327725e-01</td>\n",
       "      <td>3.547991e-01</td>\n",
       "      <td>2.227086e-01</td>\n",
       "      <td>2.239254e-01</td>\n",
       "      <td>2.848360e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8399</th>\n",
       "      <td>eca3158e</td>\n",
       "      <td>3.985148e-14</td>\n",
       "      <td>2.643913e-14</td>\n",
       "      <td>4.789223e-11</td>\n",
       "      <td>6.358623e-11</td>\n",
       "      <td>2.731834e-10</td>\n",
       "      <td>9.486391e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8400</th>\n",
       "      <td>08daeee6</td>\n",
       "      <td>6.011877e-08</td>\n",
       "      <td>2.914450e-05</td>\n",
       "      <td>4.466206e-06</td>\n",
       "      <td>1.595997e-05</td>\n",
       "      <td>1.159579e-05</td>\n",
       "      <td>3.541269e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8401</th>\n",
       "      <td>e9c513ee</td>\n",
       "      <td>3.501544e-08</td>\n",
       "      <td>7.333873e-07</td>\n",
       "      <td>4.784196e-06</td>\n",
       "      <td>1.490626e-05</td>\n",
       "      <td>2.051471e-06</td>\n",
       "      <td>5.302218e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8402</th>\n",
       "      <td>b1519fa6</td>\n",
       "      <td>2.848104e-01</td>\n",
       "      <td>4.239774e-01</td>\n",
       "      <td>4.226339e-01</td>\n",
       "      <td>3.819768e-01</td>\n",
       "      <td>3.026948e-01</td>\n",
       "      <td>3.259458e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8403</th>\n",
       "      <td>dfc89540</td>\n",
       "      <td>1.769256e-01</td>\n",
       "      <td>1.190542e-01</td>\n",
       "      <td>1.380085e-01</td>\n",
       "      <td>1.321200e-01</td>\n",
       "      <td>2.460779e-02</td>\n",
       "      <td>9.331551e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8404</th>\n",
       "      <td>8fd8c0e9</td>\n",
       "      <td>4.552343e-01</td>\n",
       "      <td>3.948152e-01</td>\n",
       "      <td>5.482550e-01</td>\n",
       "      <td>4.375779e-01</td>\n",
       "      <td>5.775682e-01</td>\n",
       "      <td>3.672767e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8405</th>\n",
       "      <td>45df6347</td>\n",
       "      <td>5.201120e-08</td>\n",
       "      <td>1.604495e-05</td>\n",
       "      <td>2.253464e-07</td>\n",
       "      <td>8.184473e-06</td>\n",
       "      <td>2.448407e-06</td>\n",
       "      <td>1.260926e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8406</th>\n",
       "      <td>bf7928d7</td>\n",
       "      <td>6.135443e-09</td>\n",
       "      <td>3.252151e-09</td>\n",
       "      <td>1.681153e-07</td>\n",
       "      <td>5.019085e-07</td>\n",
       "      <td>6.845485e-08</td>\n",
       "      <td>4.148419e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8407</th>\n",
       "      <td>7b587c05</td>\n",
       "      <td>4.449146e-06</td>\n",
       "      <td>1.572234e-04</td>\n",
       "      <td>4.446942e-05</td>\n",
       "      <td>1.035350e-04</td>\n",
       "      <td>4.939414e-06</td>\n",
       "      <td>2.563609e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8408</th>\n",
       "      <td>c2834388</td>\n",
       "      <td>1.234593e-01</td>\n",
       "      <td>7.756566e-02</td>\n",
       "      <td>6.331462e-02</td>\n",
       "      <td>1.736936e-01</td>\n",
       "      <td>5.274712e-02</td>\n",
       "      <td>2.519771e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8409</th>\n",
       "      <td>146143c3</td>\n",
       "      <td>9.999927e-01</td>\n",
       "      <td>9.999996e-01</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>9.999996e-01</td>\n",
       "      <td>9.999996e-01</td>\n",
       "      <td>9.999975e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8410</th>\n",
       "      <td>d59aee00</td>\n",
       "      <td>6.455148e-18</td>\n",
       "      <td>1.169055e-13</td>\n",
       "      <td>2.073761e-20</td>\n",
       "      <td>7.965661e-19</td>\n",
       "      <td>1.209952e-11</td>\n",
       "      <td>2.170930e-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8411</th>\n",
       "      <td>cbc0b93b</td>\n",
       "      <td>8.742154e-02</td>\n",
       "      <td>6.891542e-02</td>\n",
       "      <td>1.838135e-01</td>\n",
       "      <td>1.500871e-01</td>\n",
       "      <td>1.091372e-02</td>\n",
       "      <td>3.269133e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8412</th>\n",
       "      <td>088e2ff7</td>\n",
       "      <td>9.567771e-01</td>\n",
       "      <td>9.451587e-01</td>\n",
       "      <td>9.504524e-01</td>\n",
       "      <td>9.020713e-01</td>\n",
       "      <td>9.607627e-01</td>\n",
       "      <td>8.621675e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8413</th>\n",
       "      <td>673d33cd</td>\n",
       "      <td>2.596729e-02</td>\n",
       "      <td>6.584844e-02</td>\n",
       "      <td>1.826823e-03</td>\n",
       "      <td>4.710763e-02</td>\n",
       "      <td>3.654592e-04</td>\n",
       "      <td>5.304924e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8414</th>\n",
       "      <td>674b031e</td>\n",
       "      <td>7.734672e-08</td>\n",
       "      <td>8.454788e-08</td>\n",
       "      <td>2.784772e-08</td>\n",
       "      <td>8.925614e-07</td>\n",
       "      <td>2.121196e-07</td>\n",
       "      <td>9.440746e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8415</th>\n",
       "      <td>43db4207</td>\n",
       "      <td>1.358420e-01</td>\n",
       "      <td>1.214023e-01</td>\n",
       "      <td>1.138083e-01</td>\n",
       "      <td>1.038796e-01</td>\n",
       "      <td>4.858482e-03</td>\n",
       "      <td>1.571365e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8416</th>\n",
       "      <td>156855e1</td>\n",
       "      <td>3.144757e-30</td>\n",
       "      <td>1.605725e-23</td>\n",
       "      <td>4.975584e-28</td>\n",
       "      <td>3.743123e-36</td>\n",
       "      <td>2.032915e-19</td>\n",
       "      <td>7.301838e-18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8417</th>\n",
       "      <td>ac96cfb0</td>\n",
       "      <td>2.309585e-02</td>\n",
       "      <td>8.186708e-05</td>\n",
       "      <td>3.603250e-03</td>\n",
       "      <td>1.600608e-02</td>\n",
       "      <td>6.733650e-04</td>\n",
       "      <td>2.336607e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8418</th>\n",
       "      <td>fe45aef5</td>\n",
       "      <td>9.762937e-01</td>\n",
       "      <td>9.424389e-01</td>\n",
       "      <td>7.882308e-01</td>\n",
       "      <td>9.335779e-01</td>\n",
       "      <td>9.682497e-01</td>\n",
       "      <td>9.405996e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8419</th>\n",
       "      <td>16ee9b50</td>\n",
       "      <td>4.977454e-13</td>\n",
       "      <td>2.267374e-12</td>\n",
       "      <td>2.501841e-14</td>\n",
       "      <td>4.781171e-12</td>\n",
       "      <td>2.518948e-08</td>\n",
       "      <td>2.518590e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8420</th>\n",
       "      <td>5a599eb7</td>\n",
       "      <td>1.458463e-01</td>\n",
       "      <td>7.644506e-02</td>\n",
       "      <td>1.248203e-01</td>\n",
       "      <td>1.643127e-01</td>\n",
       "      <td>3.723660e-02</td>\n",
       "      <td>1.405674e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8421</th>\n",
       "      <td>df30d6dd</td>\n",
       "      <td>1.992619e-06</td>\n",
       "      <td>7.074945e-06</td>\n",
       "      <td>1.662818e-04</td>\n",
       "      <td>2.759178e-05</td>\n",
       "      <td>2.788007e-05</td>\n",
       "      <td>1.539492e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8422</th>\n",
       "      <td>18af95b1</td>\n",
       "      <td>9.999552e-01</td>\n",
       "      <td>9.999183e-01</td>\n",
       "      <td>9.999791e-01</td>\n",
       "      <td>9.999415e-01</td>\n",
       "      <td>9.999895e-01</td>\n",
       "      <td>9.896075e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8423</th>\n",
       "      <td>27d788c8</td>\n",
       "      <td>2.921648e-01</td>\n",
       "      <td>6.933905e-01</td>\n",
       "      <td>9.868053e-01</td>\n",
       "      <td>5.997514e-01</td>\n",
       "      <td>2.076232e-01</td>\n",
       "      <td>2.555270e-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8424 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            id  lb0.1661.csv  lb0.1717.csv  lb0.1728.csv  lb0.1978.csv  \\\n",
       "0     5941774d  4.883857e-02  3.457386e-02  7.377190e-03  3.609517e-03   \n",
       "1     4023181e  3.974485e-01  4.705284e-01  4.416436e-01  3.767341e-01   \n",
       "2     b20200e4  4.541493e-12  2.426515e-14  5.958568e-11  7.925581e-10   \n",
       "3     e7f018bb  9.999995e-01  9.999987e-01  9.999954e-01  9.999983e-01   \n",
       "4     4371c8c3  1.377228e-04  7.653040e-05  4.422425e-04  3.091512e-03   \n",
       "5     a8d9b1fd  4.990788e-02  8.502229e-02  5.834405e-02  6.308465e-02   \n",
       "6     29e7727e  1.211617e-01  1.527639e-01  1.079694e-01  1.907021e-01   \n",
       "7     92a51ffb  9.999950e-01  9.999959e-01  9.999992e-01  9.999897e-01   \n",
       "8     c769ac97  2.814740e-12  1.046281e-08  2.635662e-09  2.700044e-09   \n",
       "9     aee0547d  2.542710e-13  6.446605e-12  1.078889e-14  3.189674e-13   \n",
       "10    565b28ac  9.971104e-13  5.888663e-12  6.722855e-13  1.076952e-11   \n",
       "11    e04e9775  1.819807e-01  1.019719e-01  2.525990e-01  1.698858e-01   \n",
       "12    8e8161d1  3.857386e-05  2.738997e-04  1.202334e-04  1.461222e-04   \n",
       "13    4cf4d256  3.473218e-01  2.508824e-01  3.674485e-01  4.239623e-01   \n",
       "14    139e5324  1.150640e-07  5.397478e-08  1.083773e-04  2.080786e-05   \n",
       "15    f156976f  1.041737e-03  2.046762e-05  2.432792e-04  3.732556e-04   \n",
       "16    68a117cc  2.734506e-05  1.148563e-05  1.051761e-03  4.755504e-05   \n",
       "17    d9aa7a56  2.632210e-02  2.818458e-02  6.553245e-02  3.636210e-03   \n",
       "18    9005b143  3.735686e-02  1.252761e-01  2.325613e-01  2.137174e-01   \n",
       "19    5f6d3988  9.995091e-01  9.992446e-01  9.995149e-01  9.965498e-01   \n",
       "20    9ad70954  2.501507e-05  4.546368e-05  9.811139e-04  9.323658e-05   \n",
       "21    b9087b9e  4.356706e-01  2.627963e-01  3.184613e-01  4.025350e-01   \n",
       "22    a39a1427  9.563221e-02  1.002567e-01  1.961334e-01  1.131404e-01   \n",
       "23    82fbe8ed  5.137844e-02  2.904787e-03  6.970322e-02  5.754833e-02   \n",
       "24    1fae4879  1.721667e-15  2.377082e-18  2.513818e-22  1.065264e-20   \n",
       "25    6dd8f13d  4.883254e-11  1.074130e-13  2.447664e-11  1.598836e-09   \n",
       "26    bbad5958  1.642404e-03  2.905246e-04  2.006807e-02  7.325834e-02   \n",
       "27    54527583  1.738595e-03  1.223689e-05  1.989683e-03  2.310304e-04   \n",
       "28    be8fa29c  3.743509e-02  6.152839e-04  1.880052e-03  1.088336e-04   \n",
       "29    81a3328f  9.999998e-01  1.000000e+00  9.999980e-01  9.999989e-01   \n",
       "...        ...           ...           ...           ...           ...   \n",
       "8394  8ae30ce6  2.933512e-07  1.970363e-07  6.423959e-07  7.008944e-07   \n",
       "8395  de27ed88  5.462461e-14  4.662100e-13  3.010890e-19  2.105800e-12   \n",
       "8396  66d5196f  5.470167e-02  3.276580e-02  8.763818e-02  3.883383e-02   \n",
       "8397  d85f1858  4.493008e-01  4.202433e-01  4.632542e-01  4.503037e-01   \n",
       "8398  16dcb33a  2.501919e-01  3.327725e-01  3.547991e-01  2.227086e-01   \n",
       "8399  eca3158e  3.985148e-14  2.643913e-14  4.789223e-11  6.358623e-11   \n",
       "8400  08daeee6  6.011877e-08  2.914450e-05  4.466206e-06  1.595997e-05   \n",
       "8401  e9c513ee  3.501544e-08  7.333873e-07  4.784196e-06  1.490626e-05   \n",
       "8402  b1519fa6  2.848104e-01  4.239774e-01  4.226339e-01  3.819768e-01   \n",
       "8403  dfc89540  1.769256e-01  1.190542e-01  1.380085e-01  1.321200e-01   \n",
       "8404  8fd8c0e9  4.552343e-01  3.948152e-01  5.482550e-01  4.375779e-01   \n",
       "8405  45df6347  5.201120e-08  1.604495e-05  2.253464e-07  8.184473e-06   \n",
       "8406  bf7928d7  6.135443e-09  3.252151e-09  1.681153e-07  5.019085e-07   \n",
       "8407  7b587c05  4.449146e-06  1.572234e-04  4.446942e-05  1.035350e-04   \n",
       "8408  c2834388  1.234593e-01  7.756566e-02  6.331462e-02  1.736936e-01   \n",
       "8409  146143c3  9.999927e-01  9.999996e-01  1.000000e+00  9.999996e-01   \n",
       "8410  d59aee00  6.455148e-18  1.169055e-13  2.073761e-20  7.965661e-19   \n",
       "8411  cbc0b93b  8.742154e-02  6.891542e-02  1.838135e-01  1.500871e-01   \n",
       "8412  088e2ff7  9.567771e-01  9.451587e-01  9.504524e-01  9.020713e-01   \n",
       "8413  673d33cd  2.596729e-02  6.584844e-02  1.826823e-03  4.710763e-02   \n",
       "8414  674b031e  7.734672e-08  8.454788e-08  2.784772e-08  8.925614e-07   \n",
       "8415  43db4207  1.358420e-01  1.214023e-01  1.138083e-01  1.038796e-01   \n",
       "8416  156855e1  3.144757e-30  1.605725e-23  4.975584e-28  3.743123e-36   \n",
       "8417  ac96cfb0  2.309585e-02  8.186708e-05  3.603250e-03  1.600608e-02   \n",
       "8418  fe45aef5  9.762937e-01  9.424389e-01  7.882308e-01  9.335779e-01   \n",
       "8419  16ee9b50  4.977454e-13  2.267374e-12  2.501841e-14  4.781171e-12   \n",
       "8420  5a599eb7  1.458463e-01  7.644506e-02  1.248203e-01  1.643127e-01   \n",
       "8421  df30d6dd  1.992619e-06  7.074945e-06  1.662818e-04  2.759178e-05   \n",
       "8422  18af95b1  9.999552e-01  9.999183e-01  9.999791e-01  9.999415e-01   \n",
       "8423  27d788c8  2.921648e-01  6.933905e-01  9.868053e-01  5.997514e-01   \n",
       "\n",
       "      lb0.2168.csv  mean-(2)(4)lb0.1717.csv  \n",
       "0     2.721015e-03             8.815090e-02  \n",
       "1     4.024012e-01             4.446322e-01  \n",
       "2     6.370236e-12             3.539300e-10  \n",
       "3     9.999939e-01             9.999878e-01  \n",
       "4     2.120113e-04             3.209088e-03  \n",
       "5     7.421116e-04             2.790584e-02  \n",
       "6     1.918441e-02             1.444582e-02  \n",
       "7     9.999989e-01             9.999560e-01  \n",
       "8     3.130832e-06             3.906417e-09  \n",
       "9     1.297212e-08             4.328927e-10  \n",
       "10    1.552837e-09             5.762643e-09  \n",
       "11    9.596787e-02             7.891672e-02  \n",
       "12    6.403032e-05             3.526576e-04  \n",
       "13    1.974677e-01             3.408813e-01  \n",
       "14    8.291981e-06             1.039578e-04  \n",
       "15    9.986849e-05             1.645820e-03  \n",
       "16    5.155202e-05             4.690950e-04  \n",
       "17    1.591020e-03             7.785125e-02  \n",
       "18    3.841120e-02             2.930468e-01  \n",
       "19    9.808260e-01             9.902107e-01  \n",
       "20    2.770799e-05             7.274421e-04  \n",
       "21    2.274429e-01             3.658111e-01  \n",
       "22    6.434636e-02             6.154930e-02  \n",
       "23    2.952904e-03             1.790067e-03  \n",
       "24    2.361504e-11             1.654060e-15  \n",
       "25    1.752715e-09             9.518749e-08  \n",
       "26    1.867912e-03             4.206313e-03  \n",
       "27    3.261114e-05             1.315730e-02  \n",
       "28    3.825938e-04             3.722554e-03  \n",
       "29    9.999965e-01             9.999979e-01  \n",
       "...            ...                      ...  \n",
       "8394  2.530175e-07             5.571086e-07  \n",
       "8395  2.433944e-08             2.389352e-09  \n",
       "8396  3.911817e-03             2.362077e-02  \n",
       "8397  4.659027e-01             5.016388e-01  \n",
       "8398  2.239254e-01             2.848360e-01  \n",
       "8399  2.731834e-10             9.486391e-09  \n",
       "8400  1.159579e-05             3.541269e-05  \n",
       "8401  2.051471e-06             5.302218e-04  \n",
       "8402  3.026948e-01             3.259458e-01  \n",
       "8403  2.460779e-02             9.331551e-02  \n",
       "8404  5.775682e-01             3.672767e-01  \n",
       "8405  2.448407e-06             1.260926e-05  \n",
       "8406  6.845485e-08             4.148419e-05  \n",
       "8407  4.939414e-06             2.563609e-04  \n",
       "8408  5.274712e-02             2.519771e-02  \n",
       "8409  9.999996e-01             9.999975e-01  \n",
       "8410  1.209952e-11             2.170930e-12  \n",
       "8411  1.091372e-02             3.269133e-02  \n",
       "8412  9.607627e-01             8.621675e-01  \n",
       "8413  3.654592e-04             5.304924e-02  \n",
       "8414  2.121196e-07             9.440746e-05  \n",
       "8415  4.858482e-03             1.571365e-01  \n",
       "8416  2.032915e-19             7.301838e-18  \n",
       "8417  6.733650e-04             2.336607e-03  \n",
       "8418  9.682497e-01             9.405996e-01  \n",
       "8419  2.518948e-08             2.518590e-08  \n",
       "8420  3.723660e-02             1.405674e-01  \n",
       "8421  2.788007e-05             1.539492e-03  \n",
       "8422  9.999895e-01             9.896075e-01  \n",
       "8423  2.076232e-01             2.555270e-01  \n",
       "\n",
       "[8424 rows x 7 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sublist = os.listdir(path)\n",
    "test_df = test_data[['id']]\n",
    "for i in sublist:\n",
    "    temp = pd.read_csv(os.path.join(path,i))\n",
    "    test_df[i] = temp['is_iceberg']\n",
    "test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_sub_mmm(test_df):\n",
    "    prediction = test_df\n",
    "    prediction['is_iceberg'] = np.where(np.all(prediction[prediction.columns[1:]]>0.5,axis = 1),\n",
    "                                    np.max(prediction[prediction.columns[1:]],axis = 1),\n",
    "                                     np.where(np.all(prediction[prediction.columns[1:]]<0.5,axis = 1),np.min(prediction[prediction.columns[1:]],axis = 1),prediction['lb0.1661.csv']))\n",
    "    sub = prediction[['id','is_iceberg']]\n",
    "    sub.to_csv('sub.csv',index = False)\n",
    "    return sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"\n"
     ]
    }
   ],
   "source": [
    "sub = get_sub_mmm(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>is_iceberg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5941774d</td>\n",
       "      <td>2.721015e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4023181e</td>\n",
       "      <td>3.974485e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>b20200e4</td>\n",
       "      <td>2.426515e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e7f018bb</td>\n",
       "      <td>9.999995e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4371c8c3</td>\n",
       "      <td>7.653040e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>a8d9b1fd</td>\n",
       "      <td>7.421116e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>29e7727e</td>\n",
       "      <td>1.444582e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>92a51ffb</td>\n",
       "      <td>9.999992e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>c769ac97</td>\n",
       "      <td>2.814740e-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>aee0547d</td>\n",
       "      <td>1.078889e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>565b28ac</td>\n",
       "      <td>6.722855e-13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>e04e9775</td>\n",
       "      <td>1.819807e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>8e8161d1</td>\n",
       "      <td>3.857386e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>4cf4d256</td>\n",
       "      <td>3.473218e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>139e5324</td>\n",
       "      <td>5.397478e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>f156976f</td>\n",
       "      <td>2.046762e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>68a117cc</td>\n",
       "      <td>1.148563e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>d9aa7a56</td>\n",
       "      <td>1.591020e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>9005b143</td>\n",
       "      <td>3.735686e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>5f6d3988</td>\n",
       "      <td>9.995149e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>9ad70954</td>\n",
       "      <td>2.501507e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>b9087b9e</td>\n",
       "      <td>4.356706e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>a39a1427</td>\n",
       "      <td>6.154930e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>82fbe8ed</td>\n",
       "      <td>1.790067e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1fae4879</td>\n",
       "      <td>2.513818e-22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>6dd8f13d</td>\n",
       "      <td>1.074130e-13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>bbad5958</td>\n",
       "      <td>2.905246e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>54527583</td>\n",
       "      <td>1.223689e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>be8fa29c</td>\n",
       "      <td>1.088336e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>81a3328f</td>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8394</th>\n",
       "      <td>8ae30ce6</td>\n",
       "      <td>1.970363e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8395</th>\n",
       "      <td>de27ed88</td>\n",
       "      <td>3.010890e-19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8396</th>\n",
       "      <td>66d5196f</td>\n",
       "      <td>3.911817e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8397</th>\n",
       "      <td>d85f1858</td>\n",
       "      <td>4.493008e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8398</th>\n",
       "      <td>16dcb33a</td>\n",
       "      <td>2.501919e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8399</th>\n",
       "      <td>eca3158e</td>\n",
       "      <td>2.643913e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8400</th>\n",
       "      <td>08daeee6</td>\n",
       "      <td>6.011877e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8401</th>\n",
       "      <td>e9c513ee</td>\n",
       "      <td>3.501544e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8402</th>\n",
       "      <td>b1519fa6</td>\n",
       "      <td>2.848104e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8403</th>\n",
       "      <td>dfc89540</td>\n",
       "      <td>2.460779e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8404</th>\n",
       "      <td>8fd8c0e9</td>\n",
       "      <td>4.552343e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8405</th>\n",
       "      <td>45df6347</td>\n",
       "      <td>5.201120e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8406</th>\n",
       "      <td>bf7928d7</td>\n",
       "      <td>3.252151e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8407</th>\n",
       "      <td>7b587c05</td>\n",
       "      <td>4.449146e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8408</th>\n",
       "      <td>c2834388</td>\n",
       "      <td>2.519771e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8409</th>\n",
       "      <td>146143c3</td>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8410</th>\n",
       "      <td>d59aee00</td>\n",
       "      <td>2.073761e-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8411</th>\n",
       "      <td>cbc0b93b</td>\n",
       "      <td>1.091372e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8412</th>\n",
       "      <td>088e2ff7</td>\n",
       "      <td>9.607627e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8413</th>\n",
       "      <td>673d33cd</td>\n",
       "      <td>3.654592e-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8414</th>\n",
       "      <td>674b031e</td>\n",
       "      <td>2.784772e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8415</th>\n",
       "      <td>43db4207</td>\n",
       "      <td>4.858482e-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8416</th>\n",
       "      <td>156855e1</td>\n",
       "      <td>3.743123e-36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8417</th>\n",
       "      <td>ac96cfb0</td>\n",
       "      <td>8.186708e-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8418</th>\n",
       "      <td>fe45aef5</td>\n",
       "      <td>9.762937e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8419</th>\n",
       "      <td>16ee9b50</td>\n",
       "      <td>2.501841e-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8420</th>\n",
       "      <td>5a599eb7</td>\n",
       "      <td>3.723660e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8421</th>\n",
       "      <td>df30d6dd</td>\n",
       "      <td>1.992619e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8422</th>\n",
       "      <td>18af95b1</td>\n",
       "      <td>9.999895e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8423</th>\n",
       "      <td>27d788c8</td>\n",
       "      <td>2.921648e-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8424 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            id    is_iceberg\n",
       "0     5941774d  2.721015e-03\n",
       "1     4023181e  3.974485e-01\n",
       "2     b20200e4  2.426515e-14\n",
       "3     e7f018bb  9.999995e-01\n",
       "4     4371c8c3  7.653040e-05\n",
       "5     a8d9b1fd  7.421116e-04\n",
       "6     29e7727e  1.444582e-02\n",
       "7     92a51ffb  9.999992e-01\n",
       "8     c769ac97  2.814740e-12\n",
       "9     aee0547d  1.078889e-14\n",
       "10    565b28ac  6.722855e-13\n",
       "11    e04e9775  1.819807e-01\n",
       "12    8e8161d1  3.857386e-05\n",
       "13    4cf4d256  3.473218e-01\n",
       "14    139e5324  5.397478e-08\n",
       "15    f156976f  2.046762e-05\n",
       "16    68a117cc  1.148563e-05\n",
       "17    d9aa7a56  1.591020e-03\n",
       "18    9005b143  3.735686e-02\n",
       "19    5f6d3988  9.995149e-01\n",
       "20    9ad70954  2.501507e-05\n",
       "21    b9087b9e  4.356706e-01\n",
       "22    a39a1427  6.154930e-02\n",
       "23    82fbe8ed  1.790067e-03\n",
       "24    1fae4879  2.513818e-22\n",
       "25    6dd8f13d  1.074130e-13\n",
       "26    bbad5958  2.905246e-04\n",
       "27    54527583  1.223689e-05\n",
       "28    be8fa29c  1.088336e-04\n",
       "29    81a3328f  1.000000e+00\n",
       "...        ...           ...\n",
       "8394  8ae30ce6  1.970363e-07\n",
       "8395  de27ed88  3.010890e-19\n",
       "8396  66d5196f  3.911817e-03\n",
       "8397  d85f1858  4.493008e-01\n",
       "8398  16dcb33a  2.501919e-01\n",
       "8399  eca3158e  2.643913e-14\n",
       "8400  08daeee6  6.011877e-08\n",
       "8401  e9c513ee  3.501544e-08\n",
       "8402  b1519fa6  2.848104e-01\n",
       "8403  dfc89540  2.460779e-02\n",
       "8404  8fd8c0e9  4.552343e-01\n",
       "8405  45df6347  5.201120e-08\n",
       "8406  bf7928d7  3.252151e-09\n",
       "8407  7b587c05  4.449146e-06\n",
       "8408  c2834388  2.519771e-02\n",
       "8409  146143c3  1.000000e+00\n",
       "8410  d59aee00  2.073761e-20\n",
       "8411  cbc0b93b  1.091372e-02\n",
       "8412  088e2ff7  9.607627e-01\n",
       "8413  673d33cd  3.654592e-04\n",
       "8414  674b031e  2.784772e-08\n",
       "8415  43db4207  4.858482e-03\n",
       "8416  156855e1  3.743123e-36\n",
       "8417  ac96cfb0  8.186708e-05\n",
       "8418  fe45aef5  9.762937e-01\n",
       "8419  16ee9b50  2.501841e-14\n",
       "8420  5a599eb7  3.723660e-02\n",
       "8421  df30d6dd  1.992619e-06\n",
       "8422  18af95b1  9.999895e-01\n",
       "8423  27d788c8  2.921648e-01\n",
       "\n",
       "[8424 rows x 2 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}