Model 6-Advanced CNN with DA and self TL(cv3).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "from subprocess import check_output\n",
    "#print(check_output([\"ls\", \"../input/\"]))\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import log_loss\n",
    "from os.path import join as opj\n",
    "from matplotlib import pyplot as plt\n",
    "import tensorflow as tf\n",
    "import os\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from matplotlib import pyplot\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Input, Flatten, Activation,Add, ZeroPadding2D ,AveragePooling2D\n",
    "from keras.layers import GlobalMaxPooling2D\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.layers.merge import Concatenate\n",
    "from keras.models import Model\n",
    "from keras import initializers,regularizers\n",
    "from keras.optimizers import Adam,SGD\n",
    "from keras.initializers import glorot_uniform\n",
    "from keras.callbacks import ModelCheckpoint, Callback, EarlyStopping\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "import keras.backend as K\n",
    "K.set_image_data_format('channels_last')\n",
    "#K.set_learning_phase(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train = pd.read_json(\"input/train.json\")\n",
    "#test = pd.read_json(\"input/test.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "only_band1 = ['band1_'+str(i) for i in range(75*75)]\n",
    "only_band2 = ['band2_'+str(i) for i in range(75*75)]\n",
    "both_bands = only_band1+only_band2\n",
    "df = pd.concat([pd.DataFrame(test['band_1'].values.tolist(),columns = only_band1),\n",
    "                pd.DataFrame(test['band_2'].values.tolist(),columns = only_band2)],axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def collect_test():\n",
    "    test_data = pd.DataFrame()\n",
    "    for i in range(4):\n",
    "        test_piece = pd.read_csv('input/test_subparts/test'+str(i+1)+'.csv')\n",
    "        test_data = pd.concat([test_data,test_piece])\n",
    "        test_data = test_data.reset_index().drop(['index'],axis = 1)\n",
    "    return test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data = collect_test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_band_1=np.array([np.array(band).astype(np.float32).reshape(75, 75) for band in train[\"band_1\"]])\n",
    "X_band_2=np.array([np.array(band).astype(np.float32).reshape(75, 75) for band in train[\"band_2\"]])\n",
    "X_train = np.concatenate([X_band_1[:, :, :, np.newaxis], X_band_2[:, :, :, np.newaxis],((X_band_1+X_band_2)/2)[:, :, :, np.newaxis]], axis=-1)\n",
    "target_train=train['is_iceberg']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_band_test_1=np.array([np.array(band).astype(np.float32).reshape(75, 75) for band in test[\"band_1\"]])\n",
    "X_band_test_2=np.array([np.array(band).astype(np.float32).reshape(75, 75) for band in test[\"band_2\"]])\n",
    "X_test = np.concatenate([X_band_test_1[:, :, :, np.newaxis], X_band_test_2[:, :, :, np.newaxis], ((X_band_test_1+X_band_test_2)/2)[:, :, :, np.newaxis]], axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "X_train_cv, X_valid, y_train_cv, y_valid = train_test_split(X_train, target_train, random_state=2,stratify = target_train, train_size=0.80)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cross Validation Folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_augmentation(X_input,Y_input,batch_size = 32):\n",
    "    data_aug = ImageDataGenerator(#featurewise_center=True,\n",
    "                             #featurewise_std_normalization=True,\n",
    "                             #zca_whitening=True,\n",
    "                             rotation_range=0,\n",
    "                             width_shift_range = 0,\n",
    "                             height_shift_range = 0,\n",
    "                             zoom_range = 0,\n",
    "                             data_format = 'channels_last',\n",
    "                             horizontal_flip = True,\n",
    "                             vertical_flip = True)#,fill_mode = 'constant',cval = 0)\n",
    "    data_aug_batches = data_aug.flow(X_input,Y_input,batch_size = batch_size)\n",
    "    return data_aug_batches\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def basic_cbamd_layer(X_input,filter_num =64,kernel_size = (3,3),conv_padding = 'valid',pool_size =(2,2),pool_strides = (2,2),pool_padding = 'valid',dropout_rate = 0.2):\n",
    "    X = Conv2D(filter_num, kernel_size, padding = conv_padding)(X_input)\n",
    "    #X = BatchNormalization(axis = 3)(X)\n",
    "    X = Activation('relu')(X)\n",
    "    X = MaxPooling2D(pool_size,strides = pool_strides,padding = pool_padding)(X)\n",
    "    X = Dropout(rate = dropout_rate)(X)\n",
    "    return X\n",
    "\n",
    "def dense_layer(X_input, units = 1,activation = 'relu',dropout_rate = 0.2):\n",
    "    X = Dense(units = units,activation = activation)(X_input)\n",
    "    X = Dropout(rate = dropout_rate)(X)\n",
    "    return X\n",
    "#kernel_regularizer=regularizers.l2(0.01)\n",
    "def cnnmodel(input_shape,lr = 0.0001):\n",
    "    X_input = Input(input_shape)\n",
    "    #First composite layer\n",
    "    X = basic_cbamd_layer(X_input,\n",
    "                           filter_num =64,\n",
    "                           kernel_size = (5,5),\n",
    "                           conv_padding = 'valid',\n",
    "                           pool_size =(3,3),\n",
    "                           pool_strides = (2,2),\n",
    "                           pool_padding = 'valid',\n",
    "                           dropout_rate = 0.2)\n",
    "    #Second composite layer\n",
    "    X = basic_cbamd_layer(X,\n",
    "                           filter_num =128,\n",
    "                           kernel_size = (3,3),\n",
    "                           conv_padding = 'valid',\n",
    "                           pool_size =(3,3),\n",
    "                           pool_strides = (2,2),\n",
    "                           pool_padding = 'valid',\n",
    "                           dropout_rate = 0.2)\n",
    "    #Third composite layer\n",
    "    X = basic_cbamd_layer(X,\n",
    "                           filter_num =128,\n",
    "                           kernel_size = (3,3),\n",
    "                           conv_padding = 'valid',\n",
    "                           pool_size =(2,2),\n",
    "                           pool_strides = (2,2),\n",
    "                           pool_padding = 'valid',\n",
    "                           dropout_rate = 0.2)\n",
    "    #Forth composite layer\n",
    "    X = basic_cbamd_layer(X,\n",
    "                           filter_num =64,\n",
    "                           kernel_size = (3,3),\n",
    "                           conv_padding = 'valid',\n",
    "                           pool_size =(3,3),\n",
    "                           pool_strides = (2,2),\n",
    "                           pool_padding = 'valid',\n",
    "                           dropout_rate = 0.2)\n",
    "    #Flatten layer\n",
    "    X = Flatten()(X)\n",
    "    #First dense layer\n",
    "    X = dense_layer(X, units = 256,activation = 'relu',dropout_rate = 0.2)\n",
    "    #Second dense layer\n",
    "    X = dense_layer(X, units = 128,activation = 'relu',dropout_rate = 0.2)\n",
    "    #Decision layer\n",
    "    X = dense_layer(X, units = 1,activation = 'sigmoid',dropout_rate = 0)\n",
    "    \n",
    "    model = Model(inputs=X_input,outputs=X)\n",
    "    optimizer=Adam(lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)\n",
    "    #optimizer = SGD(lr, decay=0, momentum=0.9, nesterov=True)\n",
    "    model.compile(loss='binary_crossentropy',\n",
    "                  optimizer=optimizer,\n",
    "                  metrics=['accuracy'])\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "def get_callbacks(filepath = \".model_weights.hdf5\", patience=7):\n",
    "    #es = EarlyStopping('val_loss', patience=patience, mode=\"min\")\n",
    "    msave = ModelCheckpoint(filepath, save_best_only=True)\n",
    "    return [msave]\n",
    "\n",
    "#callbacks = get_callbacks(filepath=file_path, patience=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def fitmodel(model,X_train,y_train,X_valid,y_valid,augment = False,epochs = 50,batch_size = 32,filepath = \".model_weights.hdf5\"):\n",
    "    if augment == False:\n",
    "        result = model.fit(X_train, y_train,\n",
    "                          batch_size=batch_size,\n",
    "                          epochs=epochs,\n",
    "                          verbose=1,\n",
    "                          validation_data=(X_valid, y_valid),\n",
    "                          callbacks=get_callbacks(filepath))\n",
    "    else:\n",
    "        result = model.fit_generator(data_augmentation(X_train,y_train,batch_size = batch_size),\n",
    "                          steps_per_epoch = len(X_train_cv)/batch_size,\n",
    "                          epochs=epochs,\n",
    "                          verbose=1,\n",
    "                          validation_data=(X_valid, y_valid),\n",
    "                          callbacks=get_callbacks(filepath))\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 input_13\n",
      "1 conv2d_49\n",
      "2 activation_49\n",
      "3 max_pooling2d_49\n",
      "4 dropout_85\n",
      "5 conv2d_50\n",
      "6 activation_50\n",
      "7 max_pooling2d_50\n",
      "8 dropout_86\n",
      "9 conv2d_51\n",
      "10 activation_51\n",
      "11 max_pooling2d_51\n",
      "12 dropout_87\n",
      "13 conv2d_52\n",
      "14 activation_52\n",
      "15 max_pooling2d_52\n",
      "16 dropout_88\n",
      "17 flatten_13\n",
      "18 dense_37\n",
      "19 dropout_89\n",
      "20 dense_38\n",
      "21 dropout_90\n",
      "22 dense_39\n",
      "23 dropout_91\n"
     ]
    }
   ],
   "source": [
    "for i,layer in enumerate(Cnnmodel.layers):\n",
    "    print(i,layer.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_15 (InputLayer)        (None, 75, 75, 3)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_57 (Conv2D)           (None, 71, 71, 64)        4864      \n",
      "_________________________________________________________________\n",
      "activation_57 (Activation)   (None, 71, 71, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_57 (MaxPooling (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_99 (Dropout)         (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_58 (Conv2D)           (None, 33, 33, 128)       73856     \n",
      "_________________________________________________________________\n",
      "activation_58 (Activation)   (None, 33, 33, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_58 (MaxPooling (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "dropout_100 (Dropout)        (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_59 (Conv2D)           (None, 14, 14, 128)       147584    \n",
      "_________________________________________________________________\n",
      "activation_59 (Activation)   (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_59 (MaxPooling (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "dropout_101 (Dropout)        (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_60 (Conv2D)           (None, 5, 5, 64)          73792     \n",
      "_________________________________________________________________\n",
      "activation_60 (Activation)   (None, 5, 5, 64)          0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_60 (MaxPooling (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_102 (Dropout)        (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_15 (Flatten)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_43 (Dense)             (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_103 (Dropout)        (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_44 (Dense)             (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_104 (Dropout)        (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_45 (Dense)             (None, 1)                 129       \n",
      "_________________________________________________________________\n",
      "dropout_105 (Dropout)        (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 398,913\n",
      "Trainable params: 398,913\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_15 (InputLayer)        (None, 75, 75, 3)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_57 (Conv2D)           (None, 71, 71, 64)        4864      \n",
      "_________________________________________________________________\n",
      "activation_57 (Activation)   (None, 71, 71, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_57 (MaxPooling (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_99 (Dropout)         (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_58 (Conv2D)           (None, 33, 33, 128)       73856     \n",
      "_________________________________________________________________\n",
      "activation_58 (Activation)   (None, 33, 33, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_58 (MaxPooling (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "dropout_100 (Dropout)        (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_59 (Conv2D)           (None, 14, 14, 128)       147584    \n",
      "_________________________________________________________________\n",
      "activation_59 (Activation)   (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_59 (MaxPooling (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "dropout_101 (Dropout)        (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_60 (Conv2D)           (None, 5, 5, 64)          73792     \n",
      "_________________________________________________________________\n",
      "activation_60 (Activation)   (None, 5, 5, 64)          0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_60 (MaxPooling (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_102 (Dropout)        (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_15 (Flatten)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_43 (Dense)             (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_103 (Dropout)        (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_44 (Dense)             (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_104 (Dropout)        (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_45 (Dense)             (None, 1)                 129       \n",
      "_________________________________________________________________\n",
      "dropout_105 (Dropout)        (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 398,913\n",
      "Trainable params: 394,049\n",
      "Non-trainable params: 4,864\n",
      "_________________________________________________________________\n",
      "Epoch 1/200\n",
      "41/40 [==============================] - 8s 188ms/step - loss: 0.3125 - acc: 0.8632 - val_loss: 0.2407 - val_acc: 0.8910\n",
      "Epoch 2/200\n",
      "41/40 [==============================] - 4s 94ms/step - loss: 0.2729 - acc: 0.8871 - val_loss: 0.2239 - val_acc: 0.9034\n",
      "Epoch 3/200\n",
      "41/40 [==============================] - 4s 95ms/step - loss: 0.2596 - acc: 0.8925 - val_loss: 0.2408 - val_acc: 0.8910\n",
      "Epoch 4/200\n",
      "41/40 [==============================] - 4s 98ms/step - loss: 0.2505 - acc: 0.8932 - val_loss: 0.2118 - val_acc: 0.9097\n",
      "Epoch 5/200\n",
      "41/40 [==============================] - 4s 92ms/step - loss: 0.2419 - acc: 0.8970 - val_loss: 0.2224 - val_acc: 0.8941\n",
      "Epoch 6/200\n",
      "41/40 [==============================] - 4s 97ms/step - loss: 0.2460 - acc: 0.8937 - val_loss: 0.2285 - val_acc: 0.8941\n",
      "Epoch 7/200\n",
      "41/40 [==============================] - 4s 90ms/step - loss: 0.2316 - acc: 0.9047 - val_loss: 0.2590 - val_acc: 0.8816\n",
      "Epoch 8/200\n",
      "41/40 [==============================] - 4s 106ms/step - loss: 0.2375 - acc: 0.9039 - val_loss: 0.2042 - val_acc: 0.9190: 0.903\n",
      "Epoch 9/200\n",
      "41/40 [==============================] - 4s 97ms/step - loss: 0.2312 - acc: 0.8963 - val_loss: 0.2121 - val_acc: 0.9190\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10/200\n",
      "41/40 [==============================] - 4s 95ms/step - loss: 0.2142 - acc: 0.9131 - val_loss: 0.2126 - val_acc: 0.9065\n",
      "Epoch 11/200\n",
      "41/40 [==============================] - 4s 95ms/step - loss: 0.2458 - acc: 0.9036 - val_loss: 0.2135 - val_acc: 0.9128\n",
      "Epoch 12/200\n",
      "41/40 [==============================] - 4s 100ms/step - loss: 0.2163 - acc: 0.9039 - val_loss: 0.2163 - val_acc: 0.9034\n",
      "Epoch 13/200\n",
      "41/40 [==============================] - 4s 100ms/step - loss: 0.2187 - acc: 0.9051 - val_loss: 0.2076 - val_acc: 0.9283\n",
      "Epoch 14/200\n",
      "41/40 [==============================] - 4s 92ms/step - loss: 0.2443 - acc: 0.8827 - val_loss: 0.2226 - val_acc: 0.8941\n",
      "Epoch 15/200\n",
      "41/40 [==============================] - 4s 96ms/step - loss: 0.1971 - acc: 0.9131 - val_loss: 0.2086 - val_acc: 0.9128\n",
      "Epoch 16/200\n",
      "41/40 [==============================] - 4s 98ms/step - loss: 0.2126 - acc: 0.9123 - val_loss: 0.2188 - val_acc: 0.9065\n",
      "Epoch 17/200\n",
      "41/40 [==============================] - 4s 94ms/step - loss: 0.1968 - acc: 0.9215 - val_loss: 0.2021 - val_acc: 0.9190\n",
      "Epoch 18/200\n",
      "41/40 [==============================] - 4s 98ms/step - loss: 0.1969 - acc: 0.9215 - val_loss: 0.2197 - val_acc: 0.8972\n",
      "Epoch 19/200\n",
      "41/40 [==============================] - 4s 91ms/step - loss: 0.1954 - acc: 0.9074 - val_loss: 0.2061 - val_acc: 0.9097\n",
      "Epoch 20/200\n",
      "41/40 [==============================] - 4s 99ms/step - loss: 0.2028 - acc: 0.9150 - val_loss: 0.2048 - val_acc: 0.9128\n",
      "Epoch 21/200\n",
      "41/40 [==============================] - 4s 96ms/step - loss: 0.1927 - acc: 0.9176 - val_loss: 0.2481 - val_acc: 0.9003\n",
      "Epoch 22/200\n",
      "41/40 [==============================] - 4s 96ms/step - loss: 0.1938 - acc: 0.9150 - val_loss: 0.2180 - val_acc: 0.9097\n",
      "Epoch 23/200\n",
      "41/40 [==============================] - 4s 97ms/step - loss: 0.1803 - acc: 0.9265 - val_loss: 0.2343 - val_acc: 0.9003\n",
      "Epoch 24/200\n",
      "41/40 [==============================] - 4s 92ms/step - loss: 0.1867 - acc: 0.9222 - val_loss: 0.2353 - val_acc: 0.8816\n",
      "Epoch 25/200\n",
      "41/40 [==============================] - 4s 100ms/step - loss: 0.2067 - acc: 0.9199 - val_loss: 0.2051 - val_acc: 0.9065\n",
      "Epoch 26/200\n",
      "41/40 [==============================] - 4s 106ms/step - loss: 0.1942 - acc: 0.9260 - val_loss: 0.2470 - val_acc: 0.8879\n",
      "Epoch 27/200\n",
      "41/40 [==============================] - 4s 106ms/step - loss: 0.1931 - acc: 0.9245 - val_loss: 0.2027 - val_acc: 0.9221\n",
      "Epoch 28/200\n",
      "41/40 [==============================] - 4s 96ms/step - loss: 0.1960 - acc: 0.9207 - val_loss: 0.2155 - val_acc: 0.9065\n",
      "Epoch 29/200\n",
      "41/40 [==============================] - 4s 93ms/step - loss: 0.1729 - acc: 0.9260 - val_loss: 0.2102 - val_acc: 0.9065\n",
      "Epoch 30/200\n",
      "41/40 [==============================] - 4s 99ms/step - loss: 0.2088 - acc: 0.9074 - val_loss: 0.2314 - val_acc: 0.8816\n",
      "Epoch 31/200\n",
      "41/40 [==============================] - 4s 93ms/step - loss: 0.1798 - acc: 0.9283 - val_loss: 0.2015 - val_acc: 0.9097\n",
      "Epoch 32/200\n",
      "41/40 [==============================] - 4s 100ms/step - loss: 0.2194 - acc: 0.9051 - val_loss: 0.1986 - val_acc: 0.9190\n",
      "Epoch 33/200\n",
      "41/40 [==============================] - 4s 92ms/step - loss: 0.1856 - acc: 0.9230 - val_loss: 0.2193 - val_acc: 0.9003\n",
      "Epoch 34/200\n",
      "41/40 [==============================] - 4s 99ms/step - loss: 0.1798 - acc: 0.9222 - val_loss: 0.2067 - val_acc: 0.9097\n",
      "Epoch 35/200\n",
      "41/40 [==============================] - 4s 98ms/step - loss: 0.1851 - acc: 0.9143 - val_loss: 0.2110 - val_acc: 0.9190\n",
      "Epoch 36/200\n",
      "41/40 [==============================] - 4s 94ms/step - loss: 0.1929 - acc: 0.9181 - val_loss: 0.2004 - val_acc: 0.9128\n",
      "Epoch 37/200\n",
      "41/40 [==============================] - 4s 99ms/step - loss: 0.1730 - acc: 0.9283 - val_loss: 0.1994 - val_acc: 0.9190\n",
      "Epoch 38/200\n",
      "41/40 [==============================] - 4s 92ms/step - loss: 0.1899 - acc: 0.9265 - val_loss: 0.2026 - val_acc: 0.9221\n",
      "Epoch 39/200\n",
      "41/40 [==============================] - 4s 101ms/step - loss: 0.1752 - acc: 0.9306 - val_loss: 0.1995 - val_acc: 0.9252\n",
      "Epoch 40/200\n",
      "41/40 [==============================] - 4s 95ms/step - loss: 0.1869 - acc: 0.9283 - val_loss: 0.1918 - val_acc: 0.9159\n",
      "Epoch 41/200\n",
      "41/40 [==============================] - 4s 98ms/step - loss: 0.1878 - acc: 0.9245 - val_loss: 0.1921 - val_acc: 0.9190\n",
      "Epoch 42/200\n",
      "41/40 [==============================] - 4s 101ms/step - loss: 0.1690 - acc: 0.9367 - val_loss: 0.1916 - val_acc: 0.9128\n",
      "Epoch 43/200\n",
      "41/40 [==============================] - 4s 95ms/step - loss: 0.1802 - acc: 0.9237 - val_loss: 0.2053 - val_acc: 0.9128\n",
      "Epoch 44/200\n",
      "41/40 [==============================] - 4s 100ms/step - loss: 0.1666 - acc: 0.9352 - val_loss: 0.1921 - val_acc: 0.9221s: 0.1704 -\n",
      "Epoch 45/200\n",
      "41/40 [==============================] - 4s 92ms/step - loss: 0.1894 - acc: 0.9165 - val_loss: 0.1965 - val_acc: 0.9190\n",
      "Epoch 46/200\n",
      "41/40 [==============================] - 4s 102ms/step - loss: 0.1688 - acc: 0.9291 - val_loss: 0.2078 - val_acc: 0.9159s: 0.17\n",
      "Epoch 47/200\n",
      "41/40 [==============================] - 4s 100ms/step - loss: 0.1793 - acc: 0.9199 - val_loss: 0.1915 - val_acc: 0.9221\n",
      "Epoch 48/200\n",
      "41/40 [==============================] - 4s 96ms/step - loss: 0.1634 - acc: 0.9314 - val_loss: 0.2036 - val_acc: 0.9159A: 2\n",
      "Epoch 49/200\n",
      "41/40 [==============================] - 4s 101ms/step - loss: 0.1607 - acc: 0.9352 - val_loss: 0.1985 - val_acc: 0.9252\n",
      "Epoch 50/200\n",
      "41/40 [==============================] - 4s 93ms/step - loss: 0.1704 - acc: 0.9253 - val_loss: 0.2063 - val_acc: 0.9159\n",
      "Epoch 51/200\n",
      "41/40 [==============================] - 4s 101ms/step - loss: 0.1649 - acc: 0.9257 - val_loss: 0.2129 - val_acc: 0.9283 1s - loss:\n",
      "Epoch 52/200\n",
      "41/40 [==============================] - 4s 95ms/step - loss: 0.1685 - acc: 0.9226 - val_loss: 0.1903 - val_acc: 0.9252\n",
      "Epoch 53/200\n",
      "41/40 [==============================] - 4s 103ms/step - loss: 0.1575 - acc: 0.9405 - val_loss: 0.1957 - val_acc: 0.9159: 2s - loss:\n",
      "Epoch 54/200\n",
      "41/40 [==============================] - 4s 103ms/step - loss: 0.1525 - acc: 0.9253 - val_loss: 0.2017 - val_acc: 0.9159\n",
      "Epoch 55/200\n",
      "41/40 [==============================] - 5s 113ms/step - loss: 0.1608 - acc: 0.9329 - val_loss: 0.2021 - val_acc: 0.9159\n",
      "Epoch 56/200\n",
      "41/40 [==============================] - 4s 103ms/step - loss: 0.1875 - acc: 0.9181 - val_loss: 0.2063 - val_acc: 0.9065\n",
      "Epoch 57/200\n",
      "41/40 [==============================] - 4s 92ms/step - loss: 0.1538 - acc: 0.9337 - val_loss: 0.2017 - val_acc: 0.9221\n",
      "Epoch 58/200\n",
      "41/40 [==============================] - 4s 104ms/step - loss: 0.1411 - acc: 0.9344 - val_loss: 0.1917 - val_acc: 0.9408\n",
      "Epoch 59/200\n",
      "41/40 [==============================] - 4s 99ms/step - loss: 0.1685 - acc: 0.9314 - val_loss: 0.1848 - val_acc: 0.9377\n",
      "Epoch 60/200\n",
      "41/40 [==============================] - 4s 97ms/step - loss: 0.1598 - acc: 0.9390 - val_loss: 0.1915 - val_acc: 0.9315\n",
      "Epoch 61/200\n",
      "41/40 [==============================] - 4s 104ms/step - loss: 0.1553 - acc: 0.9398 - val_loss: 0.1824 - val_acc: 0.9346loss: 0.1498 - acc: 0.94\n",
      "Epoch 62/200\n",
      "41/40 [==============================] - 4s 103ms/step - loss: 0.1517 - acc: 0.9367 - val_loss: 0.2105 - val_acc: 0.9159\n",
      "Epoch 63/200\n",
      "41/40 [==============================] - 4s 108ms/step - loss: 0.1577 - acc: 0.9420 - val_loss: 0.1976 - val_acc: 0.9252\n",
      "Epoch 64/200\n",
      "41/40 [==============================] - 4s 100ms/step - loss: 0.1597 - acc: 0.9359 - val_loss: 0.1916 - val_acc: 0.9315\n",
      "Epoch 65/200\n",
      "41/40 [==============================] - 4s 97ms/step - loss: 0.1481 - acc: 0.9398 - val_loss: 0.1933 - val_acc: 0.9252\n",
      "Epoch 66/200\n",
      "41/40 [==============================] - 4s 99ms/step - loss: 0.1766 - acc: 0.9249 - val_loss: 0.2430 - val_acc: 0.9003\n",
      "Epoch 67/200\n",
      "41/40 [==============================] - 4s 98ms/step - loss: 0.1705 - acc: 0.9314 - val_loss: 0.2231 - val_acc: 0.9065\n",
      "Epoch 68/200\n",
      "41/40 [==============================] - 4s 104ms/step - loss: 0.1453 - acc: 0.9428 - val_loss: 0.2106 - val_acc: 0.9221\n",
      "Epoch 69/200\n",
      "41/40 [==============================] - 4s 105ms/step - loss: 0.1415 - acc: 0.9466 - val_loss: 0.2141 - val_acc: 0.9128\n",
      "Epoch 70/200\n",
      "41/40 [==============================] - 4s 98ms/step - loss: 0.1518 - acc: 0.9359 - val_loss: 0.2045 - val_acc: 0.9128\n",
      "Epoch 71/200\n",
      "41/40 [==============================] - 4s 106ms/step - loss: 0.1457 - acc: 0.9390 - val_loss: 0.1964 - val_acc: 0.9190\n",
      "Epoch 72/200\n",
      "41/40 [==============================] - 4s 97ms/step - loss: 0.1493 - acc: 0.9428 - val_loss: 0.2135 - val_acc: 0.9159\n",
      "Epoch 73/200\n",
      "41/40 [==============================] - 4s 101ms/step - loss: 0.1435 - acc: 0.9398 - val_loss: 0.2022 - val_acc: 0.9315s\n",
      "Epoch 74/200\n",
      "41/40 [==============================] - 4s 89ms/step - loss: 0.1418 - acc: 0.9379 - val_loss: 0.2067 - val_acc: 0.9159\n",
      "Epoch 75/200\n",
      "41/40 [==============================] - 5s 115ms/step - loss: 0.1385 - acc: 0.9436 - val_loss: 0.2391 - val_acc: 0.9003oss: 0.1605  - ETA: 0s - loss: 0.1581 - a\n",
      "Epoch 76/200\n",
      "41/40 [==============================] - 5s 112ms/step - loss: 0.1425 - acc: 0.9428 - val_loss: 0.2073 - val_acc: 0.9221\n",
      "Epoch 77/200\n",
      "41/40 [==============================] - 4s 91ms/step - loss: 0.1345 - acc: 0.9497 - val_loss: 0.2235 - val_acc: 0.9065\n",
      "Epoch 78/200\n",
      "41/40 [==============================] - 4s 99ms/step - loss: 0.1498 - acc: 0.9326 - val_loss: 0.2063 - val_acc: 0.9252\n",
      "Epoch 79/200\n",
      "41/40 [==============================] - 4s 95ms/step - loss: 0.1334 - acc: 0.9466 - val_loss: 0.1985 - val_acc: 0.9252\n",
      "Epoch 80/200\n",
      "41/40 [==============================] - 4s 94ms/step - loss: 0.1314 - acc: 0.9474 - val_loss: 0.1938 - val_acc: 0.9252\n",
      "Epoch 81/200\n",
      "41/40 [==============================] - 4s 99ms/step - loss: 0.1479 - acc: 0.9326 - val_loss: 0.2263 - val_acc: 0.9097\n",
      "Epoch 82/200\n",
      "41/40 [==============================] - 4s 92ms/step - loss: 0.1512 - acc: 0.9359 - val_loss: 0.2290 - val_acc: 0.9221\n",
      "Epoch 83/200\n",
      "41/40 [==============================] - 4s 96ms/step - loss: 0.1421 - acc: 0.9398 - val_loss: 0.2114 - val_acc: 0.9283\n",
      "Epoch 84/200\n",
      "41/40 [==============================] - 4s 95ms/step - loss: 0.1262 - acc: 0.9466 - val_loss: 0.2267 - val_acc: 0.9221\n",
      "Epoch 85/200\n",
      "41/40 [==============================] - 4s 96ms/step - loss: 0.1410 - acc: 0.9420 - val_loss: 0.2058 - val_acc: 0.9283\n",
      "Epoch 86/200\n",
      "41/40 [==============================] - 4s 98ms/step - loss: 0.1396 - acc: 0.9398 - val_loss: 0.2388 - val_acc: 0.9128\n",
      "Epoch 87/200\n",
      "41/40 [==============================] - 4s 95ms/step - loss: 0.1307 - acc: 0.9436 - val_loss: 0.2132 - val_acc: 0.9252\n",
      "Epoch 88/200\n",
      "41/40 [==============================] - 4s 107ms/step - loss: 0.1269 - acc: 0.9497 - val_loss: 0.2138 - val_acc: 0.9252\n",
      "Epoch 89/200\n",
      "41/40 [==============================] - 5s 110ms/step - loss: 0.1334 - acc: 0.9512 - val_loss: 0.2229 - val_acc: 0.9221\n",
      "Epoch 90/200\n",
      "41/40 [==============================] - 4s 103ms/step - loss: 0.1273 - acc: 0.9489 - val_loss: 0.2247 - val_acc: 0.9190\n",
      "Epoch 91/200\n",
      " 8/40 [====>.........................] - ETA: 2s - loss: 0.0935 - acc: 0.9648"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-36-a23858d9cabe>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      7\u001b[0m                   metrics=['accuracy'])\n\u001b[0;32m      8\u001b[0m \u001b[0mCnnmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msummary\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m \u001b[0mresult_Cnn\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfitmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCnnmodel\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mX_train_cv\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my_train_cv\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mX_valid\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my_valid\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0maugment\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mepochs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m200\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mbatch_size\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m32\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mfilepath\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'model save/Model 1-Advanced CNN with DA and CV5/2018.1.1.hdf5'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-7-99fcdf6c5fde>\u001b[0m in \u001b[0;36mfitmodel\u001b[1;34m(model, X_train, y_train, X_valid, y_valid, augment, epochs, batch_size, filepath)\u001b[0m\n\u001b[0;32m     14\u001b[0m                           \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m                           \u001b[0mvalidation_data\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_valid\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_valid\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 16\u001b[1;33m                           callbacks=get_callbacks(filepath))\n\u001b[0m\u001b[0;32m     17\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\keras\\legacy\\interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     85\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[0;32m     86\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[1;32m---> 87\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     88\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     89\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[1;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[0;32m   2119\u001b[0m                         \u001b[0mbatch_logs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0ml\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mo\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2120\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2121\u001b[1;33m                     \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2122\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2123\u001b[0m                     \u001b[1;31m# Construct epoch logs.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\keras\\callbacks.py\u001b[0m in \u001b[0;36mon_batch_end\u001b[1;34m(self, batch, logs)\u001b[0m\n\u001b[0;32m    107\u001b[0m         \u001b[0mt_before_callbacks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    108\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 109\u001b[1;33m             \u001b[0mcallback\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    110\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_delta_ts_batch_end\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mt_before_callbacks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    111\u001b[0m         \u001b[0mdelta_t_median\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmedian\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_delta_ts_batch_end\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\keras\\callbacks.py\u001b[0m in \u001b[0;36mon_batch_end\u001b[1;34m(self, batch, logs)\u001b[0m\n\u001b[0;32m    296\u001b[0m         \u001b[1;31m# will be handled by on_epoch_end.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    297\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mverbose\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mseen\u001b[0m \u001b[1;33m<\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtarget\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 298\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprogbar\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mseen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlog_values\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    299\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    300\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mon_epoch_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\keras\\utils\\generic_utils.py\u001b[0m in \u001b[0;36mupdate\u001b[1;34m(self, current, values, force)\u001b[0m\n\u001b[0;32m    368\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    369\u001b[0m             \u001b[0msys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstdout\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minfo\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 370\u001b[1;33m             \u001b[0msys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstdout\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mflush\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    371\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    372\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mverbose\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\ipykernel\\iostream.py\u001b[0m in \u001b[0;36mflush\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    316\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpub_thread\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mthread\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_alive\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    317\u001b[0m             \u001b[1;31m# wait for flush to actually get through:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 318\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpub_thread\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mschedule\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_flush\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    319\u001b[0m             \u001b[0mevt\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mthreading\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mEvent\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    320\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpub_thread\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mschedule\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mevt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\ipykernel\\iostream.py\u001b[0m in \u001b[0;36mschedule\u001b[1;34m(self, f)\u001b[0m\n\u001b[0;32m    188\u001b[0m                 \u001b[0mevent_id\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0murandom\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m16\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    189\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_events\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mevent_id\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 190\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_event_pipe\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mevent_id\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    191\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    192\u001b[0m             \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mzmq\\backend\\cython\\socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket.Socket.send\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mzmq\\backend\\cython\\socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket.Socket.send\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mzmq\\backend\\cython\\socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket._send_copy\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\zmq\\backend\\cython\\checkrc.pxd\u001b[0m in \u001b[0;36mzmq.backend.cython.checkrc._check_rc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "Cnnmodel=cnnmodel(input_shape = (75,75,3),lr = 0.0001)\n",
    "Cnnmodel.load_weights('model save/Model 1-Advanced CNN with DA and CV5/2018.1.1_local0.2069OF.hdf5')\n",
    "for layer in Cnnmodel.layers[:5]:\n",
    "    layer.trainable = False\n",
    "Cnnmodel.compile(loss='binary_crossentropy',\n",
    "                  optimizer=Adam(0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0),\n",
    "                  metrics=['accuracy'])\n",
    "Cnnmodel.summary()\n",
    "result_Cnn = fitmodel(Cnnmodel,X_train_cv,y_train_cv,X_valid,y_valid,augment = True,epochs = 200,batch_size = 32,filepath = 'model save/Model 1-Advanced CNN with DA and CV5/2018.1.1.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "321/321 [==============================] - 0s 1ms/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.18129999453858855, 0.94392523364485981]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Cnnmodel.load_weights('model save/Model 1-Advanced CNN with DA and CV5/2018.1.1.hdf5')\n",
    "Cnnmodel.evaluate(X_valid, y_valid)\n",
    "#Cnnmodel.evaluate(X_train_cv, y_train_cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_history(result,name = 'fit_history.csv',path = os.getcwd()):\n",
    "    train_loss = result.history['loss']\n",
    "    val_loss = result.history['val_loss']\n",
    "    train_accuracy = result.history['acc']\n",
    "    val_accuracy = result.history['val_acc']\n",
    "    fit_history = pd.DataFrame({'train_loss':train_loss,'val_loss':val_loss,'train_accuracy':train_accuracy,'val_accuracy':val_accuracy})\n",
    "    fit_history.to_csv(os.path.join(path,name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'result_Cnn' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-137-0f752ee40c5e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0msave_history\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult_Cnn\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'result_Cnn' is not defined"
     ]
    }
   ],
   "source": [
    "save_history(result_Cnn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Model6_CV(X_train,K=3):\n",
    "    folds = list(StratifiedKFold(n_splits=K, shuffle=True,random_state = 2).split(X_train, target_train))\n",
    "    y_train_pred_log=0\n",
    "    y_valid_pred_log = 0.0*target_train\n",
    "    for j, (train_idx, valid_idx) in enumerate(folds):\n",
    "        print('\\n===================FOLD=',j+1)\n",
    "        X_train_cv = X_train[train_idx]\n",
    "        y_train_cv = target_train[train_idx]\n",
    "        X_valid = X_train[valid_idx]\n",
    "        y_valid= target_train[valid_idx]\n",
    "        \n",
    "        #Angle\n",
    "#         X_angle_cv=X_angle[train_idx]\n",
    "#         X_angle_hold=X_angle[test_idx]\n",
    "\n",
    "        #define file path and get callbacks\n",
    "        file_path = 'model save/Model 6-Advanced CNN with DA and self TL(cv3)/Model-6 cv'+str(j+1)+'.hdf5'\n",
    "        Cnnmodel=cnnmodel(input_shape = (75,75,3),lr = 0.0001)\n",
    "        result_Cnn = fitmodel(Cnnmodel,X_train_cv,y_train_cv,X_valid,y_valid,\n",
    "                              augment = False,epochs = 100,batch_size = 32,\n",
    "                              filepath = file_path)\n",
    "        print('Satge 1,done!')\n",
    "        Cnnmodel=cnnmodel(input_shape = (75,75,3),lr = 0.0001)\n",
    "        Cnnmodel.load_weights(file_path)\n",
    "        for layer in Cnnmodel.layers[:5]:\n",
    "            layer.trainable = False\n",
    "        Cnnmodel.compile(loss='binary_crossentropy',\n",
    "                  optimizer=Adam(0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0),\n",
    "                  metrics=['accuracy'])\n",
    "        Cnnmodel.summary()\n",
    "        result_Cnn = fitmodel(Cnnmodel,X_train_cv,y_train_cv,X_valid,y_valid,\n",
    "                              augment = True,epochs = 150,batch_size = 32,\n",
    "                              filepath = file_path)\n",
    "        print('Satge 2,done!')\n",
    "#         callbacks = get_callbacks(filepath=file_path, patience=5)\n",
    "#         gen_flow = gen_flow_for_two_inputs(X_train_cv, X_angle_cv, y_train_cv)\n",
    "#         galaxyModel= getVggAngleModel()\n",
    "#         galaxyModel.fit_generator(\n",
    "#                 gen_flow,\n",
    "#                 steps_per_epoch=24,\n",
    "#                 epochs=100,\n",
    "#                 shuffle=True,\n",
    "#                 verbose=1,\n",
    "#                 validation_data=([X_holdout,X_angle_hold], Y_holdout),\n",
    "#                 callbacks=callbacks)\n",
    "\n",
    "        #Getting the Best Model\n",
    "        Cnnmodel.load_weights(filepath=file_path)\n",
    "        #Getting Training Score\n",
    "        score = Cnnmodel.evaluate(X_train_cv, y_train_cv)\n",
    "        print('Train loss:', score[0])\n",
    "        print('Train accuracy:', score[1])\n",
    "        #Getting Test Score\n",
    "        score = Cnnmodel.evaluate(X_valid, y_valid)\n",
    "        print('Valid loss:', score[0])\n",
    "        print('Valid accuracy:', score[1])\n",
    "\n",
    "        #Getting validation Score.\n",
    "        pred_valid=Cnnmodel.predict(X_valid)\n",
    "        y_valid_pred_log[valid_idx] = pred_valid.reshape(pred_valid.shape[0])\n",
    "\n",
    "        #Getting Test Scores\n",
    "        temp_test=Cnnmodel.predict(X_test)\n",
    "        y_test_pred_log+=temp_test.reshape(temp_test.shape[0])\n",
    "\n",
    "        #Getting Train Scores\n",
    "        temp_train=Cnnmodel.predict(X_train)\n",
    "        y_train_pred_log+=temp_train.reshape(temp_train.shape[0])\n",
    "\n",
    "#     y_valid_pred_log=y_valid_pred_log/K\n",
    "    y_train_pred_log=y_train_pred_log/K\n",
    "\n",
    "    print('\\n Train Log Loss Validation= ',log_loss(target_train, y_train_pred_log))\n",
    "    print(' Test Log Loss Validation= ',log_loss(target_train, y_valid_pred_log))\n",
    "    return y_test_pred_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================FOLD= 1\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 75, 75, 3)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 71, 71, 64)        4864      \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 71, 71, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 33, 33, 128)       73856     \n",
      "_________________________________________________________________\n",
      "activation_2 (Activation)    (None, 33, 33, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 14, 14, 128)       147584    \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_4 (Conv2D)            (None, 5, 5, 64)          73792     \n",
      "_________________________________________________________________\n",
      "activation_4 (Activation)    (None, 5, 5, 64)          0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_4 (MaxPooling2 (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 1)                 129       \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 398,913\n",
      "Trainable params: 398,913\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 1282 samples, validate on 322 samples\n",
      "Epoch 1/100\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\google\\protobuf\\internal\\containers.py\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m    552\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 553\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_values\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    554\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'validate_shape'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-a19816060aba>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mpreds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mModel6_CV\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mK\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m5\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-12-a47900357160>\u001b[0m in \u001b[0;36mModel6_CV\u001b[1;34m(X_train, K)\u001b[0m\n\u001b[0;32m     19\u001b[0m         result_Cnn = fitmodel(Cnnmodel,X_train_cv,y_train_cv,X_valid,y_valid,\n\u001b[0;32m     20\u001b[0m                               \u001b[0maugment\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mepochs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m100\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mbatch_size\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m32\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 21\u001b[1;33m                               filepath = file_path)\n\u001b[0m\u001b[0;32m     22\u001b[0m         \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Satge 1,done!'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     23\u001b[0m         \u001b[0mCnnmodel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcnnmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_shape\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;36m75\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m75\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mlr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0.0001\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-8-99fcdf6c5fde>\u001b[0m in \u001b[0;36mfitmodel\u001b[1;34m(model, X_train, y_train, X_valid, y_valid, augment, epochs, batch_size, filepath)\u001b[0m\n\u001b[0;32m      7\u001b[0m                           \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m                           \u001b[0mvalidation_data\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_valid\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_valid\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m                           callbacks=get_callbacks(filepath))\n\u001b[0m\u001b[0;32m     10\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m         result = model.fit_generator(data_augmentation(X_train,y_train,batch_size = batch_size),\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[0;32m   1648\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1649\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1650\u001b[1;33m                               validation_steps=validation_steps)\n\u001b[0m\u001b[0;32m   1651\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1652\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[1;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[0;32m   1211\u001b[0m                     \u001b[0mbatch_logs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'size'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1212\u001b[0m                     \u001b[0mcallbacks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1213\u001b[1;33m                     \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1214\u001b[0m                     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1215\u001b[0m                         \u001b[0mouts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2348\u001b[0m             \u001b[0mfeed_dict\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtensor\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2349\u001b[0m         \u001b[0mfetches\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdates_op\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfetches\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2350\u001b[1;33m         \u001b[0msession\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2351\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m   2352\u001b[0m                               **self.session_kwargs)\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36mget_session\u001b[1;34m()\u001b[0m\n\u001b[0;32m    179\u001b[0m                 \u001b[1;31m# not already marked as initialized.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    180\u001b[0m                 is_initialized = session.run(\n\u001b[1;32m--> 181\u001b[1;33m                     [tf.is_variable_initialized(v) for v in candidate_vars])\n\u001b[0m\u001b[0;32m    182\u001b[0m                 \u001b[0muninitialized_vars\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    183\u001b[0m                 \u001b[1;32mfor\u001b[0m \u001b[0mflag\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mv\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mis_initialized\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcandidate_vars\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    887\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    888\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 889\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    890\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    891\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1118\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1119\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1120\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1121\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1122\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1315\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1316\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[1;32m-> 1317\u001b[1;33m                            options, run_metadata)\n\u001b[0m\u001b[0;32m   1318\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1319\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1321\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1322\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1323\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1324\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1325\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1291\u001b[0m                 run_metadata):\n\u001b[0;32m   1292\u001b[0m       \u001b[1;31m# Ensure any changes to the graph are reflected in the runtime.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1293\u001b[1;33m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1294\u001b[0m       \u001b[1;32mwith\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mraise_exception_on_not_ok_status\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mstatus\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1295\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_created_with_new_api\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_extend_graph\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1347\u001b[0m         graph_def, self._current_version = self._graph._as_graph_def(\n\u001b[0;32m   1348\u001b[0m             \u001b[0mfrom_version\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_current_version\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1349\u001b[1;33m             add_shapes=self._add_shapes)\n\u001b[0m\u001b[0;32m   1350\u001b[0m         \u001b[1;31m# pylint: enable=protected-access\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1351\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\u001b[0m in \u001b[0;36m_as_graph_def\u001b[1;34m(self, from_version, add_shapes)\u001b[0m\n\u001b[0;32m   2724\u001b[0m         \u001b[0mop\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_nodes_by_id\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mop_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2725\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mfrom_version\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mop_id\u001b[0m \u001b[1;33m>\u001b[0m \u001b[0mfrom_version\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2726\u001b[1;33m           \u001b[0mgraph\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnode\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mop\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2727\u001b[0m           \u001b[1;32mif\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0madd_shapes\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2728\u001b[0m             \u001b[1;32massert\u001b[0m \u001b[1;34m\"_output_shapes\"\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mgraph\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnode\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mattr\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\google\\protobuf\\internal\\containers.py\u001b[0m in \u001b[0;36mextend\u001b[1;34m(self, elem_seq)\u001b[0m\n\u001b[0;32m    387\u001b[0m       \u001b[0mnew_element\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmessage_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    388\u001b[0m       \u001b[0mnew_element\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_SetListener\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlistener\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 389\u001b[1;33m       \u001b[0mnew_element\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mMergeFrom\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    390\u001b[0m       \u001b[0mvalues\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_element\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    391\u001b[0m     \u001b[0mlistener\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mModified\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\google\\protobuf\\internal\\python_message.py\u001b[0m in \u001b[0;36mMergeFrom\u001b[1;34m(self, msg)\u001b[0m\n\u001b[0;32m   1242\u001b[0m           \u001b[0mfield_value\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfield\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_default_constructor\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1243\u001b[0m           \u001b[0mfields\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mfield\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfield_value\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1244\u001b[1;33m         \u001b[0mfield_value\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mMergeFrom\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1245\u001b[0m       \u001b[1;32melif\u001b[0m \u001b[0mfield\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcpp_type\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mCPPTYPE_MESSAGE\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1246\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_is_present_in_parent\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\google\\protobuf\\internal\\containers.py\u001b[0m in \u001b[0;36mMergeFrom\u001b[1;34m(self, other)\u001b[0m\n\u001b[0;32m    609\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mkey\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    610\u001b[0m         \u001b[1;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 611\u001b[1;33m       \u001b[0mself\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCopyFrom\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mother\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    612\u001b[0m     \u001b[1;31m# self._message_listener.Modified() not required here, because\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    613\u001b[0m     \u001b[1;31m# mutations to submessages already propagate.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\google\\protobuf\\internal\\containers.py\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m    554\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    555\u001b[0m       \u001b[0mkey\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_key_checker\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mCheckValue\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 556\u001b[1;33m       \u001b[0mnew_element\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_message_descriptor\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_concrete_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    557\u001b[0m       \u001b[0mnew_element\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_SetListener\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_message_listener\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    558\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_values\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnew_element\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\google\\protobuf\\internal\\python_message.py\u001b[0m in \u001b[0;36minit\u001b[1;34m(self, **kwargs)\u001b[0m\n\u001b[0;32m    480\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_is_present_in_parent\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    481\u001b[0m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_listener\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmessage_listener_mod\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mNullMessageListener\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 482\u001b[1;33m     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_listener_for_children\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_Listener\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    483\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mfield_name\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfield_value\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    484\u001b[0m       \u001b[0mfield\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_GetFieldByName\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmessage_descriptor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfield_name\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\google\\protobuf\\internal\\python_message.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, parent_message)\u001b[0m\n\u001b[0;32m   1400\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_parent_message_weakref\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparent_message\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1401\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1402\u001b[1;33m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_parent_message_weakref\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mweakref\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproxy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mparent_message\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1403\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1404\u001b[0m     \u001b[1;31m# As an optimization, we also indicate directly on the listener whether\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "Model6_CV(X_train,K = 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Evaluation_CV(X_train,K=3):\n",
    "    folds = list(StratifiedKFold(n_splits=K, shuffle=True,random_state = 2).split(X_train, target_train))\n",
    "    y_test_pred_log=0\n",
    "    y_train_pred_log=0\n",
    "    y_valid_pred_log = 0.0*target_train\n",
    "    for j, (train_idx, valid_idx) in enumerate(folds):\n",
    "        print('\\n===================FOLD=',j+1)\n",
    "        X_train_cv = X_train[train_idx]\n",
    "        y_train_cv = target_train[train_idx]\n",
    "        X_valid = X_train[valid_idx]\n",
    "        y_valid= target_train[valid_idx]\n",
    "        \n",
    "        #Angle\n",
    "#         X_angle_cv=X_angle[train_idx]\n",
    "#         X_angle_hold=X_angle[test_idx]\n",
    "\n",
    "        #define file path and get callbacks\n",
    "        file_path = 'model save/Model 6-Advanced CNN with DA and self TL(cv3)/Model-6 cv'+str(j+1)+'.hdf5'\n",
    "        Cnnmodel=cnnmodel(input_shape = (75,75,3),lr = 0.0001)\n",
    "        Cnnmodel.load_weights(file_path)\n",
    "\n",
    "\n",
    "#         callbacks = get_callbacks(filepath=file_path, patience=5)\n",
    "#         gen_flow = gen_flow_for_two_inputs(X_train_cv, X_angle_cv, y_train_cv)\n",
    "#         galaxyModel= getVggAngleModel()\n",
    "#         galaxyModel.fit_generator(\n",
    "#                 gen_flow,\n",
    "#                 steps_per_epoch=24,\n",
    "#                 epochs=100,\n",
    "#                 shuffle=True,\n",
    "#                 verbose=1,\n",
    "#                 validation_data=([X_holdout,X_angle_hold], Y_holdout),\n",
    "#                 callbacks=callbacks)\n",
    "\n",
    "        #Getting the Best Model\n",
    "        #Getting Training Score\n",
    "        print('cv'+str(j+1))\n",
    "        score = Cnnmodel.evaluate(X_train_cv, y_train_cv)\n",
    "        print('Train loss:', score[0])\n",
    "        print('Train accuracy:', score[1])\n",
    "        #Getting Test Score\n",
    "        score = Cnnmodel.evaluate(X_valid, y_valid)\n",
    "        print('Valid loss:', score[0])\n",
    "        print('Valid accuracy:', score[1])\n",
    "        print('\\n')\n",
    "\n",
    "        #Getting validation Score.\n",
    "        pred_valid=Cnnmodel.predict(X_valid)\n",
    "        y_valid_pred_log[valid_idx] = pred_valid.reshape(pred_valid.shape[0])\n",
    "\n",
    "        #Getting Test Scores\n",
    "        temp_test=Cnnmodel.predict(X_test)\n",
    "        y_test_pred_log+=temp_test.reshape(temp_test.shape[0])\n",
    "\n",
    "        #Getting Train Scores\n",
    "        temp_train=Cnnmodel.predict(X_train)\n",
    "        y_train_pred_log+=temp_train.reshape(temp_train.shape[0])\n",
    "\n",
    "    y_valid_pred_log=y_valid_pred_log/K\n",
    "    y_train_pred_log=y_train_pred_log/K\n",
    "\n",
    "    print('\\n Train Log Loss Validation= ',log_loss(target_train, y_train_pred_log))\n",
    "    print(' Test Log Loss Validation= ',log_loss(target_train, y_valid_pred_log))\n",
    "    return y_test_pred_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================FOLD= 1\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         (None, 75, 75, 3)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_9 (Conv2D)            (None, 71, 71, 64)        4864      \n",
      "_________________________________________________________________\n",
      "activation_9 (Activation)    (None, 71, 71, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_9 (MaxPooling2 (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_15 (Dropout)         (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_10 (Conv2D)           (None, 33, 33, 128)       73856     \n",
      "_________________________________________________________________\n",
      "activation_10 (Activation)   (None, 33, 33, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_10 (MaxPooling (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "dropout_16 (Dropout)         (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_11 (Conv2D)           (None, 14, 14, 128)       147584    \n",
      "_________________________________________________________________\n",
      "activation_11 (Activation)   (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_11 (MaxPooling (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "dropout_17 (Dropout)         (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_12 (Conv2D)           (None, 5, 5, 64)          73792     \n",
      "_________________________________________________________________\n",
      "activation_12 (Activation)   (None, 5, 5, 64)          0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_12 (MaxPooling (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_18 (Dropout)         (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_3 (Flatten)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_19 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_20 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 1)                 129       \n",
      "_________________________________________________________________\n",
      "dropout_21 (Dropout)         (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 398,913\n",
      "Trainable params: 398,913\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "cv1\n",
      "1282/1282 [==============================] - 2s 1ms/step\n",
      "Train loss: 0.113673846965\n",
      "Train accuracy: 0.958658346334\n",
      "322/322 [==============================] - 0s 1ms/step\n",
      "Valid loss: 0.228779879416\n",
      "Valid accuracy: 0.919254658385\n",
      "\n",
      "\n",
      "\n",
      "===================FOLD= 2\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_4 (InputLayer)         (None, 75, 75, 3)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_13 (Conv2D)           (None, 71, 71, 64)        4864      \n",
      "_________________________________________________________________\n",
      "activation_13 (Activation)   (None, 71, 71, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_13 (MaxPooling (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_22 (Dropout)         (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_14 (Conv2D)           (None, 33, 33, 128)       73856     \n",
      "_________________________________________________________________\n",
      "activation_14 (Activation)   (None, 33, 33, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_14 (MaxPooling (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "dropout_23 (Dropout)         (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_15 (Conv2D)           (None, 14, 14, 128)       147584    \n",
      "_________________________________________________________________\n",
      "activation_15 (Activation)   (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_15 (MaxPooling (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "dropout_24 (Dropout)         (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_16 (Conv2D)           (None, 5, 5, 64)          73792     \n",
      "_________________________________________________________________\n",
      "activation_16 (Activation)   (None, 5, 5, 64)          0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_16 (MaxPooling (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_25 (Dropout)         (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_4 (Flatten)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_26 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_27 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_12 (Dense)             (None, 1)                 129       \n",
      "_________________________________________________________________\n",
      "dropout_28 (Dropout)         (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 398,913\n",
      "Trainable params: 398,913\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "cv2\n",
      "1283/1283 [==============================] - 2s 2ms/step\n",
      "Train loss: 0.149523646778\n",
      "Train accuracy: 0.942322681216\n",
      "321/321 [==============================] - 1s 2ms/step\n",
      "Valid loss: 0.226674056684\n",
      "Valid accuracy: 0.903426791277\n",
      "\n",
      "\n",
      "\n",
      "===================FOLD= 3\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_5 (InputLayer)         (None, 75, 75, 3)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_17 (Conv2D)           (None, 71, 71, 64)        4864      \n",
      "_________________________________________________________________\n",
      "activation_17 (Activation)   (None, 71, 71, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_17 (MaxPooling (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_29 (Dropout)         (None, 35, 35, 64)        0         \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "conv2d_18 (Conv2D)           (None, 33, 33, 128)       73856     \n",
      "_________________________________________________________________\n",
      "activation_18 (Activation)   (None, 33, 33, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_18 (MaxPooling (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "dropout_30 (Dropout)         (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_19 (Conv2D)           (None, 14, 14, 128)       147584    \n",
      "_________________________________________________________________\n",
      "activation_19 (Activation)   (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_19 (MaxPooling (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "dropout_31 (Dropout)         (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_20 (Conv2D)           (None, 5, 5, 64)          73792     \n",
      "_________________________________________________________________\n",
      "activation_20 (Activation)   (None, 5, 5, 64)          0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_20 (MaxPooling (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_32 (Dropout)         (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_5 (Flatten)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_33 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_34 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 1)                 129       \n",
      "_________________________________________________________________\n",
      "dropout_35 (Dropout)         (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 398,913\n",
      "Trainable params: 398,913\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "cv3\n",
      "1283/1283 [==============================] - 2s 1ms/step\n",
      "Train loss: 0.125841816636\n",
      "Train accuracy: 0.947778643804\n",
      "321/321 [==============================] - 0s 1ms/step\n",
      "Valid loss: 0.171666743831\n",
      "Valid accuracy: 0.92523364486\n",
      "\n",
      "\n",
      "\n",
      "===================FOLD= 4\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_6 (InputLayer)         (None, 75, 75, 3)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_21 (Conv2D)           (None, 71, 71, 64)        4864      \n",
      "_________________________________________________________________\n",
      "activation_21 (Activation)   (None, 71, 71, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_21 (MaxPooling (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_36 (Dropout)         (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_22 (Conv2D)           (None, 33, 33, 128)       73856     \n",
      "_________________________________________________________________\n",
      "activation_22 (Activation)   (None, 33, 33, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_22 (MaxPooling (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "dropout_37 (Dropout)         (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_23 (Conv2D)           (None, 14, 14, 128)       147584    \n",
      "_________________________________________________________________\n",
      "activation_23 (Activation)   (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_23 (MaxPooling (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "dropout_38 (Dropout)         (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_24 (Conv2D)           (None, 5, 5, 64)          73792     \n",
      "_________________________________________________________________\n",
      "activation_24 (Activation)   (None, 5, 5, 64)          0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_24 (MaxPooling (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_39 (Dropout)         (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_6 (Flatten)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_40 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_17 (Dense)             (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_41 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 1)                 129       \n",
      "_________________________________________________________________\n",
      "dropout_42 (Dropout)         (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 398,913\n",
      "Trainable params: 398,913\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "cv4\n",
      "1284/1284 [==============================] - 2s 1ms/step\n",
      "Train loss: 0.137632829099\n",
      "Train accuracy: 0.954049844237\n",
      "320/320 [==============================] - ETA:  - 0s 1ms/step\n",
      "Valid loss: 0.221401537955\n",
      "Valid accuracy: 0.915625\n",
      "\n",
      "\n",
      "\n",
      "===================FOLD= 5\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_7 (InputLayer)         (None, 75, 75, 3)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_25 (Conv2D)           (None, 71, 71, 64)        4864      \n",
      "_________________________________________________________________\n",
      "activation_25 (Activation)   (None, 71, 71, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_25 (MaxPooling (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_43 (Dropout)         (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_26 (Conv2D)           (None, 33, 33, 128)       73856     \n",
      "_________________________________________________________________\n",
      "activation_26 (Activation)   (None, 33, 33, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_26 (MaxPooling (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "dropout_44 (Dropout)         (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_27 (Conv2D)           (None, 14, 14, 128)       147584    \n",
      "_________________________________________________________________\n",
      "activation_27 (Activation)   (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_27 (MaxPooling (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dropout_45 (Dropout)         (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_28 (Conv2D)           (None, 5, 5, 64)          73792     \n",
      "_________________________________________________________________\n",
      "activation_28 (Activation)   (None, 5, 5, 64)          0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_28 (MaxPooling (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_46 (Dropout)         (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_7 (Flatten)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_47 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_48 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_21 (Dense)             (None, 1)                 129       \n",
      "_________________________________________________________________\n",
      "dropout_49 (Dropout)         (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 398,913\n",
      "Trainable params: 398,913\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "cv5\n",
      "1284/1284 [==============================] - 2s 1ms/step\n",
      "Train loss: 0.0868912293486\n",
      "Train accuracy: 0.969626168224\n",
      "320/320 [==============================] - 1s 2ms/step\n",
      "Valid loss: 0.20000327453\n",
      "Valid accuracy: 0.925\n",
      "\n",
      "\n",
      "\n",
      " Train Log Loss Validation=  0.127783209132\n",
      " Test Log Loss Validation=  0.209715738675\n"
     ]
    }
   ],
   "source": [
    "preds = Evaluation_CV(X_train,K=5)\n",
    "sub = test[['id']]\n",
    "sub['is_iceberg'] = preds\n",
    "sub.to_csv('5cv.csv',index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Submission(min max median stacking)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_scores(K=5):\n",
    "    y_test_pred_log=0\n",
    "    test_df = test[['id']]\n",
    "    for j in range(K):\n",
    "        print('\\n===================FOLD=',j+1)\n",
    "        file_path = 'model save/Model 6-Advanced CNN with DA and self TL(cv3)/Model-6 cv'+str(j+1)+'.hdf5'\n",
    "        Cnnmodel=cnnmodel(input_shape = (75,75,3),lr = 0.0001)\n",
    "        Cnnmodel.load_weights(file_path)\n",
    "\n",
    "        #Getting Test Scores\n",
    "        temp_test=Cnnmodel.predict(X_test)\n",
    "        test_df['cv'+str(j+1)]=temp_test.reshape(temp_test.shape[0])\n",
    "    return test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================FOLD= 1\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_8 (InputLayer)         (None, 75, 75, 3)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_29 (Conv2D)           (None, 71, 71, 64)        4864      \n",
      "_________________________________________________________________\n",
      "activation_29 (Activation)   (None, 71, 71, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_29 (MaxPooling (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_50 (Dropout)         (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_30 (Conv2D)           (None, 33, 33, 128)       73856     \n",
      "_________________________________________________________________\n",
      "activation_30 (Activation)   (None, 33, 33, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_30 (MaxPooling (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "dropout_51 (Dropout)         (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_31 (Conv2D)           (None, 14, 14, 128)       147584    \n",
      "_________________________________________________________________\n",
      "activation_31 (Activation)   (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_31 (MaxPooling (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "dropout_52 (Dropout)         (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_32 (Conv2D)           (None, 5, 5, 64)          73792     \n",
      "_________________________________________________________________\n",
      "activation_32 (Activation)   (None, 5, 5, 64)          0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_32 (MaxPooling (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_53 (Dropout)         (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_8 (Flatten)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_22 (Dense)             (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_54 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_23 (Dense)             (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_55 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_24 (Dense)             (None, 1)                 129       \n",
      "_________________________________________________________________\n",
      "dropout_56 (Dropout)         (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 398,913\n",
      "Trainable params: 398,913\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\ipykernel_launcher.py:12: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  if sys.path[0] == '':\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================FOLD= 2\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_9 (InputLayer)         (None, 75, 75, 3)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_33 (Conv2D)           (None, 71, 71, 64)        4864      \n",
      "_________________________________________________________________\n",
      "activation_33 (Activation)   (None, 71, 71, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_33 (MaxPooling (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_57 (Dropout)         (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_34 (Conv2D)           (None, 33, 33, 128)       73856     \n",
      "_________________________________________________________________\n",
      "activation_34 (Activation)   (None, 33, 33, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_34 (MaxPooling (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "dropout_58 (Dropout)         (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_35 (Conv2D)           (None, 14, 14, 128)       147584    \n",
      "_________________________________________________________________\n",
      "activation_35 (Activation)   (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_35 (MaxPooling (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "dropout_59 (Dropout)         (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_36 (Conv2D)           (None, 5, 5, 64)          73792     \n",
      "_________________________________________________________________\n",
      "activation_36 (Activation)   (None, 5, 5, 64)          0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_36 (MaxPooling (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_60 (Dropout)         (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_9 (Flatten)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_25 (Dense)             (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_61 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_26 (Dense)             (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_62 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_27 (Dense)             (None, 1)                 129       \n",
      "_________________________________________________________________\n",
      "dropout_63 (Dropout)         (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 398,913\n",
      "Trainable params: 398,913\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "===================FOLD= 3\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_10 (InputLayer)        (None, 75, 75, 3)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_37 (Conv2D)           (None, 71, 71, 64)        4864      \n",
      "_________________________________________________________________\n",
      "activation_37 (Activation)   (None, 71, 71, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_37 (MaxPooling (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_64 (Dropout)         (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_38 (Conv2D)           (None, 33, 33, 128)       73856     \n",
      "_________________________________________________________________\n",
      "activation_38 (Activation)   (None, 33, 33, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_38 (MaxPooling (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "dropout_65 (Dropout)         (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_39 (Conv2D)           (None, 14, 14, 128)       147584    \n",
      "_________________________________________________________________\n",
      "activation_39 (Activation)   (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_39 (MaxPooling (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "dropout_66 (Dropout)         (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_40 (Conv2D)           (None, 5, 5, 64)          73792     \n",
      "_________________________________________________________________\n",
      "activation_40 (Activation)   (None, 5, 5, 64)          0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_40 (MaxPooling (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_67 (Dropout)         (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_10 (Flatten)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_28 (Dense)             (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_68 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_29 (Dense)             (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_69 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_30 (Dense)             (None, 1)                 129       \n",
      "_________________________________________________________________\n",
      "dropout_70 (Dropout)         (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 398,913\n",
      "Trainable params: 398,913\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "\n",
      "===================FOLD= 4\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_11 (InputLayer)        (None, 75, 75, 3)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_41 (Conv2D)           (None, 71, 71, 64)        4864      \n",
      "_________________________________________________________________\n",
      "activation_41 (Activation)   (None, 71, 71, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_41 (MaxPooling (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_71 (Dropout)         (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_42 (Conv2D)           (None, 33, 33, 128)       73856     \n",
      "_________________________________________________________________\n",
      "activation_42 (Activation)   (None, 33, 33, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_42 (MaxPooling (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "dropout_72 (Dropout)         (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_43 (Conv2D)           (None, 14, 14, 128)       147584    \n",
      "_________________________________________________________________\n",
      "activation_43 (Activation)   (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_43 (MaxPooling (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "dropout_73 (Dropout)         (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_44 (Conv2D)           (None, 5, 5, 64)          73792     \n",
      "_________________________________________________________________\n",
      "activation_44 (Activation)   (None, 5, 5, 64)          0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_44 (MaxPooling (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_74 (Dropout)         (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_11 (Flatten)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_31 (Dense)             (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_75 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_32 (Dense)             (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_76 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_33 (Dense)             (None, 1)                 129       \n",
      "_________________________________________________________________\n",
      "dropout_77 (Dropout)         (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 398,913\n",
      "Trainable params: 398,913\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================FOLD= 5\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_12 (InputLayer)        (None, 75, 75, 3)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_45 (Conv2D)           (None, 71, 71, 64)        4864      \n",
      "_________________________________________________________________\n",
      "activation_45 (Activation)   (None, 71, 71, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_45 (MaxPooling (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_78 (Dropout)         (None, 35, 35, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_46 (Conv2D)           (None, 33, 33, 128)       73856     \n",
      "_________________________________________________________________\n",
      "activation_46 (Activation)   (None, 33, 33, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_46 (MaxPooling (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "dropout_79 (Dropout)         (None, 16, 16, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_47 (Conv2D)           (None, 14, 14, 128)       147584    \n",
      "_________________________________________________________________\n",
      "activation_47 (Activation)   (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_47 (MaxPooling (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "dropout_80 (Dropout)         (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_48 (Conv2D)           (None, 5, 5, 64)          73792     \n",
      "_________________________________________________________________\n",
      "activation_48 (Activation)   (None, 5, 5, 64)          0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_48 (MaxPooling (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_81 (Dropout)         (None, 2, 2, 64)          0         \n",
      "_________________________________________________________________\n",
      "flatten_12 (Flatten)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_34 (Dense)             (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_82 (Dropout)         (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_35 (Dense)             (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dropout_83 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_36 (Dense)             (None, 1)                 129       \n",
      "_________________________________________________________________\n",
      "dropout_84 (Dropout)         (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 398,913\n",
      "Trainable params: 398,913\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "test_df = test_scores()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sub_mmm(test_df):\n",
    "    prediction = test_df\n",
    "    prediction['is_iceberg'] = np.where(np.all(prediction[prediction.columns[1:]]>0.9,axis = 1),\n",
    "                                    np.max(prediction[prediction.columns[1:]],axis = 1),\n",
    "                                     np.where(np.all(prediction[prediction.columns[1:]]<0.1,axis = 1),np.min(prediction[prediction.columns[1:]],axis = 1),np.median(prediction[prediction.columns[1:]],axis = 1)))\n",
    "    sub = prediction[['id','is_iceberg']]\n",
    "    sub.to_csv('sub.csv',index = False)\n",
    "    return sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = get_sub_mmm(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}