Model 5-ResNet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.applications.vgg19 import VGG19\n",
    "from keras.applications.vgg19 import preprocess_input\n",
    "from keras.applications.vgg16 import VGG16\n",
    "from keras.preprocessing import image\n",
    "from keras.applications.vgg16 import preprocess_input\n",
    "from keras.applications.resnet50 import ResNet50\n",
    "from keras.models import Model\n",
    "import numpy as np \n",
    "import pandas as pd \n",
    "from subprocess import check_output\n",
    "#print(check_output([\"ls\", \"../input/\"]))\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import mean_squared_error,mean_absolute_error,log_loss\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from os.path import join as opj\n",
    "from matplotlib import pyplot as plt\n",
    "import tensorflow as tf\n",
    "import os\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers.advanced_activations import LeakyReLU, PReLU\n",
    "from keras.applications.inception_v3 import InceptionV3\n",
    "from keras.applications.xception import Xception\n",
    "from keras.applications.mobilenet import MobileNet\n",
    "from keras.layers import Concatenate, LSTM, concatenate\n",
    "from matplotlib import pyplot\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Input, Flatten, Activation,Add, ZeroPadding2D ,AveragePooling2D\n",
    "from keras.layers import GlobalMaxPooling2D, GlobalAveragePooling2D, Lambda\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.layers.merge import Concatenate\n",
    "from keras.models import Model\n",
    "from keras import initializers,regularizers\n",
    "from keras.optimizers import Adam,SGD,RMSprop\n",
    "from keras.initializers import glorot_uniform\n",
    "from keras.callbacks import ModelCheckpoint, Callback, EarlyStopping\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "import keras.backend as K\n",
    "K.set_image_data_format('channels_last')\n",
    "#K.set_learning_phase(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_json(\"input/train.json\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fill Inc_angle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\ipykernel_launcher.py:27: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "#angle_nona = train[train['inc_angle']!='na']\n",
    "#angle_nona = angle_nona.reset_index()\n",
    "only_band1 = ['band1_'+str(i) for i in range(75*75)]\n",
    "only_band2 = ['band2_'+str(i) for i in range(75*75)]\n",
    "both_bands = only_band1+only_band2\n",
    "df = pd.concat([pd.DataFrame(train['band_1'].values.tolist(),columns = only_band1),\n",
    "                pd.DataFrame(train['band_2'].values.tolist(),columns = only_band2)],axis = 1)\n",
    "df['b1_median'] = np.median(df[only_band1],axis = 1)\n",
    "df['b2_median'] = np.median(df[only_band2],axis = 1)\n",
    "df['b1_mean'] = np.mean(df[only_band1],axis = 1)\n",
    "df['b2_mean'] = np.mean(df[only_band2],axis = 1)\n",
    "df['b1_max'] = np.max(df[only_band1],axis = 1)\n",
    "df['b2_max'] = np.max(df[only_band2],axis = 1)\n",
    "df['b1_min'] = np.median(df[only_band1],axis = 1)\n",
    "df['b2_min'] = np.median(df[only_band2],axis = 1)\n",
    "df['b1_std'] = np.std(df[only_band1],axis = 1)\n",
    "df['b2_std'] = np.std(df[only_band2],axis = 1)\n",
    "stats = df.columns[-10:]\n",
    "df['inc_angle'] = train['inc_angle']\n",
    "angle_na = df[df['inc_angle']=='na']\n",
    "angle_nona = df[df['inc_angle']!='na']\n",
    "X_train = angle_nona[stats]\n",
    "X_pred = angle_na[stats]\n",
    "y_train = angle_nona['inc_angle']\n",
    "lr_model = LinearRegression().fit(X_train,y_train)\n",
    "angle_pred = lr_model.predict(X_pred)\n",
    "angle_na['inc_angle'] = angle_pred\n",
    "new = pd.concat([angle_nona,angle_na],axis = 0).sort_index()\n",
    "train = train.sort_index()\n",
    "train['inc_angle'] = new['inc_angle']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_angle = np.array(train['inc_angle'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_json(\"input/test.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_band_1=np.array([np.array(band).astype(np.float32).reshape(75, 75) for band in train[\"band_1\"]])\n",
    "X_band_2=np.array([np.array(band).astype(np.float32).reshape(75, 75) for band in train[\"band_2\"]])\n",
    "X_train = np.concatenate([X_band_1[:, :, :, np.newaxis], X_band_2[:, :, :, np.newaxis],((X_band_1+X_band_2)/2)[:, :, :, np.newaxis]], axis=-1)\n",
    "target_train=train['is_iceberg']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['b1_median'] = [np.median(i) for i in X_band_1]\n",
    "train['b2_median'] = [np.median(i) for i in X_band_2]\n",
    "train['b1_mean'] = [np.mean(i) for i in X_band_1]\n",
    "train['b2_mean'] = [np.mean(i) for i in X_band_2]\n",
    "train['b1_max'] = [np.max(i) for i in X_band_1]\n",
    "train['b2_max'] = [np.max(i) for i in X_band_2]\n",
    "train['b1_min'] = [np.min(i) for i in X_band_1]\n",
    "train['b2_min'] = [np.min(i) for i in X_band_2]\n",
    "train['b1_std'] = [np.std(i) for i in X_band_1]\n",
    "train['b2_std'] = [np.std(i) for i in X_band_2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_other = train.drop(['band_1','band_2','id','is_iceberg'],axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resize_model(size = (224,224),method = 3,align = False):\n",
    "    X_input = Input(shape = (75,75,None))\n",
    "    X = Lambda(lambda i:K.tf.image.resize_images(i,size = size,method = method,align_corners=align))(X_input)\n",
    "    model = Model(inputs = X_input,outputs = X)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def image_compare(method,ind):\n",
    "#     image_born1 = resize_model(size = (224,224),method = method,align = False)\n",
    "#     x1 = image_born1.predict(X_band_1[ind][np.newaxis,:,:,np.newaxis])\n",
    "#     image_born2 = resize_model(size = (224,224),method = method,align = True)\n",
    "#     x2 = image_born2.predict(X_band_1[ind][np.newaxis,:,:,np.newaxis])\n",
    "#     X1 = X_band_1[ind]\n",
    "#     fig,((ax1,ax2),(ax3,ax4)) = plt.subplots(2,2,figsize = (20,20))\n",
    "#     ax1.imshow(np.squeeze(x1))\n",
    "#     ax2.imshow(np.squeeze(x2))\n",
    "#     ax3.imshow(X1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "#image_compare(3,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def resize_process(input_X,size = (224,224),method = 3):\n",
    "    assert len(input_X.shape) == 3\n",
    "    transfer = resize_model(size = size,method = method)\n",
    "    transfered_list = np.squeeze(np.array([transfer.predict(i[np.newaxis,:,:,np.newaxis]) for i in input_X]))\n",
    "    return transfered_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_band_1_224 = resize_process(X_band_1)\n",
    "X_band_2_224 = resize_process(X_band_2)\n",
    "X_train = np.concatenate([X_band_1_224[:, :, :, np.newaxis], X_band_2_224[:, :, :, np.newaxis],((X_band_1_224+X_band_2_224)/2)[:, :, :, np.newaxis]], axis=-1)\n",
    "target_train=train['is_iceberg']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "X_train_cv, X_valid, X_train_other, X_valid_other , y_train_cv, y_valid = train_test_split(X_train,X_other, target_train, random_state=2,stratify = target_train, train_size=0.80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\sklearn\\model_selection\\_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "X_train_cv, X_valid, y_train_cv, y_valid = train_test_split(X_train, target_train, random_state=2,stratify = target_train, train_size=0.80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "del train\n",
    "del df\n",
    "del X_train\n",
    "del X_band_1\n",
    "del X_band_1_224\n",
    "del X_band_2\n",
    "del X_band_2_224"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normal Data Augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_augmentation(X_input,Y_input,batch_size = 32):\n",
    "    data_aug = ImageDataGenerator(#featurewise_center=True,\n",
    "                             #featurewise_std_normalization=True,\n",
    "                             #zca_whitening=True,\n",
    "                             rotation_range=0,\n",
    "                             width_shift_range = 0,\n",
    "                             height_shift_range = 0,\n",
    "                             zoom_range = 0,\n",
    "                             data_format = 'channels_last',\n",
    "                             horizontal_flip = True,\n",
    "                             vertical_flip = True)#,fill_mode = 'constant',cval = 0)\n",
    "    data_aug_batches = data_aug.flow(X_input,Y_input,batch_size = batch_size)\n",
    "    return data_aug_batches"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Augmentation with Inc_angle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "gen = ImageDataGenerator(#featurewise_center=True,\n",
    "                             #featurewise_std_normalization=True,\n",
    "                             #zca_whitening=True,\n",
    "                             rotation_range=40,\n",
    "                             width_shift_range = 0,\n",
    "                             height_shift_range = 0,\n",
    "                             zoom_range = 0.2,\n",
    "                             data_format = 'channels_last',\n",
    "                             horizontal_flip = True,\n",
    "                             vertical_flip = True,fill_mode = 'constant',cval = 0)\n",
    "\n",
    "def gen_with_angle(X1, X2, y,batch_size = 32):\n",
    "    genX1 = gen.flow(X1,y,  batch_size=batch_size,seed = 1204)\n",
    "    genX2 = gen.flow(X1,X2, batch_size=batch_size,seed = 1204)\n",
    "    while True:\n",
    "            X1i = genX1.next()\n",
    "            X2i = genX2.next()\n",
    "            #Assert arrays are equal - this was for peace of mind, but slows down training\n",
    "            #np.testing.assert_array_equal(X1i[0],X2i[0])\n",
    "            yield [X1i[0], X2i[1]], X1i[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Resnet50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "j = ResNet50(include_top=False,weights=None,input_shape=(224,224,3),classes=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'res5a_branch2a'"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "j.layers[141].name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_6 <keras.engine.topology.InputLayer object at 0x000002889B8E4550>\n",
      "conv1 <keras.layers.convolutional.Conv2D object at 0x000002889B8E4160>\n",
      "bn_conv1 <keras.layers.normalization.BatchNormalization object at 0x000002889B8E4B00>\n",
      "activation_148 <keras.layers.core.Activation object at 0x000002889B8E4080>\n",
      "max_pooling2d_4 <keras.layers.pooling.MaxPooling2D object at 0x000002889B8E2CC0>\n",
      "res2a_branch2a <keras.layers.convolutional.Conv2D object at 0x000002889B951BA8>\n",
      "bn2a_branch2a <keras.layers.normalization.BatchNormalization object at 0x000002889B970C18>\n",
      "activation_149 <keras.layers.core.Activation object at 0x000002889BB47550>\n",
      "res2a_branch2b <keras.layers.convolutional.Conv2D object at 0x000002889BB68DD8>\n",
      "bn2a_branch2b <keras.layers.normalization.BatchNormalization object at 0x000002889BB77F60>\n",
      "activation_150 <keras.layers.core.Activation object at 0x000002889BE1B518>\n",
      "res2a_branch2c <keras.layers.convolutional.Conv2D object at 0x000002889BF11A90>\n",
      "res2a_branch1 <keras.layers.convolutional.Conv2D object at 0x000002889C185160>\n",
      "bn2a_branch2c <keras.layers.normalization.BatchNormalization object at 0x000002889BFFBFD0>\n",
      "bn2a_branch1 <keras.layers.normalization.BatchNormalization object at 0x000002889C29F278>\n",
      "add_49 <keras.layers.merge.Add object at 0x000002889C4709B0>\n",
      "activation_151 <keras.layers.core.Activation object at 0x000002889C28EC18>\n",
      "res2b_branch2a <keras.layers.convolutional.Conv2D object at 0x000002889C557B70>\n",
      "bn2b_branch2a <keras.layers.normalization.BatchNormalization object at 0x000002889C574710>\n",
      "activation_152 <keras.layers.core.Activation object at 0x000002889C82EEB8>\n",
      "res2b_branch2b <keras.layers.convolutional.Conv2D object at 0x000002889C81BDA0>\n",
      "bn2b_branch2b <keras.layers.normalization.BatchNormalization object at 0x000002889C8C2E80>\n",
      "activation_153 <keras.layers.core.Activation object at 0x000002889CB84B38>\n",
      "res2b_branch2c <keras.layers.convolutional.Conv2D object at 0x000002889CBAAD68>\n",
      "bn2b_branch2c <keras.layers.normalization.BatchNormalization object at 0x000002889CD54C88>\n",
      "add_50 <keras.layers.merge.Add object at 0x000002889CE5F4A8>\n",
      "activation_154 <keras.layers.core.Activation object at 0x000002889CF14BE0>\n",
      "res2c_branch2a <keras.layers.convolutional.Conv2D object at 0x000002889D10E9B0>\n",
      "bn2c_branch2a <keras.layers.normalization.BatchNormalization object at 0x000002889CE70AC8>\n",
      "activation_155 <keras.layers.core.Activation object at 0x000002889D1EB908>\n",
      "res2c_branch2b <keras.layers.convolutional.Conv2D object at 0x000002889D1FD8D0>\n",
      "bn2c_branch2b <keras.layers.normalization.BatchNormalization object at 0x000002889D2CD710>\n",
      "activation_156 <keras.layers.core.Activation object at 0x000002889D4B10F0>\n",
      "res2c_branch2c <keras.layers.convolutional.Conv2D object at 0x000002889D66CF28>\n",
      "bn2c_branch2c <keras.layers.normalization.BatchNormalization object at 0x000002889D5A7C88>\n",
      "add_51 <keras.layers.merge.Add object at 0x000002889D8712B0>\n",
      "activation_157 <keras.layers.core.Activation object at 0x000002889D85C780>\n",
      "res3a_branch2a <keras.layers.convolutional.Conv2D object at 0x000002889D921B38>\n",
      "bn3a_branch2a <keras.layers.normalization.BatchNormalization object at 0x000002889DACFEF0>\n",
      "activation_158 <keras.layers.core.Activation object at 0x000002889D911FD0>\n",
      "res3a_branch2b <keras.layers.convolutional.Conv2D object at 0x000002889DBDA390>\n",
      "bn3a_branch2b <keras.layers.normalization.BatchNormalization object at 0x000002889DC8B9B0>\n",
      "activation_159 <keras.layers.core.Activation object at 0x000002889DE8A630>\n",
      "res3a_branch2c <keras.layers.convolutional.Conv2D object at 0x000002889DEB1518>\n",
      "res3a_branch1 <keras.layers.convolutional.Conv2D object at 0x000002889E23CB38>\n",
      "bn3a_branch2c <keras.layers.normalization.BatchNormalization object at 0x000002889DF63D30>\n",
      "bn3a_branch1 <keras.layers.normalization.BatchNormalization object at 0x000002889E30DF28>\n",
      "add_52 <keras.layers.merge.Add object at 0x000002889E5C34A8>\n",
      "activation_160 <keras.layers.core.Activation object at 0x000002889E5EA5F8>\n",
      "res3b_branch2a <keras.layers.convolutional.Conv2D object at 0x000002889E5EAE48>\n",
      "bn3b_branch2a <keras.layers.normalization.BatchNormalization object at 0x000002889E4CCAC8>\n",
      "activation_161 <keras.layers.core.Activation object at 0x000002889E7B9940>\n",
      "res3b_branch2b <keras.layers.convolutional.Conv2D object at 0x000002889E952DA0>\n",
      "bn3b_branch2b <keras.layers.normalization.BatchNormalization object at 0x000002889EA47DD8>\n",
      "activation_162 <keras.layers.core.Activation object at 0x00000288C86692B0>\n",
      "res3b_branch2c <keras.layers.convolutional.Conv2D object at 0x00000288C865C860>\n",
      "bn3b_branch2c <keras.layers.normalization.BatchNormalization object at 0x00000288C8E69F60>\n",
      "add_53 <keras.layers.merge.Add object at 0x00000288C945C5C0>\n",
      "activation_163 <keras.layers.core.Activation object at 0x00000288C9481F28>\n",
      "res3c_branch2a <keras.layers.convolutional.Conv2D object at 0x00000288C9536CC0>\n",
      "bn3c_branch2a <keras.layers.normalization.BatchNormalization object at 0x00000288C95140B8>\n",
      "activation_164 <keras.layers.core.Activation object at 0x00000288C96F1EB8>\n",
      "res3c_branch2b <keras.layers.convolutional.Conv2D object at 0x00000288C97FDEF0>\n",
      "bn3c_branch2b <keras.layers.normalization.BatchNormalization object at 0x00000288CABFEFD0>\n",
      "activation_165 <keras.layers.core.Activation object at 0x00000288CB429A90>\n",
      "res3c_branch2c <keras.layers.convolutional.Conv2D object at 0x00000288CB416CF8>\n",
      "bn3c_branch2c <keras.layers.normalization.BatchNormalization object at 0x00000288CB491E10>\n",
      "add_54 <keras.layers.merge.Add object at 0x00000288CB514FD0>\n",
      "activation_166 <keras.layers.core.Activation object at 0x00000288CB4DCF28>\n",
      "res3d_branch2a <keras.layers.convolutional.Conv2D object at 0x00000288CB4EF908>\n",
      "bn3d_branch2a <keras.layers.normalization.BatchNormalization object at 0x00000288CC3E7F98>\n",
      "activation_167 <keras.layers.core.Activation object at 0x00000288CC4427F0>\n",
      "res3d_branch2b <keras.layers.convolutional.Conv2D object at 0x00000288DF769748>\n",
      "bn3d_branch2b <keras.layers.normalization.BatchNormalization object at 0x00000288DF78DC50>\n",
      "activation_168 <keras.layers.core.Activation object at 0x00000288ECB72208>\n",
      "res3d_branch2c <keras.layers.convolutional.Conv2D object at 0x00000288EDF1BEF0>\n",
      "bn3d_branch2c <keras.layers.normalization.BatchNormalization object at 0x00000288ECBA6AC8>\n",
      "add_55 <keras.layers.merge.Add object at 0x00000288EE465358>\n",
      "activation_169 <keras.layers.core.Activation object at 0x00000288EF471630>\n",
      "res4a_branch2a <keras.layers.convolutional.Conv2D object at 0x00000288EE9CDA90>\n",
      "bn4a_branch2a <keras.layers.normalization.BatchNormalization object at 0x00000288EF481A20>\n",
      "activation_170 <keras.layers.core.Activation object at 0x00000288EFCC9240>\n",
      "res4a_branch2b <keras.layers.convolutional.Conv2D object at 0x00000288EFCB6E48>\n",
      "bn4a_branch2b <keras.layers.normalization.BatchNormalization object at 0x00000288EFF6DBE0>\n",
      "activation_171 <keras.layers.core.Activation object at 0x00000288F198B0B8>\n",
      "res4a_branch2c <keras.layers.convolutional.Conv2D object at 0x00000288F19AD4A8>\n",
      "res4a_branch1 <keras.layers.convolutional.Conv2D object at 0x00000288F1F19630>\n",
      "bn4a_branch2c <keras.layers.normalization.BatchNormalization object at 0x00000288F1D8FEF0>\n",
      "bn4a_branch1 <keras.layers.normalization.BatchNormalization object at 0x00000288F244CF60>\n",
      "add_56 <keras.layers.merge.Add object at 0x00000288F347F898>\n",
      "activation_172 <keras.layers.core.Activation object at 0x00000288F34A56D8>\n",
      "res4b_branch2a <keras.layers.convolutional.Conv2D object at 0x00000288F34B7438>\n",
      "bn4b_branch2a <keras.layers.normalization.BatchNormalization object at 0x00000288F34B7550>\n",
      "activation_173 <keras.layers.core.Activation object at 0x00000288F551BF98>\n",
      "res4b_branch2b <keras.layers.convolutional.Conv2D object at 0x00000288F3EE9518>\n",
      "bn4b_branch2b <keras.layers.normalization.BatchNormalization object at 0x00000288F5950F28>\n",
      "activation_174 <keras.layers.core.Activation object at 0x00000288F64110B8>\n",
      "res4b_branch2c <keras.layers.convolutional.Conv2D object at 0x00000288F6789FD0>\n",
      "bn4b_branch2c <keras.layers.normalization.BatchNormalization object at 0x00000288F67573C8>\n",
      "add_57 <keras.layers.merge.Add object at 0x00000288F6C7C198>\n",
      "activation_175 <keras.layers.core.Activation object at 0x00000288F6EF3EB8>\n",
      "res4c_branch2a <keras.layers.convolutional.Conv2D object at 0x00000288F6EE6C50>\n",
      "bn4c_branch2a <keras.layers.normalization.BatchNormalization object at 0x00000288F6EAFA20>\n",
      "activation_176 <keras.layers.core.Activation object at 0x00000288F7963EB8>\n",
      "res4c_branch2b <keras.layers.convolutional.Conv2D object at 0x00000288F7E9CF98>\n",
      "bn4c_branch2b <keras.layers.normalization.BatchNormalization object at 0x00000288F7CA8A90>\n",
      "activation_177 <keras.layers.core.Activation object at 0x00000288F98EF3C8>\n",
      "res4c_branch2c <keras.layers.convolutional.Conv2D object at 0x00000288F941C8D0>\n",
      "bn4c_branch2c <keras.layers.normalization.BatchNormalization object at 0x00000288FA40FEF0>\n",
      "add_58 <keras.layers.merge.Add object at 0x00000288FA9202B0>\n",
      "activation_178 <keras.layers.core.Activation object at 0x00000288FA944EF0>\n",
      "res4d_branch2a <keras.layers.convolutional.Conv2D object at 0x00000288FAE6AE80>\n",
      "bn4d_branch2a <keras.layers.normalization.BatchNormalization object at 0x00000288FAE876D8>\n",
      "activation_179 <keras.layers.core.Activation object at 0x00000288FBA45278>\n",
      "res4d_branch2b <keras.layers.convolutional.Conv2D object at 0x00000288FBA2EE80>\n",
      "bn4d_branch2b <keras.layers.normalization.BatchNormalization object at 0x00000288FD8F9C18>\n",
      "activation_180 <keras.layers.core.Activation object at 0x00000288FDA850F0>\n",
      "res4d_branch2c <keras.layers.convolutional.Conv2D object at 0x00000288FDAA74E0>\n",
      "bn4d_branch2c <keras.layers.normalization.BatchNormalization object at 0x00000288FE307F28>\n",
      "add_59 <keras.layers.merge.Add object at 0x00000288FE38FE10>\n",
      "activation_181 <keras.layers.core.Activation object at 0x00000288FE317E10>\n",
      "res4e_branch2a <keras.layers.convolutional.Conv2D object at 0x00000288FEE5DC88>\n",
      "bn4e_branch2a <keras.layers.normalization.BatchNormalization object at 0x00000288FE3A0908>\n",
      "activation_182 <keras.layers.core.Activation object at 0x00000288FF3799B0>\n",
      "res4e_branch2b <keras.layers.convolutional.Conv2D object at 0x00000288FF39EE10>\n",
      "bn4e_branch2b <keras.layers.normalization.BatchNormalization object at 0x00000288FFD59DD8>\n",
      "activation_183 <keras.layers.core.Activation object at 0x00000288FFEC5AC8>\n",
      "res4e_branch2c <keras.layers.convolutional.Conv2D object at 0x00000288FFEB5D30>\n",
      "bn4e_branch2c <keras.layers.normalization.BatchNormalization object at 0x00000289D56E2BA8>\n",
      "add_60 <keras.layers.merge.Add object at 0x00000289D5718438>\n",
      "activation_184 <keras.layers.core.Activation object at 0x00000289D574CB70>\n",
      "res4f_branch2a <keras.layers.convolutional.Conv2D object at 0x00000289D575EDD8>\n",
      "bn4f_branch2a <keras.layers.normalization.BatchNormalization object at 0x00000289D572BA20>\n",
      "activation_185 <keras.layers.core.Activation object at 0x00000289D58269B0>\n",
      "res4f_branch2b <keras.layers.convolutional.Conv2D object at 0x00000289D5836A90>\n",
      "bn4f_branch2b <keras.layers.normalization.BatchNormalization object at 0x00000289D6C22A58>\n",
      "activation_186 <keras.layers.core.Activation object at 0x00000289D6C8EFD0>\n",
      "res4f_branch2c <keras.layers.convolutional.Conv2D object at 0x00000289D6CB19B0>\n",
      "bn4f_branch2c <keras.layers.normalization.BatchNormalization object at 0x00000289D6CC3AC8>\n",
      "add_61 <keras.layers.merge.Add object at 0x00000289D6D43358>\n",
      "activation_187 <keras.layers.core.Activation object at 0x00000289D6D79C18>\n",
      "res5a_branch2a <keras.layers.convolutional.Conv2D object at 0x00000289D6DBDEF0>\n",
      "bn5a_branch2a <keras.layers.normalization.BatchNormalization object at 0x00000289D6D66AC8>\n",
      "activation_188 <keras.layers.core.Activation object at 0x00000289D6E06588>\n",
      "res5a_branch2b <keras.layers.convolutional.Conv2D object at 0x00000289D6E60EB8>\n",
      "bn5a_branch2b <keras.layers.normalization.BatchNormalization object at 0x00000289D6E443C8>\n",
      "activation_189 <keras.layers.core.Activation object at 0x00000289D6F15208>\n",
      "res5a_branch2c <keras.layers.convolutional.Conv2D object at 0x00000289D6F8EF28>\n",
      "res5a_branch1 <keras.layers.convolutional.Conv2D object at 0x00000289D70122B0>\n",
      "bn5a_branch2c <keras.layers.normalization.BatchNormalization object at 0x00000289D6F5DA20>\n",
      "bn5a_branch1 <keras.layers.normalization.BatchNormalization object at 0x00000289D7079CF8>\n",
      "add_62 <keras.layers.merge.Add object at 0x00000289D70E8DA0>\n",
      "activation_190 <keras.layers.core.Activation object at 0x00000289D710DD68>\n",
      "res5b_branch2a <keras.layers.convolutional.Conv2D object at 0x00000289D712CB00>\n",
      "bn5b_branch2a <keras.layers.normalization.BatchNormalization object at 0x00000289D70FBBE0>\n",
      "activation_191 <keras.layers.core.Activation object at 0x00000289D71F7D30>\n",
      "res5b_branch2b <keras.layers.convolutional.Conv2D object at 0x00000289D71E3BA8>\n",
      "bn5b_branch2b <keras.layers.normalization.BatchNormalization object at 0x00000289D726CE10>\n",
      "activation_192 <keras.layers.core.Activation object at 0x00000289D72DDB00>\n",
      "res5b_branch2c <keras.layers.convolutional.Conv2D object at 0x00000289D72CCF60>\n",
      "bn5b_branch2c <keras.layers.normalization.BatchNormalization object at 0x00000289DC14ABE0>\n",
      "add_63 <keras.layers.merge.Add object at 0x00000289DC180470>\n",
      "activation_193 <keras.layers.core.Activation object at 0x00000289DC1B3BA8>\n",
      "res5c_branch2a <keras.layers.convolutional.Conv2D object at 0x00000289DC1C6710>\n",
      "bn5c_branch2a <keras.layers.normalization.BatchNormalization object at 0x00000289DC18EA90>\n",
      "activation_194 <keras.layers.core.Activation object at 0x00000289DC28D9E8>\n",
      "res5c_branch2b <keras.layers.convolutional.Conv2D object at 0x00000289DC29FAC8>\n",
      "bn5c_branch2b <keras.layers.normalization.BatchNormalization object at 0x00000289DC31BA90>\n",
      "activation_195 <keras.layers.core.Activation object at 0x00000289DC353198>\n",
      "res5c_branch2c <keras.layers.convolutional.Conv2D object at 0x00000289DC388780>\n",
      "bn5c_branch2c <keras.layers.normalization.BatchNormalization object at 0x00000289DC3BCF98>\n",
      "add_64 <keras.layers.merge.Add object at 0x00000289DC43E710>\n",
      "activation_196 <keras.layers.core.Activation object at 0x00000289DC473C50>\n",
      "avg_pool <keras.layers.pooling.AveragePooling2D object at 0x00000289DC4B7F28>\n"
     ]
    }
   ],
   "source": [
    "for layer in j.layers:\n",
    "    print(layer.name,layer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def resmodel(lr = 0.0001):\n",
    "    #input_other = Input(shape=(11,), name=\"angle\")\n",
    "    #First composite layer\n",
    "    base_model = ResNet50(include_top=False,weights='imagenet',input_shape=(224,224,3),classes=1)\n",
    "    for layer in base_model.layers[:141]:\n",
    "        layer.trainable = False\n",
    "    #base_model2 = VGG19(include_top=False,weights = 'imagenet',input_tensor = base_model.input,input_shape = (75,75,3),classes = 1)\n",
    "#     for layer in base_model2.layers[:7]:\n",
    "#         layer.trainable = False\n",
    "#     for layer in base_model2.layers:\n",
    "#         layer.name = layer.name+'(2)'\n",
    "#     Y = base_model2.output\n",
    "#     W = base_model.output\n",
    "    W = base_model.output\n",
    "    W = Flatten()(W)\n",
    "    W = Dropout(0.3)(W)\n",
    "#     Y = GlobalMaxPooling2D()(Y)\n",
    "#     W = concatenate([W, input_other])\n",
    "#     W = Dense(1024,activation= 'relu')(W)\n",
    "#     W = Dropout(0.3)(W)\n",
    "#     W = Dense(1024,activation= 'relu')(W)\n",
    "#     W = Dropout(0.3)(W)\n",
    "#     Y = concatenate([Y, input_other])\n",
    "#     Y = Dense(1024,activation= 'relu')(Y)\n",
    "#     Y = Dropout(0.3)(Y)\n",
    "#     Y = Dense(1024,activation= 'relu')(Y)\n",
    "#     Y = Dropout(0.3)(Y)\n",
    "    #X = Dense(256,activation= 'relu')(X)\n",
    "    #X = BatchNormalization()(X)\n",
    "    #X = Dropout(0.2)(X)\n",
    "    #Decision layer\n",
    "#     X = concatenate([W,Y])\n",
    "#     X = Dense(1024,activation= 'relu')(X)\n",
    "    W = Dense(1,activation = 'sigmoid')(W)\n",
    "    \n",
    "    model = Model(inputs=base_model.input,outputs=W)\n",
    "    #optimizer=Adam(lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)\n",
    "    optimizer = SGD(lr, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "    model.compile(loss='binary_crossentropy',\n",
    "                  optimizer=optimizer,\n",
    "                  metrics=['accuracy'])\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "def get_callbacks(filepath = \".model_weights.hdf5\", patience=7):\n",
    "    #es = EarlyStopping('val_loss', patience=patience, mode=\"min\")\n",
    "    msave = ModelCheckpoint(filepath, save_best_only=True)\n",
    "    return [msave]\n",
    "\n",
    "#callbacks = get_callbacks(filepath=file_path, patience=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_3 (InputLayer)            (None, 224, 224, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv1 (Conv2D)                  (None, 112, 112, 64) 9472        input_3[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "bn_conv1 (BatchNormalization)   (None, 112, 112, 64) 256         conv1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 112, 112, 64) 0           bn_conv1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2D)  (None, 55, 55, 64)   0           activation_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "res2a_branch2a (Conv2D)         (None, 55, 55, 64)   4160        max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "bn2a_branch2a (BatchNormalizati (None, 55, 55, 64)   256         res2a_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 55, 55, 64)   0           bn2a_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2a_branch2b (Conv2D)         (None, 55, 55, 64)   36928       activation_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2a_branch2b (BatchNormalizati (None, 55, 55, 64)   256         res2a_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_3 (Activation)       (None, 55, 55, 64)   0           bn2a_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2a_branch2c (Conv2D)         (None, 55, 55, 256)  16640       activation_3[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "res2a_branch1 (Conv2D)          (None, 55, 55, 256)  16640       max_pooling2d_1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "bn2a_branch2c (BatchNormalizati (None, 55, 55, 256)  1024        res2a_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "bn2a_branch1 (BatchNormalizatio (None, 55, 55, 256)  1024        res2a_branch1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 55, 55, 256)  0           bn2a_branch2c[0][0]              \n",
      "                                                                 bn2a_branch1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_4 (Activation)       (None, 55, 55, 256)  0           add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res2b_branch2a (Conv2D)         (None, 55, 55, 64)   16448       activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2b_branch2a (BatchNormalizati (None, 55, 55, 64)   256         res2b_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_5 (Activation)       (None, 55, 55, 64)   0           bn2b_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2b_branch2b (Conv2D)         (None, 55, 55, 64)   36928       activation_5[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2b_branch2b (BatchNormalizati (None, 55, 55, 64)   256         res2b_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_6 (Activation)       (None, 55, 55, 64)   0           bn2b_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2b_branch2c (Conv2D)         (None, 55, 55, 256)  16640       activation_6[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2b_branch2c (BatchNormalizati (None, 55, 55, 256)  1024        res2b_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 55, 55, 256)  0           bn2b_branch2c[0][0]              \n",
      "                                                                 activation_4[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_7 (Activation)       (None, 55, 55, 256)  0           add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res2c_branch2a (Conv2D)         (None, 55, 55, 64)   16448       activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2c_branch2a (BatchNormalizati (None, 55, 55, 64)   256         res2c_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_8 (Activation)       (None, 55, 55, 64)   0           bn2c_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2c_branch2b (Conv2D)         (None, 55, 55, 64)   36928       activation_8[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2c_branch2b (BatchNormalizati (None, 55, 55, 64)   256         res2c_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_9 (Activation)       (None, 55, 55, 64)   0           bn2c_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res2c_branch2c (Conv2D)         (None, 55, 55, 256)  16640       activation_9[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "bn2c_branch2c (BatchNormalizati (None, 55, 55, 256)  1024        res2c_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_3 (Add)                     (None, 55, 55, 256)  0           bn2c_branch2c[0][0]              \n",
      "                                                                 activation_7[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_10 (Activation)      (None, 55, 55, 256)  0           add_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res3a_branch2a (Conv2D)         (None, 28, 28, 128)  32896       activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3a_branch2a (BatchNormalizati (None, 28, 28, 128)  512         res3a_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_11 (Activation)      (None, 28, 28, 128)  0           bn3a_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3a_branch2b (Conv2D)         (None, 28, 28, 128)  147584      activation_11[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3a_branch2b (BatchNormalizati (None, 28, 28, 128)  512         res3a_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_12 (Activation)      (None, 28, 28, 128)  0           bn3a_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3a_branch2c (Conv2D)         (None, 28, 28, 512)  66048       activation_12[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3a_branch1 (Conv2D)          (None, 28, 28, 512)  131584      activation_10[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3a_branch2c (BatchNormalizati (None, 28, 28, 512)  2048        res3a_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "bn3a_branch1 (BatchNormalizatio (None, 28, 28, 512)  2048        res3a_branch1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_4 (Add)                     (None, 28, 28, 512)  0           bn3a_branch2c[0][0]              \n",
      "                                                                 bn3a_branch1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_13 (Activation)      (None, 28, 28, 512)  0           add_4[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res3b_branch2a (Conv2D)         (None, 28, 28, 128)  65664       activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3b_branch2a (BatchNormalizati (None, 28, 28, 128)  512         res3b_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_14 (Activation)      (None, 28, 28, 128)  0           bn3b_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3b_branch2b (Conv2D)         (None, 28, 28, 128)  147584      activation_14[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3b_branch2b (BatchNormalizati (None, 28, 28, 128)  512         res3b_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_15 (Activation)      (None, 28, 28, 128)  0           bn3b_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3b_branch2c (Conv2D)         (None, 28, 28, 512)  66048       activation_15[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3b_branch2c (BatchNormalizati (None, 28, 28, 512)  2048        res3b_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_5 (Add)                     (None, 28, 28, 512)  0           bn3b_branch2c[0][0]              \n",
      "                                                                 activation_13[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_16 (Activation)      (None, 28, 28, 512)  0           add_5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res3c_branch2a (Conv2D)         (None, 28, 28, 128)  65664       activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3c_branch2a (BatchNormalizati (None, 28, 28, 128)  512         res3c_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_17 (Activation)      (None, 28, 28, 128)  0           bn3c_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3c_branch2b (Conv2D)         (None, 28, 28, 128)  147584      activation_17[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3c_branch2b (BatchNormalizati (None, 28, 28, 128)  512         res3c_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_18 (Activation)      (None, 28, 28, 128)  0           bn3c_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3c_branch2c (Conv2D)         (None, 28, 28, 512)  66048       activation_18[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3c_branch2c (BatchNormalizati (None, 28, 28, 512)  2048        res3c_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_6 (Add)                     (None, 28, 28, 512)  0           bn3c_branch2c[0][0]              \n",
      "                                                                 activation_16[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_19 (Activation)      (None, 28, 28, 512)  0           add_6[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res3d_branch2a (Conv2D)         (None, 28, 28, 128)  65664       activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3d_branch2a (BatchNormalizati (None, 28, 28, 128)  512         res3d_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_20 (Activation)      (None, 28, 28, 128)  0           bn3d_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3d_branch2b (Conv2D)         (None, 28, 28, 128)  147584      activation_20[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3d_branch2b (BatchNormalizati (None, 28, 28, 128)  512         res3d_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_21 (Activation)      (None, 28, 28, 128)  0           bn3d_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res3d_branch2c (Conv2D)         (None, 28, 28, 512)  66048       activation_21[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn3d_branch2c (BatchNormalizati (None, 28, 28, 512)  2048        res3d_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_7 (Add)                     (None, 28, 28, 512)  0           bn3d_branch2c[0][0]              \n",
      "                                                                 activation_19[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_22 (Activation)      (None, 28, 28, 512)  0           add_7[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res4a_branch2a (Conv2D)         (None, 14, 14, 256)  131328      activation_22[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4a_branch2a (BatchNormalizati (None, 14, 14, 256)  1024        res4a_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_23 (Activation)      (None, 14, 14, 256)  0           bn4a_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4a_branch2b (Conv2D)         (None, 14, 14, 256)  590080      activation_23[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4a_branch2b (BatchNormalizati (None, 14, 14, 256)  1024        res4a_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_24 (Activation)      (None, 14, 14, 256)  0           bn4a_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4a_branch2c (Conv2D)         (None, 14, 14, 1024) 263168      activation_24[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4a_branch1 (Conv2D)          (None, 14, 14, 1024) 525312      activation_22[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4a_branch2c (BatchNormalizati (None, 14, 14, 1024) 4096        res4a_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "bn4a_branch1 (BatchNormalizatio (None, 14, 14, 1024) 4096        res4a_branch1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_8 (Add)                     (None, 14, 14, 1024) 0           bn4a_branch2c[0][0]              \n",
      "                                                                 bn4a_branch1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_25 (Activation)      (None, 14, 14, 1024) 0           add_8[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res4b_branch2a (Conv2D)         (None, 14, 14, 256)  262400      activation_25[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4b_branch2a (BatchNormalizati (None, 14, 14, 256)  1024        res4b_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_26 (Activation)      (None, 14, 14, 256)  0           bn4b_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4b_branch2b (Conv2D)         (None, 14, 14, 256)  590080      activation_26[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4b_branch2b (BatchNormalizati (None, 14, 14, 256)  1024        res4b_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_27 (Activation)      (None, 14, 14, 256)  0           bn4b_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4b_branch2c (Conv2D)         (None, 14, 14, 1024) 263168      activation_27[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4b_branch2c (BatchNormalizati (None, 14, 14, 1024) 4096        res4b_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_9 (Add)                     (None, 14, 14, 1024) 0           bn4b_branch2c[0][0]              \n",
      "                                                                 activation_25[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_28 (Activation)      (None, 14, 14, 1024) 0           add_9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "res4c_branch2a (Conv2D)         (None, 14, 14, 256)  262400      activation_28[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4c_branch2a (BatchNormalizati (None, 14, 14, 256)  1024        res4c_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_29 (Activation)      (None, 14, 14, 256)  0           bn4c_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4c_branch2b (Conv2D)         (None, 14, 14, 256)  590080      activation_29[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4c_branch2b (BatchNormalizati (None, 14, 14, 256)  1024        res4c_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_30 (Activation)      (None, 14, 14, 256)  0           bn4c_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4c_branch2c (Conv2D)         (None, 14, 14, 1024) 263168      activation_30[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4c_branch2c (BatchNormalizati (None, 14, 14, 1024) 4096        res4c_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_10 (Add)                    (None, 14, 14, 1024) 0           bn4c_branch2c[0][0]              \n",
      "                                                                 activation_28[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_31 (Activation)      (None, 14, 14, 1024) 0           add_10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res4d_branch2a (Conv2D)         (None, 14, 14, 256)  262400      activation_31[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4d_branch2a (BatchNormalizati (None, 14, 14, 256)  1024        res4d_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_32 (Activation)      (None, 14, 14, 256)  0           bn4d_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4d_branch2b (Conv2D)         (None, 14, 14, 256)  590080      activation_32[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4d_branch2b (BatchNormalizati (None, 14, 14, 256)  1024        res4d_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_33 (Activation)      (None, 14, 14, 256)  0           bn4d_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4d_branch2c (Conv2D)         (None, 14, 14, 1024) 263168      activation_33[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4d_branch2c (BatchNormalizati (None, 14, 14, 1024) 4096        res4d_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_11 (Add)                    (None, 14, 14, 1024) 0           bn4d_branch2c[0][0]              \n",
      "                                                                 activation_31[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_34 (Activation)      (None, 14, 14, 1024) 0           add_11[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res4e_branch2a (Conv2D)         (None, 14, 14, 256)  262400      activation_34[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4e_branch2a (BatchNormalizati (None, 14, 14, 256)  1024        res4e_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_35 (Activation)      (None, 14, 14, 256)  0           bn4e_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4e_branch2b (Conv2D)         (None, 14, 14, 256)  590080      activation_35[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4e_branch2b (BatchNormalizati (None, 14, 14, 256)  1024        res4e_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_36 (Activation)      (None, 14, 14, 256)  0           bn4e_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4e_branch2c (Conv2D)         (None, 14, 14, 1024) 263168      activation_36[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4e_branch2c (BatchNormalizati (None, 14, 14, 1024) 4096        res4e_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_12 (Add)                    (None, 14, 14, 1024) 0           bn4e_branch2c[0][0]              \n",
      "                                                                 activation_34[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_37 (Activation)      (None, 14, 14, 1024) 0           add_12[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res4f_branch2a (Conv2D)         (None, 14, 14, 256)  262400      activation_37[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4f_branch2a (BatchNormalizati (None, 14, 14, 256)  1024        res4f_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_38 (Activation)      (None, 14, 14, 256)  0           bn4f_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4f_branch2b (Conv2D)         (None, 14, 14, 256)  590080      activation_38[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4f_branch2b (BatchNormalizati (None, 14, 14, 256)  1024        res4f_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_39 (Activation)      (None, 14, 14, 256)  0           bn4f_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res4f_branch2c (Conv2D)         (None, 14, 14, 1024) 263168      activation_39[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn4f_branch2c (BatchNormalizati (None, 14, 14, 1024) 4096        res4f_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_13 (Add)                    (None, 14, 14, 1024) 0           bn4f_branch2c[0][0]              \n",
      "                                                                 activation_37[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_40 (Activation)      (None, 14, 14, 1024) 0           add_13[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res5a_branch2a (Conv2D)         (None, 7, 7, 512)    524800      activation_40[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5a_branch2a (BatchNormalizati (None, 7, 7, 512)    2048        res5a_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_41 (Activation)      (None, 7, 7, 512)    0           bn5a_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5a_branch2b (Conv2D)         (None, 7, 7, 512)    2359808     activation_41[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5a_branch2b (BatchNormalizati (None, 7, 7, 512)    2048        res5a_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_42 (Activation)      (None, 7, 7, 512)    0           bn5a_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5a_branch2c (Conv2D)         (None, 7, 7, 2048)   1050624     activation_42[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5a_branch1 (Conv2D)          (None, 7, 7, 2048)   2099200     activation_40[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5a_branch2c (BatchNormalizati (None, 7, 7, 2048)   8192        res5a_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "bn5a_branch1 (BatchNormalizatio (None, 7, 7, 2048)   8192        res5a_branch1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "add_14 (Add)                    (None, 7, 7, 2048)   0           bn5a_branch2c[0][0]              \n",
      "                                                                 bn5a_branch1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_43 (Activation)      (None, 7, 7, 2048)   0           add_14[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res5b_branch2a (Conv2D)         (None, 7, 7, 512)    1049088     activation_43[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5b_branch2a (BatchNormalizati (None, 7, 7, 512)    2048        res5b_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_44 (Activation)      (None, 7, 7, 512)    0           bn5b_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5b_branch2b (Conv2D)         (None, 7, 7, 512)    2359808     activation_44[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5b_branch2b (BatchNormalizati (None, 7, 7, 512)    2048        res5b_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_45 (Activation)      (None, 7, 7, 512)    0           bn5b_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5b_branch2c (Conv2D)         (None, 7, 7, 2048)   1050624     activation_45[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5b_branch2c (BatchNormalizati (None, 7, 7, 2048)   8192        res5b_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_15 (Add)                    (None, 7, 7, 2048)   0           bn5b_branch2c[0][0]              \n",
      "                                                                 activation_43[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_46 (Activation)      (None, 7, 7, 2048)   0           add_15[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "res5c_branch2a (Conv2D)         (None, 7, 7, 512)    1049088     activation_46[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5c_branch2a (BatchNormalizati (None, 7, 7, 512)    2048        res5c_branch2a[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_47 (Activation)      (None, 7, 7, 512)    0           bn5c_branch2a[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5c_branch2b (Conv2D)         (None, 7, 7, 512)    2359808     activation_47[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5c_branch2b (BatchNormalizati (None, 7, 7, 512)    2048        res5c_branch2b[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "activation_48 (Activation)      (None, 7, 7, 512)    0           bn5c_branch2b[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "res5c_branch2c (Conv2D)         (None, 7, 7, 2048)   1050624     activation_48[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "bn5c_branch2c (BatchNormalizati (None, 7, 7, 2048)   8192        res5c_branch2c[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "add_16 (Add)                    (None, 7, 7, 2048)   0           bn5c_branch2c[0][0]              \n",
      "                                                                 activation_46[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "activation_49 (Activation)      (None, 7, 7, 2048)   0           add_16[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "avg_pool (AveragePooling2D)     (None, 1, 1, 2048)   0           activation_49[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 2048)         0           avg_pool[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 2048)         0           flatten_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 1)            2049        dropout_1[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 23,589,761\n",
      "Trainable params: 14,978,049\n",
      "Non-trainable params: 8,611,712\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "Resmodel=resmodel(lr = 0.0001)\n",
    "def fitmodel(model,X_train,y_train,X_valid,y_valid,augment = False,with_angle = False,epochs = 50,batch_size = 32,filepath = \".model_weights.hdf5\"):\n",
    "    if augment == False :\n",
    "        result = model.fit(X_train, y_train,batch_size=batch_size,epochs=epochs,verbose=1,shuffle = True,\n",
    "                           validation_data=(X_valid, y_valid),callbacks=get_callbacks(filepath))\n",
    "    else:\n",
    "        if with_angle==False:\n",
    "            result = model.fit_generator(data_augmentation(X_train,y_train,batch_size = batch_size),\n",
    "                                         steps_per_epoch = len(X_train_cv)/batch_size,epochs=epochs,verbose=1,shuffle = True,\n",
    "                                         validation_data=(X_valid, y_valid),callbacks=get_callbacks(filepath))\n",
    "        else:\n",
    "            result = model.fit_generator(gen_with_angle(X_train,X_train_other,y_train,batch_size = batch_size),\n",
    "                                         steps_per_epoch = len(X_train_cv)/batch_size,epochs=epochs,verbose=1,shuffle = True,\n",
    "                                         validation_data=([X_valid,X_valid_other], y_valid),callbacks=get_callbacks(filepath))\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "81/80 [==============================] - 75s 929ms/step - loss: 0.3077 - acc: 0.8722 - val_loss: 0.3662 - val_acc: 0.8318\n",
      "Epoch 2/100\n",
      "81/80 [==============================] - 74s 916ms/step - loss: 0.2444 - acc: 0.8950 - val_loss: 0.3362 - val_acc: 0.8505\n",
      "Epoch 3/100\n",
      "81/80 [==============================] - 75s 930ms/step - loss: 0.2716 - acc: 0.8894 - val_loss: 0.3262 - val_acc: 0.8536\n",
      "Epoch 4/100\n",
      "81/80 [==============================] - 74s 908ms/step - loss: 0.2425 - acc: 0.8971 - val_loss: 0.3226 - val_acc: 0.8536\n",
      "Epoch 5/100\n",
      "80/80 [============================>.] - ETA: 0s - loss: 0.2470 - acc: 0.8953"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-18-3803646e9c58>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mresult_Res\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfitmodel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mResmodel\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mX_train_cv\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my_train_cv\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mX_valid\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my_valid\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0maugment\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mwith_angle\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mepochs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m100\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mbatch_size\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m16\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mfilepath\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'model save/Model 5-ResNet/first.hdf5'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-16-3c62f73fd8af>\u001b[0m in \u001b[0;36mfitmodel\u001b[1;34m(model, X_train, y_train, X_valid, y_valid, augment, with_angle, epochs, batch_size, filepath)\u001b[0m\n\u001b[0;32m      8\u001b[0m             result = model.fit_generator(data_augmentation(X_train,y_train,batch_size = batch_size),\n\u001b[0;32m      9\u001b[0m                                          \u001b[0msteps_per_epoch\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mepochs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mepochs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mshuffle\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 10\u001b[1;33m                                          validation_data=(X_valid, y_valid),callbacks=get_callbacks(filepath))\n\u001b[0m\u001b[0;32m     11\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m             result = model.fit_generator(gen_with_angle(X_train,X_train_other,y_train,batch_size = batch_size),\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\keras\\legacy\\interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     85\u001b[0m                 warnings.warn('Update your `' + object_name +\n\u001b[0;32m     86\u001b[0m                               '` call to the Keras 2 API: ' + signature, stacklevel=2)\n\u001b[1;32m---> 87\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     88\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     89\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[1;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[0;32m   2142\u001b[0m                                 \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2143\u001b[0m                                 \u001b[0msample_weight\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mval_sample_weights\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2144\u001b[1;33m                                 verbose=0)\n\u001b[0m\u001b[0;32m   2145\u001b[0m                         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mval_outs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2146\u001b[0m                             \u001b[0mval_outs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mval_outs\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36mevaluate\u001b[1;34m(self, x, y, batch_size, verbose, sample_weight, steps)\u001b[0m\n\u001b[0;32m   1725\u001b[0m                                \u001b[0mbatch_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1726\u001b[0m                                \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1727\u001b[1;33m                                steps=steps)\n\u001b[0m\u001b[0;32m   1728\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1729\u001b[0m     def predict(self, x,\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\keras\\engine\\training.py\u001b[0m in \u001b[0;36m_test_loop\u001b[1;34m(self, f, ins, batch_size, verbose, steps)\u001b[0m\n\u001b[0;32m   1368\u001b[0m                     \u001b[0mins_batch\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_slice_arrays\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_ids\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1369\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1370\u001b[1;33m                 \u001b[0mbatch_outs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1371\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1372\u001b[0m                     \u001b[1;32mif\u001b[0m \u001b[0mbatch_index\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, inputs)\u001b[0m\n\u001b[0;32m   2350\u001b[0m         \u001b[0msession\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2351\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[1;32m-> 2352\u001b[1;33m                               **self.session_kwargs)\n\u001b[0m\u001b[0;32m   2353\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2354\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    887\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    888\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 889\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    890\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    891\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1118\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1119\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1120\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1121\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1122\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1315\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1316\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[1;32m-> 1317\u001b[1;33m                            options, run_metadata)\n\u001b[0m\u001b[0;32m   1318\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1319\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1321\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1322\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1323\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1324\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1325\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1300\u001b[0m           return tf_session.TF_Run(session, options,\n\u001b[0;32m   1301\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1302\u001b[1;33m                                    status, run_metadata)\n\u001b[0m\u001b[0;32m   1303\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1304\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "result_Res = fitmodel(Resmodel,X_train_cv,y_train_cv,X_valid,y_valid,augment = True,with_angle=False,epochs = 100,batch_size = 16,filepath = 'model save/Model 5-ResNet/first.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# vgg = VGG19(weights='imagenet')\n",
    "# model = Model(inputs=base_model.input, outputs=base_model.get_layer('block4_pool').output)\n",
    "\n",
    "# img_path = 'elephant.jpg'\n",
    "# img = image.load_img(img_path, target_size=(224, 224))\n",
    "# x = image.img_to_array(img)\n",
    "# x = np.expand_dims(x, axis=0)\n",
    "# x = preprocess_input(x)\n",
    "\n",
    "# block4_pool_features = model.predict(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_train=train['is_iceberg']\n",
    "test['inc_angle']=pd.to_numeric(test['inc_angle'], errors='coerce')\n",
    "train['inc_angle']=pd.to_numeric(train['inc_angle'], errors='coerce')#We have only 133 NAs.\n",
    "train['inc_angle']=train['inc_angle'].fillna(method='pad')\n",
    "X_angle=train['inc_angle']\n",
    "test['inc_angle']=pd.to_numeric(test['inc_angle'], errors='coerce')\n",
    "X_test_angle=test['inc_angle']\n",
    "\n",
    "#Generate the training data\n",
    "X_band_1=np.array([np.array(band).astype(np.float32).reshape(75, 75) for band in train[\"band_1\"]])\n",
    "X_band_2=np.array([np.array(band).astype(np.float32).reshape(75, 75) for band in train[\"band_2\"]])\n",
    "X_band_3=(X_band_1+X_band_2)/2\n",
    "#X_band_3=np.array([np.full((75, 75), angel).astype(np.float32) for angel in train[\"inc_angle\"]])\n",
    "X_train = np.concatenate([X_band_1[:, :, :, np.newaxis]\n",
    "                          , X_band_2[:, :, :, np.newaxis]\n",
    "                         , X_band_3[:, :, :, np.newaxis]], axis=-1)\n",
    "\n",
    "\n",
    "\n",
    "X_band_test_1=np.array([np.array(band).astype(np.float32).reshape(75, 75) for band in test[\"band_1\"]])\n",
    "X_band_test_2=np.array([np.array(band).astype(np.float32).reshape(75, 75) for band in test[\"band_2\"]])\n",
    "X_band_test_3=(X_band_test_1+X_band_test_2)/2\n",
    "#X_band_test_3=np.array([np.full((75, 75), angel).astype(np.float32) for angel in test[\"inc_angle\"]])\n",
    "X_test = np.concatenate([X_band_test_1[:, :, :, np.newaxis]\n",
    "                          , X_band_test_2[:, :, :, np.newaxis]\n",
    "                         , X_band_test_3[:, :, :, np.newaxis]], axis=-1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "batch_size=64\n",
    "# Define the image transformations here\n",
    "gen = ImageDataGenerator(horizontal_flip = True,\n",
    "                         vertical_flip = True,\n",
    "                         width_shift_range = 0.,\n",
    "                         height_shift_range = 0.,\n",
    "                         channel_shift_range=0,\n",
    "                         zoom_range = 0.2,\n",
    "                         rotation_range = 10)\n",
    "\n",
    "# Here is the function that merges our two generators\n",
    "# We use the exact same generator with the same random seed for both the y and angle arrays\n",
    "def gen_flow_for_two_inputs(X1, X2, y):\n",
    "    genX1 = gen.flow(X1,y,  batch_size=batch_size)\n",
    "    genX2 = gen.flow(X1,X2, batch_size=batch_size)\n",
    "    while True:\n",
    "            X1i = genX1.next()\n",
    "            X2i = genX2.next()\n",
    "            #Assert arrays are equal - this was for peace of mind, but slows down training\n",
    "            #np.testing.assert_array_equal(X1i[0],X2i[0])\n",
    "            yield [X1i[0], X2i[1]], X1i[1]\n",
    "\n",
    "# Finally create generator\n",
    "def get_callbacks(filepath, patience=2):\n",
    "    es = EarlyStopping('val_loss', patience=10, mode=\"min\")\n",
    "    msave = ModelCheckpoint(filepath, save_best_only=True)\n",
    "    return [es, msave]\n",
    "\n",
    "\n",
    "def getVggAngleModel():\n",
    "    input_2 = Input(shape=[1], name=\"angle\")\n",
    "    angle_layer = Dense(1, )(input_2)\n",
    "    base_model = VGG16(weights='imagenet', include_top=False, \n",
    "                 input_shape=X_train.shape[1:], classes=1)\n",
    "    x = base_model.get_layer('block5_pool').output\n",
    "    \n",
    "\n",
    "    x = GlobalMaxPooling2D()(x)\n",
    "    merge_one = concatenate([x, angle_layer])\n",
    "    merge_one = Dense(512, activation='relu', name='fc2')(merge_one)\n",
    "    merge_one = Dropout(0.3)(merge_one)\n",
    "    merge_one = Dense(512, activation='relu', name='fc3')(merge_one)\n",
    "    merge_one = Dropout(0.3)(merge_one)\n",
    "    \n",
    "    predictions = Dense(1, activation='sigmoid')(merge_one)\n",
    "    \n",
    "    model = Model(input=[base_model.input, input_2], output=predictions)\n",
    "    \n",
    "    sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "    model.compile(loss='binary_crossentropy',\n",
    "                  optimizer=sgd,\n",
    "                  metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "\n",
    "#Using K-fold Cross Validation with Data Augmentation.\n",
    "def myAngleCV(X_train, X_angle, X_test):\n",
    "    K=3\n",
    "    folds = list(StratifiedKFold(n_splits=K, shuffle=True).split(X_train, target_train))\n",
    "    y_test_pred_log = 0\n",
    "    y_train_pred_log=0\n",
    "    y_valid_pred_log = 0.0*target_train\n",
    "    for j, (train_idx, test_idx) in enumerate(folds):\n",
    "        print('\\n===================FOLD=',j)\n",
    "        X_train_cv = X_train[train_idx]\n",
    "        y_train_cv = target_train[train_idx]\n",
    "        X_holdout = X_train[test_idx]\n",
    "        Y_holdout= target_train[test_idx]\n",
    "        \n",
    "        #Angle\n",
    "        X_angle_cv=X_angle[train_idx]\n",
    "        X_angle_hold=X_angle[test_idx]\n",
    "\n",
    "        #define file path and get callbacks\n",
    "        file_path = \"%s_aug_model_weights.hdf5\"%j\n",
    "        callbacks = get_callbacks(filepath=file_path, patience=5)\n",
    "        gen_flow = gen_flow_for_two_inputs(X_train_cv, X_angle_cv, y_train_cv)\n",
    "        galaxyModel= getVggAngleModel()\n",
    "        galaxyModel.fit_generator(\n",
    "                gen_flow,\n",
    "                steps_per_epoch=24,\n",
    "                epochs=100,\n",
    "                shuffle=True,\n",
    "                verbose=1,\n",
    "                validation_data=([X_holdout,X_angle_hold], Y_holdout),\n",
    "                callbacks=callbacks)\n",
    "\n",
    "        #Getting the Best Model\n",
    "        galaxyModel.load_weights(filepath=file_path)\n",
    "        #Getting Training Score\n",
    "        score = galaxyModel.evaluate([X_train_cv,X_angle_cv], y_train_cv, verbose=0)\n",
    "        print('Train loss:', score[0])\n",
    "        print('Train accuracy:', score[1])\n",
    "        #Getting Test Score\n",
    "        score = galaxyModel.evaluate([X_holdout,X_angle_hold], Y_holdout, verbose=0)\n",
    "        print('Test loss:', score[0])\n",
    "        print('Test accuracy:', score[1])\n",
    "\n",
    "        #Getting validation Score.\n",
    "        pred_valid=galaxyModel.predict([X_holdout,X_angle_hold])\n",
    "        y_valid_pred_log[test_idx] = pred_valid.reshape(pred_valid.shape[0])\n",
    "\n",
    "        #Getting Test Scores\n",
    "        temp_test=galaxyModel.predict([X_test, X_test_angle])\n",
    "        y_test_pred_log+=temp_test.reshape(temp_test.shape[0])\n",
    "\n",
    "        #Getting Train Scores\n",
    "        temp_train=galaxyModel.predict([X_train, X_angle])\n",
    "        y_train_pred_log+=temp_train.reshape(temp_train.shape[0])\n",
    "\n",
    "    y_test_pred_log=y_test_pred_log/K\n",
    "    y_train_pred_log=y_train_pred_log/K\n",
    "\n",
    "    print('\\n Train Log Loss Validation= ',log_loss(target_train, y_train_pred_log))\n",
    "    print(' Test Log Loss Validation= ',log_loss(target_train, y_valid_pred_log))\n",
    "    return y_test_pred_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "===================FOLD= 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\yltbe\\anaconda3\\envs\\tensorflow_gpu\\lib\\site-packages\\ipykernel_launcher.py:48: UserWarning: Update your `Model` call to the Keras 2 API: `Model(inputs=[<tf.Tenso..., outputs=Tensor(\"de...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "24/24 [==============================] - 32s 1s/step - loss: 0.6953 - acc: 0.5833 - val_loss: 0.4926 - val_acc: 0.7495\n",
      "Epoch 2/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.5155 - acc: 0.7449 - val_loss: 0.3744 - val_acc: 0.8336\n",
      "Epoch 3/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.3888 - acc: 0.8194 - val_loss: 0.3639 - val_acc: 0.8430\n",
      "Epoch 4/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.3237 - acc: 0.8450 - val_loss: 0.2685 - val_acc: 0.8673\n",
      "Epoch 5/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.3022 - acc: 0.8698 - val_loss: 0.2496 - val_acc: 0.8860\n",
      "Epoch 6/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2476 - acc: 0.8973 - val_loss: 0.3560 - val_acc: 0.8168\n",
      "Epoch 7/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2658 - acc: 0.8857 - val_loss: 0.2267 - val_acc: 0.9065\n",
      "Epoch 8/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2364 - acc: 0.9013 - val_loss: 0.2282 - val_acc: 0.9047\n",
      "Epoch 9/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2382 - acc: 0.9020 - val_loss: 0.3160 - val_acc: 0.8449\n",
      "Epoch 10/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2346 - acc: 0.8958 - val_loss: 0.2820 - val_acc: 0.8654\n",
      "Epoch 11/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2306 - acc: 0.8989 - val_loss: 0.2318 - val_acc: 0.9084\n",
      "Epoch 12/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2253 - acc: 0.9010 - val_loss: 0.2941 - val_acc: 0.8748\n",
      "Epoch 13/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1938 - acc: 0.9184 - val_loss: 0.2277 - val_acc: 0.9065\n",
      "Epoch 14/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2057 - acc: 0.9123 - val_loss: 0.2440 - val_acc: 0.8860\n",
      "Epoch 15/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1971 - acc: 0.9173 - val_loss: 0.2543 - val_acc: 0.8860\n",
      "Epoch 16/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1815 - acc: 0.9239 - val_loss: 0.2425 - val_acc: 0.8991\n",
      "Epoch 17/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1830 - acc: 0.9202 - val_loss: 0.2433 - val_acc: 0.9215\n",
      "Train loss: 0.191277268392\n",
      "Train accuracy: 0.920486435977\n",
      "Test loss: 0.226746345625\n",
      "Test accuracy: 0.906542055295\n",
      "\n",
      "===================FOLD= 1\n",
      "Epoch 1/100\n",
      "24/24 [==============================] - 33s 1s/step - loss: 0.7420 - acc: 0.5786 - val_loss: 0.4296 - val_acc: 0.7533\n",
      "Epoch 2/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.4794 - acc: 0.7551 - val_loss: 0.3174 - val_acc: 0.8430\n",
      "Epoch 3/100\n",
      "24/24 [==============================] - 30s 1s/step - loss: 0.3393 - acc: 0.8402 - val_loss: 0.2835 - val_acc: 0.8785\n",
      "Epoch 4/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2822 - acc: 0.8667 - val_loss: 0.2645 - val_acc: 0.8879\n",
      "Epoch 5/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2914 - acc: 0.8724 - val_loss: 0.2177 - val_acc: 0.9028\n",
      "Epoch 6/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2624 - acc: 0.8994 - val_loss: 0.2534 - val_acc: 0.8879\n",
      "Epoch 7/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2494 - acc: 0.8962 - val_loss: 0.2460 - val_acc: 0.8935\n",
      "Epoch 8/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2442 - acc: 0.8953 - val_loss: 0.2444 - val_acc: 0.8897\n",
      "Epoch 9/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2104 - acc: 0.9114 - val_loss: 0.2217 - val_acc: 0.9028\n",
      "Epoch 10/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2072 - acc: 0.9134 - val_loss: 0.2195 - val_acc: 0.9028\n",
      "Epoch 11/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2023 - acc: 0.9234 - val_loss: 0.2049 - val_acc: 0.9121\n",
      "Epoch 12/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1942 - acc: 0.9176 - val_loss: 0.2156 - val_acc: 0.9047\n",
      "Epoch 13/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1907 - acc: 0.9278 - val_loss: 0.2116 - val_acc: 0.9009\n",
      "Epoch 14/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1940 - acc: 0.9208 - val_loss: 0.2098 - val_acc: 0.9103\n",
      "Epoch 15/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1828 - acc: 0.9278 - val_loss: 0.2122 - val_acc: 0.9084\n",
      "Epoch 16/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1811 - acc: 0.9337 - val_loss: 0.2065 - val_acc: 0.9103\n",
      "Epoch 17/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1822 - acc: 0.9240 - val_loss: 0.2098 - val_acc: 0.8953\n",
      "Epoch 18/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1593 - acc: 0.9386 - val_loss: 0.2295 - val_acc: 0.9084\n",
      "Epoch 19/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1644 - acc: 0.9369 - val_loss: 0.2043 - val_acc: 0.9121\n",
      "Epoch 20/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1574 - acc: 0.9377 - val_loss: 0.2098 - val_acc: 0.9028\n",
      "Epoch 21/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1457 - acc: 0.9461 - val_loss: 0.2362 - val_acc: 0.9047\n",
      "Epoch 22/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1465 - acc: 0.9435 - val_loss: 0.2690 - val_acc: 0.8953\n",
      "Epoch 23/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1453 - acc: 0.9395 - val_loss: 0.1991 - val_acc: 0.9103\n",
      "Epoch 24/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1205 - acc: 0.9541 - val_loss: 0.1905 - val_acc: 0.9103\n",
      "Epoch 25/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1236 - acc: 0.9498 - val_loss: 0.2218 - val_acc: 0.9103\n",
      "Epoch 26/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1369 - acc: 0.9531 - val_loss: 0.2603 - val_acc: 0.8897\n",
      "Epoch 27/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1346 - acc: 0.9455 - val_loss: 0.1952 - val_acc: 0.9103\n",
      "Epoch 28/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1131 - acc: 0.9539 - val_loss: 0.2873 - val_acc: 0.8953\n",
      "Epoch 29/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1262 - acc: 0.9526 - val_loss: 0.2304 - val_acc: 0.8991\n",
      "Epoch 30/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1333 - acc: 0.9422 - val_loss: 0.2704 - val_acc: 0.9009\n",
      "Epoch 31/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1111 - acc: 0.9552 - val_loss: 0.2433 - val_acc: 0.9159\n",
      "Epoch 32/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1064 - acc: 0.9527 - val_loss: 0.2664 - val_acc: 0.9084\n",
      "Epoch 33/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1063 - acc: 0.9633 - val_loss: 0.2843 - val_acc: 0.8972\n",
      "Epoch 34/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.0898 - acc: 0.9630 - val_loss: 0.2328 - val_acc: 0.9140\n",
      "Train loss: 0.0764470952402\n",
      "Train accuracy: 0.971936389149\n",
      "Test loss: 0.190533665073\n",
      "Test accuracy: 0.910280373498\n",
      "\n",
      "===================FOLD= 2\n",
      "Epoch 1/100\n",
      "24/24 [==============================] - 34s 1s/step - loss: 0.6442 - acc: 0.6121 - val_loss: 0.3414 - val_acc: 0.8371\n",
      "Epoch 2/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.4116 - acc: 0.7967 - val_loss: 0.3091 - val_acc: 0.8614\n",
      "Epoch 3/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2955 - acc: 0.8677 - val_loss: 0.2756 - val_acc: 0.8839\n",
      "Epoch 4/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2573 - acc: 0.8843 - val_loss: 0.3537 - val_acc: 0.8502\n",
      "Epoch 5/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2558 - acc: 0.8811 - val_loss: 0.3268 - val_acc: 0.8633\n",
      "Epoch 6/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2286 - acc: 0.8992 - val_loss: 0.2980 - val_acc: 0.8895\n",
      "Epoch 7/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2243 - acc: 0.9063 - val_loss: 0.3067 - val_acc: 0.8652\n",
      "Epoch 8/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2076 - acc: 0.9112 - val_loss: 0.2605 - val_acc: 0.9101\n",
      "Epoch 9/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.2108 - acc: 0.9109 - val_loss: 0.2778 - val_acc: 0.8876\n",
      "Epoch 10/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1868 - acc: 0.9220 - val_loss: 0.2465 - val_acc: 0.9082\n",
      "Epoch 11/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1913 - acc: 0.9203 - val_loss: 0.2766 - val_acc: 0.9026\n",
      "Epoch 12/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1814 - acc: 0.9213 - val_loss: 0.2627 - val_acc: 0.9082\n",
      "Epoch 13/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1596 - acc: 0.9313 - val_loss: 0.3071 - val_acc: 0.8970\n",
      "Epoch 14/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1720 - acc: 0.9267 - val_loss: 0.2607 - val_acc: 0.9101\n",
      "Epoch 15/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1782 - acc: 0.9283 - val_loss: 0.2563 - val_acc: 0.9026\n",
      "Epoch 16/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1720 - acc: 0.9340 - val_loss: 0.2516 - val_acc: 0.9120\n",
      "Epoch 17/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1547 - acc: 0.9409 - val_loss: 0.2480 - val_acc: 0.9082\n",
      "Epoch 18/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1553 - acc: 0.9323 - val_loss: 0.2714 - val_acc: 0.9120\n",
      "Epoch 19/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1529 - acc: 0.9277 - val_loss: 0.2778 - val_acc: 0.9045\n",
      "Epoch 20/100\n",
      "24/24 [==============================] - 29s 1s/step - loss: 0.1637 - acc: 0.9287 - val_loss: 0.2640 - val_acc: 0.9120\n",
      "Train loss: 0.151602413777\n",
      "Train accuracy: 0.937383177682\n",
      "Test loss: 0.246453694302\n",
      "Test accuracy: 0.908239700821\n",
      "\n",
      " Train Log Loss Validation=  0.145135322851\n",
      " Test Log Loss Validation=  0.221228850137\n"
     ]
    }
   ],
   "source": [
    "preds=myAngleCV(X_train, X_angle, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.17434557,  0.33020672,  0.01869651, ...,  0.0418326 ,\n",
       "        0.9970749 ,  0.01072063], dtype=float32)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "p = pd.read_csv('sub.csv')\n",
    "submission = pd.DataFrame()\n",
    "submission['id']=p['id']\n",
    "submission['is_iceberg']=preds\n",
    "submission.to_csv('sub1.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}