u-net-all_multi-graph.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### U-Net architecture\n",
    "\n",
    "See [here](https://github.com/jocicmarko/ultrasound-nerve-segmentation/blob/master/train.py#L19) for code and [here](https://arxiv.org/pdf/1505.04597.pdf) for the original literature."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def restart_kernel(restart=False):\n",
    "    import IPython\n",
    "    app = IPython.Application.instance()\n",
    "    if restart:\n",
    "        app.kernel.do_shutdown(True)\n",
    "\n",
    "restart_kernel(False)\n",
    "\n",
    "import os\n",
    "os.environ[\"THEANO_FLAGS\"] = \"optimizer_excluding='inplace_opt',lib.cnmem=0\"#,profile='True'\"\n",
    "#os.environ[\"CUDA_LAUNCH_BLOCKING\"] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n",
      "Using gpu device 0: Tesla K80 (CNMeM is disabled, cuDNN 5105)\n",
      "/home/ubuntu/anaconda3/lib/python3.5/site-packages/theano/sandbox/cuda/__init__.py:600: UserWarning: Your cuDNN version is more recent than the one Theano officially supports. If you see any problems, try updating Theano or downgrading cuDNN to version 5.\n",
      "  warnings.warn(warn)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from keras.models import Model\n",
    "from keras.layers import Reshape, Input, merge, Activation, Convolution2D, MaxPooling2D, UpSampling2D, ZeroPadding2D, Cropping2D, BatchNormalization, Dropout\n",
    "from keras.optimizers import Adadelta, Adam\n",
    "from keras.callbacks import CSVLogger, ModelCheckpoint, EarlyStopping, TensorBoard, ReduceLROnPlateau\n",
    "from keras import backend as K\n",
    "from keras.regularizers import l2\n",
    "from keras.constraints import maxnorm\n",
    "from keras.layers.advanced_activations import LeakyReLU\n",
    "\n",
    "K.set_image_dim_ordering('th')  # Theano dimension ordering in this code\n",
    "# \"tf\" assumes (rows, cols, channels) while \"th\" assumes (channels, rows, cols)\n",
    "# Possibly change this around natively in the data so the backend doesn't have to switch them\n",
    "# Only necessary if I use TF!\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "from pushbullet import Pushbullet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is run # 104\n"
     ]
    }
   ],
   "source": [
    "# Pushbullet notifier\n",
    "def push(title='Done!',text=''):\n",
    "    Pushbullet('o.YFPNNPfGRekivaCGHa4qMSgjZt8zJ6FL').devices[0].push_note(title,text)\n",
    "    \n",
    "# Import the training data\n",
    "def import_data(img_h,img_w):\n",
    "    x = np.load('./data/training-data/tiled_patched_images_576_576.npy','r')\n",
    "    y = np.load('./data/training-data/tiled_patched_masks_576_576.npy','r')\n",
    "    #y = y.reshape((y.shape[0],y.shape[1],y.shape[2]*y.shape[3]))\n",
    "    '''\n",
    "    Classes:\n",
    "    0 Buildings - large building, residential, non-residential, fuel storage facility, fortified building\n",
    "    1 Misc. Manmade structures \n",
    "    2 Road \n",
    "    3 Track - poor/dirt/cart track, footpath/trail\n",
    "    4 Trees - woodland, hedgerows, groups of trees, standalone trees\n",
    "    5 Crops - contour ploughing/cropland, grain (wheat) crops, row (potatoes, turnips) crops\n",
    "    6 Waterway \n",
    "    7 Standing water\n",
    "    8 Vehicle Large - large vehicle (e.g. lorry, truck,bus), logistics vehicle\n",
    "    9 Vehicle Small - small vehicle (car, van), motorbike\n",
    "    '''    \n",
    "    return x, y\n",
    "\n",
    "img_h,img_w = 160,160\n",
    "x, y = import_data(img_h,img_w)\n",
    "\n",
    "# Increment the counter\n",
    "def counter():\n",
    "    run = np.load('./data/misc/run_counter.npy')\n",
    "    run += 1\n",
    "    np.save('./data/misc/run_counter.npy',run)\n",
    "    return run\n",
    "run = counter()\n",
    "\n",
    "# Set the counter to a specific value\n",
    "def set_counter(run):\n",
    "    run = run\n",
    "    np.save('./data/misc/run_counter.npy',run)\n",
    "    return run\n",
    "# Uncomment the next line to manually set the counter if something goes wrong\n",
    "run = set_counter(104)\n",
    "print('This is run # %i' %run)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 11.8 s, sys: 207 ms, total: 12 s\n",
      "Wall time: 12 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "def compiler(img_rows = x.shape[2],img_cols = x.shape[3],\n",
    "            nfilters = 64,activation = 'relu',init = 'he_normal',\n",
    "            lr=0.001,decay=0.0,p=[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]):\n",
    "    \n",
    "    def jaccard(y_true, y_pred,smooth=1.):\n",
    "        y_true_f = K.flatten(y_true)\n",
    "        y_pred_f = K.flatten(y_pred)\n",
    "        intersection = K.sum(y_true_f * y_pred_f)\n",
    "        return (intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) - intersection + smooth)\n",
    "    \n",
    "    def Conv2DReluBatchNorm(n_filter, w_filter, h_filter, inputs, activation, init='he_uniform',dropout=0.2):\n",
    "        return BatchNormalization(mode=2, axis=1)(Activation(activation=activation)((Convolution2D(n_filter, w_filter, h_filter, border_mode='same',init=init,W_constraint = maxnorm(3))(inputs))))\n",
    "        \n",
    "    def up_conv(nfilters,filter_factor,inputs,init=init,activation=activation):\n",
    "        return BatchNormalization(mode=2, axis=1)(Activation(activation=activation)(Convolution2D(nfilters*filter_factor, 2, 2, border_mode='same',init=init,W_constraint = maxnorm(3))(UpSampling2D(size=(2, 2))(inputs))))\n",
    "\n",
    "    # Universal inputs\n",
    "    inputs = Input((20, img_rows, img_cols))\n",
    "\n",
    "    # Model_0    \n",
    "    conv1_0 = Conv2DReluBatchNorm(nfilters, 3, 3, inputs, activation=activation,init=init)\n",
    "    conv1_0 = Conv2DReluBatchNorm(nfilters, 3, 3, conv1_0, activation=activation,init=init)\n",
    "    pool1_0 = MaxPooling2D(pool_size=(2, 2))(conv1_0)\n",
    "    pool1_0 = Dropout(p=p[0])(pool1_0)\n",
    "\n",
    "    conv2_0 = Conv2DReluBatchNorm(nfilters*2, 3, 3, pool1_0, activation=activation,init=init)\n",
    "    conv2_0 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv2_0, activation=activation,init=init)\n",
    "    pool2_0 = MaxPooling2D(pool_size=(2, 2))(conv2_0)\n",
    "    pool2_0 = Dropout(p=p[1])(pool2_0)\n",
    "\n",
    "    conv3_0 = Conv2DReluBatchNorm(nfilters*4, 3, 3, pool2_0, activation=activation,init=init)\n",
    "    conv3_0 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv3_0, activation=activation,init=init)\n",
    "    pool3_0 = MaxPooling2D(pool_size=(2, 2))(conv3_0)\n",
    "    pool3_0 = Dropout(p=p[2])(pool3_0)\n",
    "\n",
    "    conv4_0 = Conv2DReluBatchNorm(nfilters*8, 3, 3, pool3_0, activation=activation,init=init)\n",
    "    conv4_0 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv4_0, activation=activation,init=init)\n",
    "    pool4_0 = MaxPooling2D(pool_size=(2, 2))(conv4_0)\n",
    "    pool4_0 = Dropout(p=p[3])(pool4_0)\n",
    "\n",
    "    conv5_0 = Conv2DReluBatchNorm(nfilters*16, 3, 3, pool4_0, activation=activation,init=init)\n",
    "    conv5_0 = Conv2DReluBatchNorm(nfilters*16, 3, 3, conv5_0, activation=activation,init=init)\n",
    "    conv5_0 = Dropout(p=p[4])(conv5_0)\n",
    "        \n",
    "    up6_0 = merge([up_conv(nfilters,8,conv5_0), conv4_0], mode='concat', concat_axis=1)\n",
    "    conv6_0 = Conv2DReluBatchNorm(nfilters*8, 3, 3, up6_0, activation=activation,init=init)\n",
    "    conv6_0 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv6_0, activation=activation,init=init)\n",
    "    conv6_0 = Dropout(p=p[5])(conv6_0)\n",
    "\n",
    "    up7_0 = merge([up_conv(nfilters,4,conv6_0), conv3_0], mode='concat', concat_axis=1)\n",
    "    conv7_0 = Conv2DReluBatchNorm(nfilters*4, 3, 3, up7_0, activation=activation,init=init)\n",
    "    conv7_0 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv7_0, activation=activation,init=init)\n",
    "    conv7_0 = Dropout(p=p[6])(conv7_0)\n",
    "\n",
    "    up8_0 = merge([up_conv(nfilters,2,conv7_0), conv2_0], mode='concat', concat_axis=1)\n",
    "    conv8_0 = Conv2DReluBatchNorm(nfilters*2, 3, 3, up8_0, activation=activation,init=init)\n",
    "    conv8_0 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv8_0, activation=activation,init=init)\n",
    "    conv8_0 = Dropout(p=p[7])(conv8_0)\n",
    "\n",
    "    up9_0 = merge([up_conv(nfilters,1,conv8_0), conv1_0], mode='concat', concat_axis=1)\n",
    "    conv9_0 = Conv2DReluBatchNorm(nfilters, 3, 3, up9_0, activation=activation,init=init)\n",
    "    conv9_0 = Conv2DReluBatchNorm(nfilters, 3, 3, conv9_0, activation=activation,init=init)\n",
    "    conv9_0 = Dropout(p=p[8])(conv9_0)\n",
    "    \n",
    "    conv10_0 = Conv2DReluBatchNorm(1, 1, 1, conv9_0, activation='relu',init=init)\n",
    "    output_0 = Activation(activation='sigmoid',name='class_0')(conv10_0)\n",
    "\n",
    "    # Model_1\n",
    "    conv1_1 = Conv2DReluBatchNorm(nfilters, 3, 3, inputs, activation=activation,init=init)\n",
    "    conv1_1 = Conv2DReluBatchNorm(nfilters, 3, 3, conv1_1, activation=activation,init=init)\n",
    "    pool1_1 = MaxPooling2D(pool_size=(2, 2))(conv1_1)\n",
    "    pool1_1 = Dropout(p=p[0])(pool1_1)\n",
    "\n",
    "    conv2_1 = Conv2DReluBatchNorm(nfilters*2, 3, 3, pool1_1, activation=activation,init=init)\n",
    "    conv2_1 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv2_1, activation=activation,init=init)\n",
    "    pool2_1 = MaxPooling2D(pool_size=(2, 2))(conv2_1)\n",
    "    pool2_1 = Dropout(p=p[1])(pool2_1)\n",
    "\n",
    "    conv3_1 = Conv2DReluBatchNorm(nfilters*4, 3, 3, pool2_1, activation=activation,init=init)\n",
    "    conv3_1 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv3_1, activation=activation,init=init)\n",
    "    pool3_1 = MaxPooling2D(pool_size=(2, 2))(conv3_1)\n",
    "    pool3_1 = Dropout(p=p[2])(pool3_1)\n",
    "\n",
    "    conv4_1 = Conv2DReluBatchNorm(nfilters*8, 3, 3, pool3_1, activation=activation,init=init)\n",
    "    conv4_1 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv4_1, activation=activation,init=init)\n",
    "    pool4_1 = MaxPooling2D(pool_size=(2, 2))(conv4_1)\n",
    "    pool4_1 = Dropout(p=p[3])(pool4_1)\n",
    "\n",
    "    conv5_1 = Conv2DReluBatchNorm(nfilters*16, 3, 3, pool4_1, activation=activation,init=init)\n",
    "    conv5_1 = Conv2DReluBatchNorm(nfilters*16, 3, 3, conv5_1, activation=activation,init=init)\n",
    "    conv5_1 = Dropout(p=p[4])(conv5_1)\n",
    "        \n",
    "    up6_1 = merge([up_conv(nfilters,8,conv5_1), conv4_1], mode='concat', concat_axis=1)\n",
    "    conv6_1 = Conv2DReluBatchNorm(nfilters*8, 3, 3, up6_1, activation=activation,init=init)\n",
    "    conv6_1 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv6_1, activation=activation,init=init)\n",
    "    conv6_1 = Dropout(p=p[5])(conv6_1)\n",
    "\n",
    "    up7_1 = merge([up_conv(nfilters,4,conv6_1), conv3_1], mode='concat', concat_axis=1)\n",
    "    conv7_1 = Conv2DReluBatchNorm(nfilters*4, 3, 3, up7_1, activation=activation,init=init)\n",
    "    conv7_1 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv7_1, activation=activation,init=init)\n",
    "    conv7_1 = Dropout(p=p[6])(conv7_1)\n",
    "\n",
    "    up8_1 = merge([up_conv(nfilters,2,conv7_1), conv2_1], mode='concat', concat_axis=1)\n",
    "    conv8_1 = Conv2DReluBatchNorm(nfilters*2, 3, 3, up8_1, activation=activation,init=init)\n",
    "    conv8_1 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv8_1, activation=activation,init=init)\n",
    "    conv8_1 = Dropout(p=p[7])(conv8_1)\n",
    "\n",
    "    up9_1 = merge([up_conv(nfilters,1,conv8_1), conv1_1], mode='concat', concat_axis=1)\n",
    "    conv9_1 = Conv2DReluBatchNorm(nfilters, 3, 3, up9_1, activation=activation,init=init)\n",
    "    conv9_1 = Conv2DReluBatchNorm(nfilters, 3, 3, conv9_1, activation=activation,init=init)\n",
    "    conv9_1 = Dropout(p=p[8])(conv9_1)\n",
    "    \n",
    "    conv10_1 = Conv2DReluBatchNorm(1, 1, 1, conv9_1, activation='relu',init=init)\n",
    "    output_1 = Activation(activation='sigmoid',name='class_1')(conv10_1)    \n",
    "    \n",
    "    # Model_2\n",
    "    conv1_2 = Conv2DReluBatchNorm(nfilters, 3, 3, inputs, activation=activation,init=init)\n",
    "    conv1_2 = Conv2DReluBatchNorm(nfilters, 3, 3, conv1_2, activation=activation,init=init)\n",
    "    pool1_2 = MaxPooling2D(pool_size=(2, 2))(conv1_2)\n",
    "    pool1_2 = Dropout(p=p[0])(pool1_2)\n",
    "\n",
    "    conv2_2 = Conv2DReluBatchNorm(nfilters*2, 3, 3, pool1_2, activation=activation,init=init)\n",
    "    conv2_2 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv2_2, activation=activation,init=init)\n",
    "    pool2_2 = MaxPooling2D(pool_size=(2, 2))(conv2_2)\n",
    "    pool2_2 = Dropout(p=p[1])(pool2_2)\n",
    "\n",
    "    conv3_2 = Conv2DReluBatchNorm(nfilters*4, 3, 3, pool2_2, activation=activation,init=init)\n",
    "    conv3_2 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv3_2, activation=activation,init=init)\n",
    "    pool3_2 = MaxPooling2D(pool_size=(2, 2))(conv3_2)\n",
    "    pool3_2 = Dropout(p=p[2])(pool3_2)\n",
    "\n",
    "    conv4_2 = Conv2DReluBatchNorm(nfilters*8, 3, 3, pool3_2, activation=activation,init=init)\n",
    "    conv4_2 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv4_2, activation=activation,init=init)\n",
    "    pool4_2 = MaxPooling2D(pool_size=(2, 2))(conv4_2)\n",
    "    pool4_2 = Dropout(p=p[3])(pool4_2)\n",
    "\n",
    "    conv5_2 = Conv2DReluBatchNorm(nfilters*16, 3, 3, pool4_2, activation=activation,init=init)\n",
    "    conv5_2 = Conv2DReluBatchNorm(nfilters*16, 3, 3, conv5_2, activation=activation,init=init)\n",
    "    conv5_2 = Dropout(p=p[4])(conv5_2)\n",
    "        \n",
    "    up6_2 = merge([up_conv(nfilters,8,conv5_2), conv4_2], mode='concat', concat_axis=1)\n",
    "    conv6_2 = Conv2DReluBatchNorm(nfilters*8, 3, 3, up6_2, activation=activation,init=init)\n",
    "    conv6_2 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv6_2, activation=activation,init=init)\n",
    "    conv6_2 = Dropout(p=p[5])(conv6_2)\n",
    "\n",
    "    up7_2 = merge([up_conv(nfilters,4,conv6_2), conv3_2], mode='concat', concat_axis=1)\n",
    "    conv7_2 = Conv2DReluBatchNorm(nfilters*4, 3, 3, up7_2, activation=activation,init=init)\n",
    "    conv7_2 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv7_2, activation=activation,init=init)\n",
    "    conv7_2 = Dropout(p=p[6])(conv7_2)\n",
    "\n",
    "    up8_2 = merge([up_conv(nfilters,2,conv7_2), conv2_2], mode='concat', concat_axis=1)\n",
    "    conv8_2 = Conv2DReluBatchNorm(nfilters*2, 3, 3, up8_2, activation=activation,init=init)\n",
    "    conv8_2 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv8_2, activation=activation,init=init)\n",
    "    conv8_2 = Dropout(p=p[7])(conv8_2)\n",
    "\n",
    "    up9_2 = merge([up_conv(nfilters,1,conv8_2), conv1_2], mode='concat', concat_axis=1)\n",
    "    conv9_2 = Conv2DReluBatchNorm(nfilters, 3, 3, up9_2, activation=activation,init=init)\n",
    "    conv9_2 = Conv2DReluBatchNorm(nfilters, 3, 3, conv9_2, activation=activation,init=init)\n",
    "    conv9_2 = Dropout(p=p[8])(conv9_2)\n",
    "    \n",
    "    conv10_2 = Conv2DReluBatchNorm(1, 1, 1, conv9_2, activation='relu',init=init)\n",
    "    output_2 = Activation(activation='sigmoid',name='class_2')(conv10_2)\n",
    "\n",
    "    # Model_3\n",
    "    conv1_3 = Conv2DReluBatchNorm(nfilters, 3, 3, inputs, activation=activation,init=init)\n",
    "    conv1_3 = Conv2DReluBatchNorm(nfilters, 3, 3, conv1_3, activation=activation,init=init)\n",
    "    pool1_3 = MaxPooling2D(pool_size=(2, 2))(conv1_3)\n",
    "    pool1_3 = Dropout(p=p[0])(pool1_3)\n",
    "\n",
    "    conv2_3 = Conv2DReluBatchNorm(nfilters*2, 3, 3, pool1_3, activation=activation,init=init)\n",
    "    conv2_3 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv2_3, activation=activation,init=init)\n",
    "    pool2_3 = MaxPooling2D(pool_size=(2, 2))(conv2_3)\n",
    "    pool2_3 = Dropout(p=p[1])(pool2_3)\n",
    "\n",
    "    conv3_3 = Conv2DReluBatchNorm(nfilters*4, 3, 3, pool2_3, activation=activation,init=init)\n",
    "    conv3_3 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv3_3, activation=activation,init=init)\n",
    "    pool3_3 = MaxPooling2D(pool_size=(2, 2))(conv3_3)\n",
    "    pool3_3 = Dropout(p=p[2])(pool3_3)\n",
    "\n",
    "    conv4_3 = Conv2DReluBatchNorm(nfilters*8, 3, 3, pool3_3, activation=activation,init=init)\n",
    "    conv4_3 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv4_3, activation=activation,init=init)\n",
    "    pool4_3 = MaxPooling2D(pool_size=(2, 2))(conv4_3)\n",
    "    pool4_3 = Dropout(p=p[3])(pool4_3)\n",
    "\n",
    "    conv5_3 = Conv2DReluBatchNorm(nfilters*16, 3, 3, pool4_3, activation=activation,init=init)\n",
    "    conv5_3 = Conv2DReluBatchNorm(nfilters*16, 3, 3, conv5_3, activation=activation,init=init)\n",
    "    conv5_3 = Dropout(p=p[4])(conv5_3)\n",
    "        \n",
    "    up6_3 = merge([up_conv(nfilters,8,conv5_3), conv4_3], mode='concat', concat_axis=1)\n",
    "    conv6_3 = Conv2DReluBatchNorm(nfilters*8, 3, 3, up6_3, activation=activation,init=init)\n",
    "    conv6_3 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv6_3, activation=activation,init=init)\n",
    "    conv6_3 = Dropout(p=p[5])(conv6_3)\n",
    "\n",
    "    up7_3 = merge([up_conv(nfilters,4,conv6_3), conv3_3], mode='concat', concat_axis=1)\n",
    "    conv7_3 = Conv2DReluBatchNorm(nfilters*4, 3, 3, up7_3, activation=activation,init=init)\n",
    "    conv7_3 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv7_3, activation=activation,init=init)\n",
    "    conv7_3 = Dropout(p=p[6])(conv7_3)\n",
    "\n",
    "    up8_3 = merge([up_conv(nfilters,2,conv7_3), conv2_3], mode='concat', concat_axis=1)\n",
    "    conv8_3 = Conv2DReluBatchNorm(nfilters*2, 3, 3, up8_3, activation=activation,init=init)\n",
    "    conv8_3 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv8_3, activation=activation,init=init)\n",
    "    conv8_3 = Dropout(p=p[7])(conv8_3)\n",
    "\n",
    "    up9_3 = merge([up_conv(nfilters,1,conv8_3), conv1_3], mode='concat', concat_axis=1)\n",
    "    conv9_3 = Conv2DReluBatchNorm(nfilters, 3, 3, up9_3, activation=activation,init=init)\n",
    "    conv9_3 = Conv2DReluBatchNorm(nfilters, 3, 3, conv9_3, activation=activation,init=init)\n",
    "    conv9_3 = Dropout(p=p[8])(conv9_3)\n",
    "    \n",
    "    conv10_3 = Conv2DReluBatchNorm(1, 1, 1, conv9_3, activation='relu',init=init)\n",
    "    output_3 = Activation(activation='sigmoid',name='class_3')(conv10_3)\n",
    "\n",
    "    # Model_4\n",
    "    conv1_4 = Conv2DReluBatchNorm(nfilters, 3, 3, inputs, activation=activation,init=init)\n",
    "    conv1_4 = Conv2DReluBatchNorm(nfilters, 3, 3, conv1_4, activation=activation,init=init)\n",
    "    pool1_4 = MaxPooling2D(pool_size=(2, 2))(conv1_4)\n",
    "    pool1_4 = Dropout(p=p[0])(pool1_4)\n",
    "\n",
    "    conv2_4 = Conv2DReluBatchNorm(nfilters*2, 3, 3, pool1_4, activation=activation,init=init)\n",
    "    conv2_4 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv2_4, activation=activation,init=init)\n",
    "    pool2_4 = MaxPooling2D(pool_size=(2, 2))(conv2_4)\n",
    "    pool2_4 = Dropout(p=p[1])(pool2_4)\n",
    "\n",
    "    conv3_4 = Conv2DReluBatchNorm(nfilters*4, 3, 3, pool2_4, activation=activation,init=init)\n",
    "    conv3_4 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv3_4, activation=activation,init=init)\n",
    "    pool3_4 = MaxPooling2D(pool_size=(2, 2))(conv3_4)\n",
    "    pool3_4 = Dropout(p=p[2])(pool3_4)\n",
    "\n",
    "    conv4_4 = Conv2DReluBatchNorm(nfilters*8, 3, 3, pool3_4, activation=activation,init=init)\n",
    "    conv4_4 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv4_4, activation=activation,init=init)\n",
    "    pool4_4 = MaxPooling2D(pool_size=(2, 2))(conv4_4)\n",
    "    pool4_4 = Dropout(p=p[3])(pool4_4)\n",
    "\n",
    "    conv5_4 = Conv2DReluBatchNorm(nfilters*16, 3, 3, pool4_4, activation=activation,init=init)\n",
    "    conv5_4 = Conv2DReluBatchNorm(nfilters*16, 3, 3, conv5_4, activation=activation,init=init)\n",
    "    conv5_4 = Dropout(p=p[4])(conv5_4)\n",
    "        \n",
    "    up6_4 = merge([up_conv(nfilters,8,conv5_4), conv4_4], mode='concat', concat_axis=1)\n",
    "    conv6_4 = Conv2DReluBatchNorm(nfilters*8, 3, 3, up6_4, activation=activation,init=init)\n",
    "    conv6_4 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv6_4, activation=activation,init=init)\n",
    "    conv6_4 = Dropout(p=p[5])(conv6_4)\n",
    "\n",
    "    up7_4 = merge([up_conv(nfilters,4,conv6_4), conv3_4], mode='concat', concat_axis=1)\n",
    "    conv7_4 = Conv2DReluBatchNorm(nfilters*4, 3, 3, up7_4, activation=activation,init=init)\n",
    "    conv7_4 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv7_4, activation=activation,init=init)\n",
    "    conv7_4 = Dropout(p=p[6])(conv7_4)\n",
    "\n",
    "    up8_4 = merge([up_conv(nfilters,2,conv7_4), conv2_4], mode='concat', concat_axis=1)\n",
    "    conv8_4 = Conv2DReluBatchNorm(nfilters*2, 3, 3, up8_4, activation=activation,init=init)\n",
    "    conv8_4 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv8_4, activation=activation,init=init)\n",
    "    conv8_4 = Dropout(p=p[7])(conv8_4)\n",
    "\n",
    "    up9_4 = merge([up_conv(nfilters,1,conv8_4), conv1_4], mode='concat', concat_axis=1)\n",
    "    conv9_4 = Conv2DReluBatchNorm(nfilters, 3, 3, up9_4, activation=activation,init=init)\n",
    "    conv9_4 = Conv2DReluBatchNorm(nfilters, 3, 3, conv9_4, activation=activation,init=init)\n",
    "    conv9_4 = Dropout(p=p[8])(conv9_4)\n",
    "    \n",
    "    conv10_4 = Conv2DReluBatchNorm(1, 1, 1, conv9_4, activation='relu',init=init)\n",
    "    output_4 = Activation(activation='sigmoid',name='class_4')(conv10_4)\n",
    "\n",
    "    # Model_5\n",
    "    conv1_5 = Conv2DReluBatchNorm(nfilters, 3, 3, inputs, activation=activation,init=init)\n",
    "    conv1_5 = Conv2DReluBatchNorm(nfilters, 3, 3, conv1_5, activation=activation,init=init)\n",
    "    pool1_5 = MaxPooling2D(pool_size=(2, 2))(conv1_5)\n",
    "    pool1_5 = Dropout(p=p[0])(pool1_5)\n",
    "\n",
    "    conv2_5 = Conv2DReluBatchNorm(nfilters*2, 3, 3, pool1_5, activation=activation,init=init)\n",
    "    conv2_5 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv2_5, activation=activation,init=init)\n",
    "    pool2_5 = MaxPooling2D(pool_size=(2, 2))(conv2_5)\n",
    "    pool2_5 = Dropout(p=p[1])(pool2_5)\n",
    "\n",
    "    conv3_5 = Conv2DReluBatchNorm(nfilters*4, 3, 3, pool2_5, activation=activation,init=init)\n",
    "    conv3_5 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv3_5, activation=activation,init=init)\n",
    "    pool3_5 = MaxPooling2D(pool_size=(2, 2))(conv3_5)\n",
    "    pool3_5 = Dropout(p=p[2])(pool3_5)\n",
    "\n",
    "    conv4_5 = Conv2DReluBatchNorm(nfilters*8, 3, 3, pool3_5, activation=activation,init=init)\n",
    "    conv4_5 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv4_5, activation=activation,init=init)\n",
    "    pool4_5 = MaxPooling2D(pool_size=(2, 2))(conv4_5)\n",
    "    pool4_5 = Dropout(p=p[3])(pool4_5)\n",
    "\n",
    "    conv5_5 = Conv2DReluBatchNorm(nfilters*16, 3, 3, pool4_5, activation=activation,init=init)\n",
    "    conv5_5 = Conv2DReluBatchNorm(nfilters*16, 3, 3, conv5_5, activation=activation,init=init)\n",
    "    conv5_5 = Dropout(p=p[4])(conv5_5)\n",
    "        \n",
    "    up6_5 = merge([up_conv(nfilters,8,conv5_5), conv4_5], mode='concat', concat_axis=1)\n",
    "    conv6_5 = Conv2DReluBatchNorm(nfilters*8, 3, 3, up6_5, activation=activation,init=init)\n",
    "    conv6_5 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv6_5, activation=activation,init=init)\n",
    "    conv6_5 = Dropout(p=p[5])(conv6_5)\n",
    "\n",
    "    up7_5 = merge([up_conv(nfilters,4,conv6_5), conv3_5], mode='concat', concat_axis=1)\n",
    "    conv7_5 = Conv2DReluBatchNorm(nfilters*4, 3, 3, up7_5, activation=activation,init=init)\n",
    "    conv7_5 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv7_5, activation=activation,init=init)\n",
    "    conv7_5 = Dropout(p=p[6])(conv7_5)\n",
    "\n",
    "    up8_5 = merge([up_conv(nfilters,2,conv7_5), conv2_5], mode='concat', concat_axis=1)\n",
    "    conv8_5 = Conv2DReluBatchNorm(nfilters*2, 3, 3, up8_5, activation=activation,init=init)\n",
    "    conv8_5 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv8_5, activation=activation,init=init)\n",
    "    conv8_5 = Dropout(p=p[7])(conv8_5)\n",
    "\n",
    "    up9_5 = merge([up_conv(nfilters,1,conv8_5), conv1_5], mode='concat', concat_axis=1)\n",
    "    conv9_5 = Conv2DReluBatchNorm(nfilters, 3, 3, up9_5, activation=activation,init=init)\n",
    "    conv9_5 = Conv2DReluBatchNorm(nfilters, 3, 3, conv9_5, activation=activation,init=init)\n",
    "    conv9_5 = Dropout(p=p[8])(conv9_5)\n",
    "    \n",
    "    conv10_5 = Conv2DReluBatchNorm(1, 1, 1, conv9_5, activation='relu',init=init)\n",
    "    output_5 = Activation(activation='sigmoid',name='class_5')(conv10_5)\n",
    "\n",
    "    # Model_6\n",
    "    conv1_6 = Conv2DReluBatchNorm(nfilters, 3, 3, inputs, activation=activation,init=init)\n",
    "    conv1_6 = Conv2DReluBatchNorm(nfilters, 3, 3, conv1_6, activation=activation,init=init)\n",
    "    pool1_6 = MaxPooling2D(pool_size=(2, 2))(conv1_6)\n",
    "    pool1_6 = Dropout(p=p[0])(pool1_6)\n",
    "\n",
    "    conv2_6 = Conv2DReluBatchNorm(nfilters*2, 3, 3, pool1_6, activation=activation,init=init)\n",
    "    conv2_6 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv2_6, activation=activation,init=init)\n",
    "    pool2_6 = MaxPooling2D(pool_size=(2, 2))(conv2_6)\n",
    "    pool2_6 = Dropout(p=p[1])(pool2_6)\n",
    "\n",
    "    conv3_6 = Conv2DReluBatchNorm(nfilters*4, 3, 3, pool2_6, activation=activation,init=init)\n",
    "    conv3_6 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv3_6, activation=activation,init=init)\n",
    "    pool3_6 = MaxPooling2D(pool_size=(2, 2))(conv3_6)\n",
    "    pool3_6 = Dropout(p=p[2])(pool3_6)\n",
    "\n",
    "    conv4_6 = Conv2DReluBatchNorm(nfilters*8, 3, 3, pool3_6, activation=activation,init=init)\n",
    "    conv4_6 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv4_6, activation=activation,init=init)\n",
    "    pool4_6 = MaxPooling2D(pool_size=(2, 2))(conv4_6)\n",
    "    pool4_6 = Dropout(p=p[3])(pool4_6)\n",
    "\n",
    "    conv5_6 = Conv2DReluBatchNorm(nfilters*16, 3, 3, pool4_6, activation=activation,init=init)\n",
    "    conv5_6 = Conv2DReluBatchNorm(nfilters*16, 3, 3, conv5_6, activation=activation,init=init)\n",
    "    conv5_6 = Dropout(p=p[4])(conv5_6)\n",
    "        \n",
    "    up6_6 = merge([up_conv(nfilters,8,conv5_6), conv4_6], mode='concat', concat_axis=1)\n",
    "    conv6_6 = Conv2DReluBatchNorm(nfilters*8, 3, 3, up6_6, activation=activation,init=init)\n",
    "    conv6_6 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv6_6, activation=activation,init=init)\n",
    "    conv6_6 = Dropout(p=p[5])(conv6_6)\n",
    "\n",
    "    up7_6 = merge([up_conv(nfilters,4,conv6_6), conv3_6], mode='concat', concat_axis=1)\n",
    "    conv7_6 = Conv2DReluBatchNorm(nfilters*4, 3, 3, up7_6, activation=activation,init=init)\n",
    "    conv7_6 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv7_6, activation=activation,init=init)\n",
    "    conv7_6 = Dropout(p=p[6])(conv7_6)\n",
    "\n",
    "    up8_6 = merge([up_conv(nfilters,2,conv7_6), conv2_6], mode='concat', concat_axis=1)\n",
    "    conv8_6 = Conv2DReluBatchNorm(nfilters*2, 3, 3, up8_6, activation=activation,init=init)\n",
    "    conv8_6 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv8_6, activation=activation,init=init)\n",
    "    conv8_6 = Dropout(p=p[7])(conv8_6)\n",
    "\n",
    "    up9_6 = merge([up_conv(nfilters,1,conv8_6), conv1_6], mode='concat', concat_axis=1)\n",
    "    conv9_6 = Conv2DReluBatchNorm(nfilters, 3, 3, up9_6, activation=activation,init=init)\n",
    "    conv9_6 = Conv2DReluBatchNorm(nfilters, 3, 3, conv9_6, activation=activation,init=init)\n",
    "    conv9_6 = Dropout(p=p[8])(conv9_6)\n",
    "    \n",
    "    conv10_6 = Conv2DReluBatchNorm(1, 1, 1, conv9_6, activation='relu',init=init)\n",
    "    output_6 = Activation(activation='sigmoid',name='class_6')(conv10_6)\n",
    "\n",
    "    # Model_7\n",
    "    conv1_7 = Conv2DReluBatchNorm(nfilters, 3, 3, inputs, activation=activation,init=init)\n",
    "    conv1_7 = Conv2DReluBatchNorm(nfilters, 3, 3, conv1_7, activation=activation,init=init)\n",
    "    pool1_7 = MaxPooling2D(pool_size=(2, 2))(conv1_7)\n",
    "    pool1_7 = Dropout(p=p[0])(pool1_7)\n",
    "\n",
    "    conv2_7 = Conv2DReluBatchNorm(nfilters*2, 3, 3, pool1_7, activation=activation,init=init)\n",
    "    conv2_7 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv2_7, activation=activation,init=init)\n",
    "    pool2_7 = MaxPooling2D(pool_size=(2, 2))(conv2_7)\n",
    "    pool2_7 = Dropout(p=p[1])(pool2_7)\n",
    "\n",
    "    conv3_7 = Conv2DReluBatchNorm(nfilters*4, 3, 3, pool2_7, activation=activation,init=init)\n",
    "    conv3_7 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv3_7, activation=activation,init=init)\n",
    "    pool3_7 = MaxPooling2D(pool_size=(2, 2))(conv3_7)\n",
    "    pool3_7 = Dropout(p=p[2])(pool3_7)\n",
    "\n",
    "    conv4_7 = Conv2DReluBatchNorm(nfilters*8, 3, 3, pool3_7, activation=activation,init=init)\n",
    "    conv4_7 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv4_7, activation=activation,init=init)\n",
    "    pool4_7 = MaxPooling2D(pool_size=(2, 2))(conv4_7)\n",
    "    pool4_7 = Dropout(p=p[3])(pool4_7)\n",
    "\n",
    "    conv5_7 = Conv2DReluBatchNorm(nfilters*16, 3, 3, pool4_7, activation=activation,init=init)\n",
    "    conv5_7 = Conv2DReluBatchNorm(nfilters*16, 3, 3, conv5_7, activation=activation,init=init)\n",
    "    conv5_7 = Dropout(p=p[4])(conv5_7)\n",
    "        \n",
    "    up6_7 = merge([up_conv(nfilters,8,conv5_7), conv4_7], mode='concat', concat_axis=1)\n",
    "    conv6_7 = Conv2DReluBatchNorm(nfilters*8, 3, 3, up6_7, activation=activation,init=init)\n",
    "    conv6_7 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv6_7, activation=activation,init=init)\n",
    "    conv6_7 = Dropout(p=p[5])(conv6_7)\n",
    "\n",
    "    up7_7 = merge([up_conv(nfilters,4,conv6_7), conv3_7], mode='concat', concat_axis=1)\n",
    "    conv7_7 = Conv2DReluBatchNorm(nfilters*4, 3, 3, up7_7, activation=activation,init=init)\n",
    "    conv7_7 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv7_7, activation=activation,init=init)\n",
    "    conv7_7 = Dropout(p=p[6])(conv7_7)\n",
    "\n",
    "    up8_7 = merge([up_conv(nfilters,2,conv7_7), conv2_7], mode='concat', concat_axis=1)\n",
    "    conv8_7 = Conv2DReluBatchNorm(nfilters*2, 3, 3, up8_7, activation=activation,init=init)\n",
    "    conv8_7 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv8_7, activation=activation,init=init)\n",
    "    conv8_7 = Dropout(p=p[7])(conv8_7)\n",
    "\n",
    "    up9_7 = merge([up_conv(nfilters,1,conv8_7), conv1_7], mode='concat', concat_axis=1)\n",
    "    conv9_7 = Conv2DReluBatchNorm(nfilters, 3, 3, up9_7, activation=activation,init=init)\n",
    "    conv9_7 = Conv2DReluBatchNorm(nfilters, 3, 3, conv9_7, activation=activation,init=init)\n",
    "    conv9_7 = Dropout(p=p[8])(conv9_7)\n",
    "    \n",
    "    conv10_7 = Conv2DReluBatchNorm(1, 1, 1, conv9_7, activation='relu',init=init)\n",
    "    output_7 = Activation(activation='sigmoid',name='class_7')(conv10_7)\n",
    "\n",
    "    # Model_8\n",
    "    conv1_8 = Conv2DReluBatchNorm(nfilters, 3, 3, inputs, activation=activation,init=init)\n",
    "    conv1_8 = Conv2DReluBatchNorm(nfilters, 3, 3, conv1_8, activation=activation,init=init)\n",
    "    pool1_8 = MaxPooling2D(pool_size=(2, 2))(conv1_8)\n",
    "    pool1_8 = Dropout(p=p[0])(pool1_8)\n",
    "\n",
    "    conv2_8 = Conv2DReluBatchNorm(nfilters*2, 3, 3, pool1_8, activation=activation,init=init)\n",
    "    conv2_8 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv2_8, activation=activation,init=init)\n",
    "    pool2_8 = MaxPooling2D(pool_size=(2, 2))(conv2_8)\n",
    "    pool2_8 = Dropout(p=p[1])(pool2_8)\n",
    "\n",
    "    conv3_8 = Conv2DReluBatchNorm(nfilters*4, 3, 3, pool2_8, activation=activation,init=init)\n",
    "    conv3_8 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv3_8, activation=activation,init=init)\n",
    "    pool3_8 = MaxPooling2D(pool_size=(2, 2))(conv3_8)\n",
    "    pool3_8 = Dropout(p=p[2])(pool3_8)\n",
    "\n",
    "    conv4_8 = Conv2DReluBatchNorm(nfilters*8, 3, 3, pool3_8, activation=activation,init=init)\n",
    "    conv4_8 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv4_8, activation=activation,init=init)\n",
    "    pool4_8 = MaxPooling2D(pool_size=(2, 2))(conv4_8)\n",
    "    pool4_8 = Dropout(p=p[3])(pool4_8)\n",
    "\n",
    "    conv5_8 = Conv2DReluBatchNorm(nfilters*16, 3, 3, pool4_8, activation=activation,init=init)\n",
    "    conv5_8 = Conv2DReluBatchNorm(nfilters*16, 3, 3, conv5_8, activation=activation,init=init)\n",
    "    conv5_8 = Dropout(p=p[4])(conv5_8)\n",
    "        \n",
    "    up6_8 = merge([up_conv(nfilters,8,conv5_8), conv4_8], mode='concat', concat_axis=1)\n",
    "    conv6_8 = Conv2DReluBatchNorm(nfilters*8, 3, 3, up6_8, activation=activation,init=init)\n",
    "    conv6_8 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv6_8, activation=activation,init=init)\n",
    "    conv6_8 = Dropout(p=p[5])(conv6_8)\n",
    "\n",
    "    up7_8 = merge([up_conv(nfilters,4,conv6_8), conv3_8], mode='concat', concat_axis=1)\n",
    "    conv7_8 = Conv2DReluBatchNorm(nfilters*4, 3, 3, up7_8, activation=activation,init=init)\n",
    "    conv7_8 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv7_8, activation=activation,init=init)\n",
    "    conv7_8 = Dropout(p=p[6])(conv7_8)\n",
    "\n",
    "    up8_8 = merge([up_conv(nfilters,2,conv7_8), conv2_8], mode='concat', concat_axis=1)\n",
    "    conv8_8 = Conv2DReluBatchNorm(nfilters*2, 3, 3, up8_8, activation=activation,init=init)\n",
    "    conv8_8 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv8_8, activation=activation,init=init)\n",
    "    conv8_8 = Dropout(p=p[7])(conv8_8)\n",
    "\n",
    "    up9_8 = merge([up_conv(nfilters,1,conv8_8), conv1_8], mode='concat', concat_axis=1)\n",
    "    conv9_8 = Conv2DReluBatchNorm(nfilters, 3, 3, up9_8, activation=activation,init=init)\n",
    "    conv9_8 = Conv2DReluBatchNorm(nfilters, 3, 3, conv9_8, activation=activation,init=init)\n",
    "    conv9_8 = Dropout(p=p[8])(conv9_8)\n",
    "    \n",
    "    conv10_8 = Conv2DReluBatchNorm(1, 1, 1, conv9_8, activation='relu',init=init)\n",
    "    output_8 = Activation(activation='sigmoid',name='class_8')(conv10_8)\n",
    "\n",
    "    # Model_9\n",
    "    conv1_9 = Conv2DReluBatchNorm(nfilters, 3, 3, inputs, activation=activation,init=init)\n",
    "    conv1_9 = Conv2DReluBatchNorm(nfilters, 3, 3, conv1_9, activation=activation,init=init)\n",
    "    pool1_9 = MaxPooling2D(pool_size=(2, 2))(conv1_9)\n",
    "    pool1_9 = Dropout(p=p[0])(pool1_9)\n",
    "\n",
    "    conv2_9 = Conv2DReluBatchNorm(nfilters*2, 3, 3, pool1_9, activation=activation,init=init)\n",
    "    conv2_9 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv2_9, activation=activation,init=init)\n",
    "    pool2_9 = MaxPooling2D(pool_size=(2, 2))(conv2_9)\n",
    "    pool2_9 = Dropout(p=p[1])(pool2_9)\n",
    "\n",
    "    conv3_9 = Conv2DReluBatchNorm(nfilters*4, 3, 3, pool2_9, activation=activation,init=init)\n",
    "    conv3_9 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv3_9, activation=activation,init=init)\n",
    "    pool3_9 = MaxPooling2D(pool_size=(2, 2))(conv3_9)\n",
    "    pool3_9 = Dropout(p=p[2])(pool3_9)\n",
    "\n",
    "    conv4_9 = Conv2DReluBatchNorm(nfilters*8, 3, 3, pool3_9, activation=activation,init=init)\n",
    "    conv4_9 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv4_9, activation=activation,init=init)\n",
    "    pool4_9 = MaxPooling2D(pool_size=(2, 2))(conv4_9)\n",
    "    pool4_9 = Dropout(p=p[3])(pool4_9)\n",
    "\n",
    "    conv5_9 = Conv2DReluBatchNorm(nfilters*16, 3, 3, pool4_9, activation=activation,init=init)\n",
    "    conv5_9 = Conv2DReluBatchNorm(nfilters*16, 3, 3, conv5_9, activation=activation,init=init)\n",
    "    conv5_9 = Dropout(p=p[4])(conv5_9)\n",
    "        \n",
    "    up6_9 = merge([up_conv(nfilters,8,conv5_9), conv4_9], mode='concat', concat_axis=1)\n",
    "    conv6_9 = Conv2DReluBatchNorm(nfilters*8, 3, 3, up6_9, activation=activation,init=init)\n",
    "    conv6_9 = Conv2DReluBatchNorm(nfilters*8, 3, 3, conv6_9, activation=activation,init=init)\n",
    "    conv6_9 = Dropout(p=p[5])(conv6_9)\n",
    "\n",
    "    up7_9 = merge([up_conv(nfilters,4,conv6_9), conv3_9], mode='concat', concat_axis=1)\n",
    "    conv7_9 = Conv2DReluBatchNorm(nfilters*4, 3, 3, up7_9, activation=activation,init=init)\n",
    "    conv7_9 = Conv2DReluBatchNorm(nfilters*4, 3, 3, conv7_9, activation=activation,init=init)\n",
    "    conv7_9 = Dropout(p=p[6])(conv7_9)\n",
    "\n",
    "    up8_9 = merge([up_conv(nfilters,2,conv7_9), conv2_9], mode='concat', concat_axis=1)\n",
    "    conv8_9 = Conv2DReluBatchNorm(nfilters*2, 3, 3, up8_9, activation=activation,init=init)\n",
    "    conv8_9 = Conv2DReluBatchNorm(nfilters*2, 3, 3, conv8_9, activation=activation,init=init)\n",
    "    conv8_9 = Dropout(p=p[7])(conv8_9)\n",
    "\n",
    "    up9_9 = merge([up_conv(nfilters,1,conv8_9), conv1_9], mode='concat', concat_axis=1)\n",
    "    conv9_9 = Conv2DReluBatchNorm(nfilters, 3, 3, up9_9, activation=activation,init=init)\n",
    "    conv9_9 = Conv2DReluBatchNorm(nfilters, 3, 3, conv9_9, activation=activation,init=init)\n",
    "    conv9_9 = Dropout(p=p[8])(conv9_9)\n",
    "    \n",
    "    conv10_9 = Conv2DReluBatchNorm(1, 1, 1, conv9_9, activation='relu',init=init)\n",
    "    output_9 = Activation(activation='sigmoid',name='class_9')(conv10_9)\n",
    "\n",
    "    # Specify the model\n",
    "    model = Model(input=inputs, output=[output_0,output_1,output_2,output_3,output_4,output_5,output_6,output_7,output_8,output_9])\n",
    "    \n",
    "    model.compile(optimizer=Adam(lr=lr,decay=decay), loss='binary_crossentropy', loss_weights = list(np.load('./data/misc/weights.npy')), metrics=[jaccard])\n",
    "    \n",
    "    return model\n",
    "\n",
    "p=[0.1,0.2,0.3,0.4,0.5,0.4,0.3,0.2,0.1]\n",
    "\n",
    "model = compiler(img_rows=x.shape[2],img_cols=x.shape[3],\n",
    "            nfilters=8,activation='relu',init='he_normal',\n",
    "            lr=0.001,p=p)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is run # 104\n",
      "Train on 259 samples, validate on 65 samples\n",
      "Epoch 1/1000\n",
      "259/259 [==============================] - 170s - loss: 78.3569 - activation_24_loss: 0.7200 - activation_48_loss: 0.7200 - activation_72_loss: 0.7524 - activation_96_loss: 0.7145 - activation_120_loss: 0.6731 - activation_144_loss: 0.6473 - activation_168_loss: 0.7086 - activation_192_loss: 0.7206 - activation_216_loss: 0.7199 - activation_240_loss: 0.7132 - activation_24_jaccard: 0.0653 - activation_48_jaccard: 0.0210 - activation_72_jaccard: 0.0117 - activation_96_jaccard: 0.0336 - activation_120_jaccard: 0.1685 - activation_144_jaccard: 0.2715 - activation_168_jaccard: 0.0088 - activation_192_jaccard: 0.0018 - activation_216_jaccard: 9.7923e-05 - activation_240_jaccard: 5.6605e-04 - val_loss: 72.9953 - val_activation_24_loss: 0.6622 - val_activation_48_loss: 0.6744 - val_activation_72_loss: 0.6737 - val_activation_96_loss: 0.6716 - val_activation_120_loss: 0.6080 - val_activation_144_loss: 0.6030 - val_activation_168_loss: 0.6723 - val_activation_192_loss: 0.6718 - val_activation_216_loss: 0.6703 - val_activation_240_loss: 0.6702 - val_activation_24_jaccard: 0.0579 - val_activation_48_jaccard: 0.0171 - val_activation_72_jaccard: 0.0093 - val_activation_96_jaccard: 0.0276 - val_activation_120_jaccard: 0.1921 - val_activation_144_jaccard: 0.2482 - val_activation_168_jaccard: 0.0032 - val_activation_192_jaccard: 0.0033 - val_activation_216_jaccard: 9.0701e-05 - val_activation_240_jaccard: 3.7448e-04\n",
      "Epoch 2/1000\n",
      "259/259 [==============================] - 161s - loss: 71.4491 - activation_24_loss: 0.6315 - activation_48_loss: 0.6590 - activation_72_loss: 0.6584 - activation_96_loss: 0.6589 - activation_120_loss: 0.5801 - activation_144_loss: 0.5677 - activation_168_loss: 0.6511 - activation_192_loss: 0.6564 - activation_216_loss: 0.6564 - activation_240_loss: 0.6560 - activation_24_jaccard: 0.0719 - activation_48_jaccard: 0.0204 - activation_72_jaccard: 0.0118 - activation_96_jaccard: 0.0328 - activation_120_jaccard: 0.1897 - activation_144_jaccard: 0.3134 - activation_168_jaccard: 0.0088 - activation_192_jaccard: 0.0018 - activation_216_jaccard: 9.7462e-05 - activation_240_jaccard: 5.5814e-04 - val_loss: 70.0220 - val_activation_24_loss: 0.6158 - val_activation_48_loss: 0.6431 - val_activation_72_loss: 0.6371 - val_activation_96_loss: 0.6447 - val_activation_120_loss: 0.5567 - val_activation_144_loss: 0.5453 - val_activation_168_loss: 0.6407 - val_activation_192_loss: 0.6415 - val_activation_216_loss: 0.6441 - val_activation_240_loss: 0.6408 - val_activation_24_jaccard: 0.0543 - val_activation_48_jaccard: 0.0170 - val_activation_72_jaccard: 0.0093 - val_activation_96_jaccard: 0.0277 - val_activation_120_jaccard: 0.2064 - val_activation_144_jaccard: 0.2798 - val_activation_168_jaccard: 0.0032 - val_activation_192_jaccard: 0.0031 - val_activation_216_jaccard: 9.1216e-05 - val_activation_240_jaccard: 3.7545e-04\n",
      "Epoch 3/1000\n",
      "259/259 [==============================] - 162s - loss: 68.5553 - activation_24_loss: 0.5953 - activation_48_loss: 0.6325 - activation_72_loss: 0.6279 - activation_96_loss: 0.6344 - activation_120_loss: 0.5474 - activation_144_loss: 0.5263 - activation_168_loss: 0.6230 - activation_192_loss: 0.6299 - activation_216_loss: 0.6300 - activation_240_loss: 0.6297 - activation_24_jaccard: 0.0736 - activation_48_jaccard: 0.0203 - activation_72_jaccard: 0.0118 - activation_96_jaccard: 0.0330 - activation_120_jaccard: 0.1994 - activation_144_jaccard: 0.3313 - activation_168_jaccard: 0.0089 - activation_192_jaccard: 0.0018 - activation_216_jaccard: 9.6607e-05 - activation_240_jaccard: 5.5854e-04 - val_loss: 67.1419 - val_activation_24_loss: 0.5880 - val_activation_48_loss: 0.6195 - val_activation_72_loss: 0.6098 - val_activation_96_loss: 0.6219 - val_activation_120_loss: 0.5306 - val_activation_144_loss: 0.5060 - val_activation_168_loss: 0.6157 - val_activation_192_loss: 0.6171 - val_activation_216_loss: 0.6171 - val_activation_240_loss: 0.6163 - val_activation_24_jaccard: 0.0598 - val_activation_48_jaccard: 0.0169 - val_activation_72_jaccard: 0.0099 - val_activation_96_jaccard: 0.0280 - val_activation_120_jaccard: 0.2124 - val_activation_144_jaccard: 0.3021 - val_activation_168_jaccard: 0.0033 - val_activation_192_jaccard: 0.0031 - val_activation_216_jaccard: 9.1651e-05 - val_activation_240_jaccard: 3.7520e-04\n",
      "Epoch 4/1000\n",
      "259/259 [==============================] - 164s - loss: 65.9400 - activation_24_loss: 0.5742 - activation_48_loss: 0.6096 - activation_72_loss: 0.6006 - activation_96_loss: 0.6122 - activation_120_loss: 0.5212 - activation_144_loss: 0.5330 - activation_168_loss: 0.5987 - activation_192_loss: 0.6060 - activation_216_loss: 0.6059 - activation_240_loss: 0.6059 - activation_24_jaccard: 0.0729 - activation_48_jaccard: 0.0203 - activation_72_jaccard: 0.0126 - activation_96_jaccard: 0.0330 - activation_120_jaccard: 0.2094 - activation_144_jaccard: 0.3166 - activation_168_jaccard: 0.0093 - activation_192_jaccard: 0.0017 - activation_216_jaccard: 9.7175e-05 - activation_240_jaccard: 5.5914e-04 - val_loss: 64.6750 - val_activation_24_loss: 0.5665 - val_activation_48_loss: 0.5976 - val_activation_72_loss: 0.5855 - val_activation_96_loss: 0.5995 - val_activation_120_loss: 0.5097 - val_activation_144_loss: 0.5022 - val_activation_168_loss: 0.5927 - val_activation_192_loss: 0.5942 - val_activation_216_loss: 0.5945 - val_activation_240_loss: 0.5937 - val_activation_24_jaccard: 0.0623 - val_activation_48_jaccard: 0.0169 - val_activation_72_jaccard: 0.0102 - val_activation_96_jaccard: 0.0286 - val_activation_120_jaccard: 0.2191 - val_activation_144_jaccard: 0.2952 - val_activation_168_jaccard: 0.0033 - val_activation_192_jaccard: 0.0031 - val_activation_216_jaccard: 9.2037e-05 - val_activation_240_jaccard: 3.7575e-04\n",
      "Epoch 5/1000\n",
      "259/259 [==============================] - 163s - loss: 63.4876 - activation_24_loss: 0.5498 - activation_48_loss: 0.5882 - activation_72_loss: 0.5785 - activation_96_loss: 0.5915 - activation_120_loss: 0.5030 - activation_144_loss: 0.4894 - activation_168_loss: 0.5759 - activation_192_loss: 0.5842 - activation_216_loss: 0.5833 - activation_240_loss: 0.5835 - activation_24_jaccard: 0.0785 - activation_48_jaccard: 0.0202 - activation_72_jaccard: 0.0125 - activation_96_jaccard: 0.0331 - activation_120_jaccard: 0.2153 - activation_144_jaccard: 0.3510 - activation_168_jaccard: 0.0096 - activation_192_jaccard: 0.0018 - activation_216_jaccard: 9.7935e-05 - activation_240_jaccard: 5.5911e-04 - val_loss: 62.2333 - val_activation_24_loss: 0.5469 - val_activation_48_loss: 0.5764 - val_activation_72_loss: 0.5630 - val_activation_96_loss: 0.5796 - val_activation_120_loss: 0.4901 - val_activation_144_loss: 0.5072 - val_activation_168_loss: 0.5703 - val_activation_192_loss: 0.5726 - val_activation_216_loss: 0.5718 - val_activation_240_loss: 0.5716 - val_activation_24_jaccard: 0.0669 - val_activation_48_jaccard: 0.0169 - val_activation_72_jaccard: 0.0107 - val_activation_96_jaccard: 0.0279 - val_activation_120_jaccard: 0.2253 - val_activation_144_jaccard: 0.2937 - val_activation_168_jaccard: 0.0034 - val_activation_192_jaccard: 0.0031 - val_activation_216_jaccard: 9.2768e-05 - val_activation_240_jaccard: 3.7615e-04\n",
      "Epoch 6/1000\n",
      "259/259 [==============================] - 163s - loss: 61.1187 - activation_24_loss: 0.5228 - activation_48_loss: 0.5679 - activation_72_loss: 0.5587 - activation_96_loss: 0.5711 - activation_120_loss: 0.4808 - activation_144_loss: 0.4787 - activation_168_loss: 0.5540 - activation_192_loss: 0.5624 - activation_216_loss: 0.5615 - activation_240_loss: 0.5619 - activation_24_jaccard: 0.0830 - activation_48_jaccard: 0.0202 - activation_72_jaccard: 0.0125 - activation_96_jaccard: 0.0333 - activation_120_jaccard: 0.2229 - activation_144_jaccard: 0.3554 - activation_168_jaccard: 0.0099 - activation_192_jaccard: 0.0017 - activation_216_jaccard: 9.8071e-05 - activation_240_jaccard: 5.5937e-04 - val_loss: 59.9638 - val_activation_24_loss: 0.5247 - val_activation_48_loss: 0.5566 - val_activation_72_loss: 0.5423 - val_activation_96_loss: 0.5586 - val_activation_120_loss: 0.4703 - val_activation_144_loss: 0.4696 - val_activation_168_loss: 0.5491 - val_activation_192_loss: 0.5523 - val_activation_216_loss: 0.5509 - val_activation_240_loss: 0.5509 - val_activation_24_jaccard: 0.0668 - val_activation_48_jaccard: 0.0169 - val_activation_72_jaccard: 0.0111 - val_activation_96_jaccard: 0.0282 - val_activation_120_jaccard: 0.2358 - val_activation_144_jaccard: 0.3110 - val_activation_168_jaccard: 0.0038 - val_activation_192_jaccard: 0.0031 - val_activation_216_jaccard: 9.3224e-05 - val_activation_240_jaccard: 3.7660e-04\n",
      "Epoch 7/1000\n",
      "259/259 [==============================] - 163s - loss: 58.9048 - activation_24_loss: 0.5031 - activation_48_loss: 0.5483 - activation_72_loss: 0.5368 - activation_96_loss: 0.5515 - activation_120_loss: 0.4644 - activation_144_loss: 0.5008 - activation_168_loss: 0.5333 - activation_192_loss: 0.5420 - activation_216_loss: 0.5410 - activation_240_loss: 0.5422 - activation_24_jaccard: 0.0849 - activation_48_jaccard: 0.0202 - activation_72_jaccard: 0.0130 - activation_96_jaccard: 0.0338 - activation_120_jaccard: 0.2271 - activation_144_jaccard: 0.3377 - activation_168_jaccard: 0.0102 - activation_192_jaccard: 0.0017 - activation_216_jaccard: 9.8541e-05 - activation_240_jaccard: 5.5975e-04 - val_loss: 57.8125 - val_activation_24_loss: 0.5055 - val_activation_48_loss: 0.5372 - val_activation_72_loss: 0.5230 - val_activation_96_loss: 0.5388 - val_activation_120_loss: 0.4524 - val_activation_144_loss: 0.4691 - val_activation_168_loss: 0.5296 - val_activation_192_loss: 0.5320 - val_activation_216_loss: 0.5307 - val_activation_240_loss: 0.5345 - val_activation_24_jaccard: 0.0715 - val_activation_48_jaccard: 0.0168 - val_activation_72_jaccard: 0.0108 - val_activation_96_jaccard: 0.0293 - val_activation_120_jaccard: 0.2437 - val_activation_144_jaccard: 0.3125 - val_activation_168_jaccard: 0.0036 - val_activation_192_jaccard: 0.0031 - val_activation_216_jaccard: 9.3537e-05 - val_activation_240_jaccard: 3.7707e-04\n",
      "Epoch 8/1000\n",
      "259/259 [==============================] - 162s - loss: 56.7609 - activation_24_loss: 0.4948 - activation_48_loss: 0.5300 - activation_72_loss: 0.5168 - activation_96_loss: 0.5329 - activation_120_loss: 0.4442 - activation_144_loss: 0.4716 - activation_168_loss: 0.5126 - activation_192_loss: 0.5223 - activation_216_loss: 0.5214 - activation_240_loss: 0.5220 - activation_24_jaccard: 0.0814 - activation_48_jaccard: 0.0201 - activation_72_jaccard: 0.0135 - activation_96_jaccard: 0.0344 - activation_120_jaccard: 0.2385 - activation_144_jaccard: 0.3508 - activation_168_jaccard: 0.0107 - activation_192_jaccard: 0.0017 - activation_216_jaccard: 9.9094e-05 - activation_240_jaccard: 5.6066e-04 - val_loss: 55.6894 - val_activation_24_loss: 0.4889 - val_activation_48_loss: 0.5192 - val_activation_72_loss: 0.5036 - val_activation_96_loss: 0.5206 - val_activation_120_loss: 0.4356 - val_activation_144_loss: 0.4553 - val_activation_168_loss: 0.5096 - val_activation_192_loss: 0.5132 - val_activation_216_loss: 0.5115 - val_activation_240_loss: 0.5119 - val_activation_24_jaccard: 0.0712 - val_activation_48_jaccard: 0.0168 - val_activation_72_jaccard: 0.0115 - val_activation_96_jaccard: 0.0298 - val_activation_120_jaccard: 0.2486 - val_activation_144_jaccard: 0.3103 - val_activation_168_jaccard: 0.0041 - val_activation_192_jaccard: 0.0030 - val_activation_216_jaccard: 9.4467e-05 - val_activation_240_jaccard: 3.7762e-04\n",
      "Epoch 9/1000\n",
      "259/259 [==============================] - 159s - loss: 54.7208 - activation_24_loss: 0.4703 - activation_48_loss: 0.5122 - activation_72_loss: 0.4978 - activation_96_loss: 0.5157 - activation_120_loss: 0.4296 - activation_144_loss: 0.4633 - activation_168_loss: 0.4955 - activation_192_loss: 0.5036 - activation_216_loss: 0.5027 - activation_240_loss: 0.5030 - activation_24_jaccard: 0.0866 - activation_48_jaccard: 0.0201 - activation_72_jaccard: 0.0137 - activation_96_jaccard: 0.0345 - activation_120_jaccard: 0.2435 - activation_144_jaccard: 0.3557 - activation_168_jaccard: 0.0106 - activation_192_jaccard: 0.0017 - activation_216_jaccard: 9.9623e-05 - activation_240_jaccard: 5.6217e-04 - val_loss: 53.7221 - val_activation_24_loss: 0.4691 - val_activation_48_loss: 0.5014 - val_activation_72_loss: 0.4851 - val_activation_96_loss: 0.5042 - val_activation_120_loss: 0.4312 - val_activation_144_loss: 0.4443 - val_activation_168_loss: 0.4915 - val_activation_192_loss: 0.4948 - val_activation_216_loss: 0.4935 - val_activation_240_loss: 0.4932 - val_activation_24_jaccard: 0.0747 - val_activation_48_jaccard: 0.0168 - val_activation_72_jaccard: 0.0117 - val_activation_96_jaccard: 0.0288 - val_activation_120_jaccard: 0.2529 - val_activation_144_jaccard: 0.3179 - val_activation_168_jaccard: 0.0043 - val_activation_192_jaccard: 0.0030 - val_activation_216_jaccard: 9.4741e-05 - val_activation_240_jaccard: 3.7818e-04\n",
      "Epoch 10/1000\n",
      "259/259 [==============================] - 160s - loss: 52.7555 - activation_24_loss: 0.4590 - activation_48_loss: 0.4950 - activation_72_loss: 0.4806 - activation_96_loss: 0.4991 - activation_120_loss: 0.4202 - activation_144_loss: 0.4456 - activation_168_loss: 0.4763 - activation_192_loss: 0.4856 - activation_216_loss: 0.4845 - activation_240_loss: 0.4850 - activation_24_jaccard: 0.0873 - activation_48_jaccard: 0.0200 - activation_72_jaccard: 0.0139 - activation_96_jaccard: 0.0350 - activation_120_jaccard: 0.2452 - activation_144_jaccard: 0.3645 - activation_168_jaccard: 0.0115 - activation_192_jaccard: 0.0017 - activation_216_jaccard: 1.0020e-04 - activation_240_jaccard: 5.6174e-04 - val_loss: 51.7767 - val_activation_24_loss: 0.4565 - val_activation_48_loss: 0.4846 - val_activation_72_loss: 0.4673 - val_activation_96_loss: 0.4872 - val_activation_120_loss: 0.4199 - val_activation_144_loss: 0.4360 - val_activation_168_loss: 0.4755 - val_activation_192_loss: 0.4774 - val_activation_216_loss: 0.4755 - val_activation_240_loss: 0.4756 - val_activation_24_jaccard: 0.0742 - val_activation_48_jaccard: 0.0167 - val_activation_72_jaccard: 0.0121 - val_activation_96_jaccard: 0.0302 - val_activation_120_jaccard: 0.2471 - val_activation_144_jaccard: 0.3255 - val_activation_168_jaccard: 0.0036 - val_activation_192_jaccard: 0.0030 - val_activation_216_jaccard: 9.5197e-05 - val_activation_240_jaccard: 3.7873e-04\n",
      "Epoch 11/1000\n",
      "259/259 [==============================] - 160s - loss: 50.8953 - activation_24_loss: 0.4392 - activation_48_loss: 0.4789 - activation_72_loss: 0.4648 - activation_96_loss: 0.4824 - activation_120_loss: 0.4087 - activation_144_loss: 0.4322 - activation_168_loss: 0.4590 - activation_192_loss: 0.4685 - activation_216_loss: 0.4674 - activation_240_loss: 0.4675 - activation_24_jaccard: 0.0888 - activation_48_jaccard: 0.0200 - activation_72_jaccard: 0.0139 - activation_96_jaccard: 0.0357 - activation_120_jaccard: 0.2474 - activation_144_jaccard: 0.3752 - activation_168_jaccard: 0.0116 - activation_192_jaccard: 0.0017 - activation_216_jaccard: 1.0081e-04 - activation_240_jaccard: 5.6269e-04 - val_loss: 49.9884 - val_activation_24_loss: 0.4367 - val_activation_48_loss: 0.4685 - val_activation_72_loss: 0.4516 - val_activation_96_loss: 0.4712 - val_activation_120_loss: 0.4036 - val_activation_144_loss: 0.4485 - val_activation_168_loss: 0.4567 - val_activation_192_loss: 0.4606 - val_activation_216_loss: 0.4592 - val_activation_240_loss: 0.4586 - val_activation_24_jaccard: 0.0780 - val_activation_48_jaccard: 0.0167 - val_activation_72_jaccard: 0.0124 - val_activation_96_jaccard: 0.0306 - val_activation_120_jaccard: 0.2667 - val_activation_144_jaccard: 0.3222 - val_activation_168_jaccard: 0.0042 - val_activation_192_jaccard: 0.0030 - val_activation_216_jaccard: 9.5746e-05 - val_activation_240_jaccard: 3.7930e-04\n",
      "Epoch 12/1000\n",
      "259/259 [==============================] - 161s - loss: 49.1393 - activation_24_loss: 0.4246 - activation_48_loss: 0.4633 - activation_72_loss: 0.4479 - activation_96_loss: 0.4681 - activation_120_loss: 0.3939 - activation_144_loss: 0.4265 - activation_168_loss: 0.4426 - activation_192_loss: 0.4518 - activation_216_loss: 0.4514 - activation_240_loss: 0.4509 - activation_24_jaccard: 0.0915 - activation_48_jaccard: 0.0200 - activation_72_jaccard: 0.0143 - activation_96_jaccard: 0.0362 - activation_120_jaccard: 0.2580 - activation_144_jaccard: 0.3812 - activation_168_jaccard: 0.0124 - activation_192_jaccard: 0.0017 - activation_216_jaccard: 1.0139e-04 - activation_240_jaccard: 5.6272e-04 - val_loss: 48.2340 - val_activation_24_loss: 0.4200 - val_activation_48_loss: 0.4534 - val_activation_72_loss: 0.4352 - val_activation_96_loss: 0.4573 - val_activation_120_loss: 0.3921 - val_activation_144_loss: 0.4331 - val_activation_168_loss: 0.4410 - val_activation_192_loss: 0.4443 - val_activation_216_loss: 0.4431 - val_activation_240_loss: 0.4423 - val_activation_24_jaccard: 0.0733 - val_activation_48_jaccard: 0.0167 - val_activation_72_jaccard: 0.0125 - val_activation_96_jaccard: 0.0306 - val_activation_120_jaccard: 0.2653 - val_activation_144_jaccard: 0.3269 - val_activation_168_jaccard: 0.0044 - val_activation_192_jaccard: 0.0030 - val_activation_216_jaccard: 9.6370e-05 - val_activation_240_jaccard: 3.7989e-04\n",
      "Epoch 13/1000\n",
      "259/259 [==============================] - 161s - loss: 47.4000 - activation_24_loss: 0.4182 - activation_48_loss: 0.4483 - activation_72_loss: 0.4317 - activation_96_loss: 0.4535 - activation_120_loss: 0.3845 - activation_144_loss: 0.4052 - activation_168_loss: 0.4271 - activation_192_loss: 0.4358 - activation_216_loss: 0.4354 - activation_240_loss: 0.4351 - activation_24_jaccard: 0.0879 - activation_48_jaccard: 0.0198 - activation_72_jaccard: 0.0145 - activation_96_jaccard: 0.0358 - activation_120_jaccard: 0.2630 - activation_144_jaccard: 0.3947 - activation_168_jaccard: 0.0127 - activation_192_jaccard: 0.0017 - activation_216_jaccard: 1.0200e-04 - activation_240_jaccard: 5.6282e-04 - val_loss: 46.5722 - val_activation_24_loss: 0.4217 - val_activation_48_loss: 0.4389 - val_activation_72_loss: 0.4193 - val_activation_96_loss: 0.4421 - val_activation_120_loss: 0.3807 - val_activation_144_loss: 0.4512 - val_activation_168_loss: 0.4259 - val_activation_192_loss: 0.4285 - val_activation_216_loss: 0.4278 - val_activation_240_loss: 0.4268 - val_activation_24_jaccard: 0.0840 - val_activation_48_jaccard: 0.0166 - val_activation_72_jaccard: 0.0132 - val_activation_96_jaccard: 0.0308 - val_activation_120_jaccard: 0.2740 - val_activation_144_jaccard: 0.3178 - val_activation_168_jaccard: 0.0046 - val_activation_192_jaccard: 0.0031 - val_activation_216_jaccard: 9.7087e-05 - val_activation_240_jaccard: 3.8048e-04\n",
      "Epoch 14/1000\n",
      "259/259 [==============================] - 161s - loss: 45.7383 - activation_24_loss: 0.3947 - activation_48_loss: 0.4341 - activation_72_loss: 0.4165 - activation_96_loss: 0.4386 - activation_120_loss: 0.3728 - activation_144_loss: 0.4380 - activation_168_loss: 0.4126 - activation_192_loss: 0.4206 - activation_216_loss: 0.4200 - activation_240_loss: 0.4198 - activation_24_jaccard: 0.0973 - activation_48_jaccard: 0.0199 - activation_72_jaccard: 0.0150 - activation_96_jaccard: 0.0368 - activation_120_jaccard: 0.2673 - activation_144_jaccard: 0.3663 - activation_168_jaccard: 0.0132 - activation_192_jaccard: 0.0017 - activation_216_jaccard: 1.0260e-04 - activation_240_jaccard: 5.6325e-04 - val_loss: 44.9620 - val_activation_24_loss: 0.3951 - val_activation_48_loss: 0.4244 - val_activation_72_loss: 0.4049 - val_activation_96_loss: 0.4274 - val_activation_120_loss: 0.3775 - val_activation_144_loss: 0.4109 - val_activation_168_loss: 0.4118 - val_activation_192_loss: 0.4167 - val_activation_216_loss: 0.4127 - val_activation_240_loss: 0.4119 - val_activation_24_jaccard: 0.0839 - val_activation_48_jaccard: 0.0166 - val_activation_72_jaccard: 0.0133 - val_activation_96_jaccard: 0.0322 - val_activation_120_jaccard: 0.2677 - val_activation_144_jaccard: 0.3401 - val_activation_168_jaccard: 0.0041 - val_activation_192_jaccard: 0.0030 - val_activation_216_jaccard: 9.7625e-05 - val_activation_240_jaccard: 3.8110e-04\n",
      "Epoch 15/1000\n",
      " 40/259 [===>..........................] - ETA: 128s - loss: 44.7988 - activation_24_loss: 0.3750 - activation_48_loss: 0.4330 - activation_72_loss: 0.4121 - activation_96_loss: 0.4324 - activation_120_loss: 0.3741 - activation_144_loss: 0.3978 - activation_168_loss: 0.4040 - activation_192_loss: 0.4114 - activation_216_loss: 0.4113 - activation_240_loss: 0.4115 - activation_24_jaccard: 0.1278 - activation_48_jaccard: 0.0291 - activation_72_jaccard: 0.0127 - activation_96_jaccard: 0.0452 - activation_120_jaccard: 0.2540 - activation_144_jaccard: 0.3431 - activation_168_jaccard: 0.0168 - activation_192_jaccard: 7.8180e-04 - activation_216_jaccard: 2.7390e-04 - activation_240_jaccard: 9.4439e-04"
     ]
    }
   ],
   "source": [
    "def trainer(model,fit=True,use_existing=False):\n",
    "    print('This is run # %i' %run)\n",
    "    \n",
    "    if use_existing:\n",
    "        model.load_weights('./data/weights/model_weights_all-classes_run_{}.hdf5'.format(run))\n",
    "        \n",
    "    if fit:\n",
    "        quitter = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=50, verbose=1, mode='auto')\n",
    "        lrreducer = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=20, verbose=1, mode='auto', epsilon=0.001, cooldown=2, min_lr=0)\n",
    "        model_checkpoint = ModelCheckpoint('./data/weights/model_weights_all-classes_run_{}.hdf5'.format(run), monitor='val_loss', save_best_only=True)\n",
    "        csvlogger = CSVLogger('./data/logs/training_log_run_'+str(run), separator=',', append=True)\n",
    "        # tensorboard = TensorBoard(log_dir='./data/logs/'+'tensorboard_all-classes-run_{:04d}'.format(run), histogram_freq=0, write_graph=True, write_images=False)\n",
    "        # tensorboard --logdir=data/logs\n",
    "        \n",
    "        model.fit(x, [y[:,i:i+1,:,:] for i in range(10)],\n",
    "                  batch_size=5,\n",
    "                  nb_epoch=1000,\n",
    "                  verbose=1,\n",
    "                  shuffle=True,\n",
    "                  callbacks=[model_checkpoint,csvlogger],\n",
    "                  validation_split=0.2,\n",
    "                  initial_epoch=0)\n",
    "            \n",
    "    preds = model.predict(x, verbose=1)\n",
    "    np.save('./data/predictions/predictions_run_{}.npy'.format(run), preds)\n",
    "    \n",
    "    return model\n",
    "\n",
    "model = trainer(model,fit=True,use_existing=False)\n",
    "model.save('./data/models/u-net-complete-model-run_{}_all-classes.hdf5'.format(run))\n",
    "push('Training is done',\n",
    "     'Train loss: %f, train jaccard: %f, val loss %f, val jaccard%f' %(model.history.history['loss'][-1],model.history.history['jaccard'][-1],model.history.history['val_loss'][-1],model.history.history['val_jaccard'][-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}