Rare label detection model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "import keras\n",
    "from sklearn.model_selection import KFold\n",
    "from tensorflow.keras.layers import Dropout, Flatten, Dense, InputLayer\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, Callback, ReduceLROnPlateau\n",
    "from tensorflow.keras import Sequential\n",
    "from tensorflow.keras.applications.vgg16 import VGG16\n",
    "from tensorflow.keras.applications.resnet import ResNet50\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from tensorflow.keras.optimizers import Adam, SGD\n",
    "from tensorflow.keras.backend import clear_session\n",
    "import tensorflow as tf\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "INPUT_SHAPE = (128, 128, 3) # Image Dimensions\n",
    "BATCH_SIZE = 128\n",
    "DROPOUT_RATE = 0.5\n",
    "EPOCHS = 25\n",
    "LR = 0.0001 # Learning Rate\n",
    "REG_STRENGTH = 0.01 # Regularization Strength\n",
    "NFOLDS =2 # No of folds for cross validation\n",
    "WORKERS = 4 # Multithreading no of threads\n",
    "MAXQ = 10 # Max Queue size for multithreading\n",
    "THRES = [0.2] * 17 # Threshold for truth value of label, applied on sigmoid output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_name</th>\n",
       "      <th>tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>train_0</td>\n",
       "      <td>haze primary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>train_1</td>\n",
       "      <td>agriculture clear primary water</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>train_2</td>\n",
       "      <td>clear primary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>train_3</td>\n",
       "      <td>clear primary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>train_4</td>\n",
       "      <td>agriculture clear habitation primary road</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40474</td>\n",
       "      <td>train_40474</td>\n",
       "      <td>clear primary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40475</td>\n",
       "      <td>train_40475</td>\n",
       "      <td>cloudy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40476</td>\n",
       "      <td>train_40476</td>\n",
       "      <td>agriculture clear primary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40477</td>\n",
       "      <td>train_40477</td>\n",
       "      <td>agriculture clear primary road</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>40478</td>\n",
       "      <td>train_40478</td>\n",
       "      <td>agriculture cultivation partly_cloudy primary</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>40479 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        image_name                                           tags\n",
       "0          train_0                                   haze primary\n",
       "1          train_1                agriculture clear primary water\n",
       "2          train_2                                  clear primary\n",
       "3          train_3                                  clear primary\n",
       "4          train_4      agriculture clear habitation primary road\n",
       "...            ...                                            ...\n",
       "40474  train_40474                                  clear primary\n",
       "40475  train_40475                                         cloudy\n",
       "40476  train_40476                      agriculture clear primary\n",
       "40477  train_40477                 agriculture clear primary road\n",
       "40478  train_40478  agriculture cultivation partly_cloudy primary\n",
       "\n",
       "[40479 rows x 2 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv('/home/jupyter/train_v2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_PATH = '/home/jupyter/train-jpg'\n",
    "TRAIN_CSV_PATH = 'rare.csv'\n",
    "\n",
    "df_train = pd.read_csv('train_v2.csv')\n",
    "# Construct dataframes holding training and test data information\n",
    "df_train_rare = pd.read_csv(TRAIN_CSV_PATH, index_col=0)\n",
    "\n",
    "# These files are numpy Arrays with shape Nx1\n",
    "X_train_files_rare = np.array(df_train_rare['image_name'].tolist())\n",
    "X_train_files_rare.reshape((X_train_files_rare.shape[0], 1))\n",
    "y_train_rare = np.array(df_train_rare['tags'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_name</th>\n",
       "      <th>tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>5958</td>\n",
       "      <td>train_5958.jpg</td>\n",
       "      <td>clear primary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11498</td>\n",
       "      <td>train_11498.jpg</td>\n",
       "      <td>blow_down</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12188</td>\n",
       "      <td>train_12188.jpg</td>\n",
       "      <td>artisinal_mine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29246</td>\n",
       "      <td>train_29246.jpg</td>\n",
       "      <td>selective_logging</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17744</td>\n",
       "      <td>train_17744.jpg</td>\n",
       "      <td>clear cultivation primary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6590</td>\n",
       "      <td>train_6590.jpg</td>\n",
       "      <td>clear primary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21302</td>\n",
       "      <td>train_21302.jpg</td>\n",
       "      <td>clear primary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12962</td>\n",
       "      <td>train_12962.jpg</td>\n",
       "      <td>bare_ground</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32612</td>\n",
       "      <td>train_32612.jpg</td>\n",
       "      <td>partly_cloudy primary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4040</td>\n",
       "      <td>train_4040.jpg</td>\n",
       "      <td>bare_ground</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4180 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            image_name                       tags\n",
       "5958    train_5958.jpg              clear primary\n",
       "11498  train_11498.jpg                  blow_down\n",
       "12188  train_12188.jpg             artisinal_mine\n",
       "29246  train_29246.jpg          selective_logging\n",
       "17744  train_17744.jpg  clear cultivation primary\n",
       "...                ...                        ...\n",
       "6590    train_6590.jpg              clear primary\n",
       "21302  train_21302.jpg              clear primary\n",
       "12962  train_12962.jpg                bare_ground\n",
       "32612  train_32612.jpg      partly_cloudy primary\n",
       "4040    train_4040.jpg                bare_ground\n",
       "\n",
       "[4180 rows x 2 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_rare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('conventional_mine', 99),\n",
       " ('blow_down', 100),\n",
       " ('cloudy', 114),\n",
       " ('haze', 138),\n",
       " ('habitation', 194),\n",
       " ('slash_burn', 208),\n",
       " ('cultivation', 212),\n",
       " ('blooming', 331),\n",
       " ('artisinal_mine', 338),\n",
       " ('selective_logging', 339),\n",
       " ('water', 356),\n",
       " ('partly_cloudy', 366),\n",
       " ('road', 389),\n",
       " ('agriculture', 611),\n",
       " ('bare_ground', 861),\n",
       " ('clear', 1378),\n",
       " ('primary', 1849)]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rare = sorted(['conventional_mine', 'blow_down', 'slash_burn', 'blooming', 'artisinal_mine', 'selective_logging', 'bare_ground'])\n",
    "labels_count = {}\n",
    "\n",
    "for tag in df_train_rare['tags'].values:\n",
    "    labels_in_tag = tag.split(' ')\n",
    "    for label in labels_in_tag:\n",
    "        if label in labels_count:\n",
    "            labels_count[label] += 1\n",
    "        else:\n",
    "            labels_count[label] = 0\n",
    "import operator\n",
    "sorted(labels_count.items(), key=operator.itemgetter(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['artisinal_mine',\n",
       " 'bare_ground',\n",
       " 'blooming',\n",
       " 'blow_down',\n",
       " 'conventional_mine',\n",
       " 'selective_logging',\n",
       " 'slash_burn']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(df_train, labels):\n",
    "    X = []\n",
    "    y = []\n",
    "\n",
    "    for img_name, tags in df_train.values:\n",
    "        try:\n",
    "            img = cv2.imread('/home/jupyter/train-jpg/{}'.format(img_name))\n",
    "            X.append(cv2.resize(img, (128,128)))\n",
    "\n",
    "            y_label = np.zeros(len(labels))\n",
    "            for tag in tags.split(' '):\n",
    "                try:\n",
    "                    idx = labels.index(tag)\n",
    "                    y_label[idx] = 1\n",
    "                except Exception as e:\n",
    "                    pass\n",
    "                \n",
    "            y.append(y_label)\n",
    "        \n",
    "        except Exception as e:\n",
    "            print('Issue with loading', img_name)\n",
    "            \n",
    "    return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = load_data(df_train_rare, rare)\n",
    "X = np.array(X).reshape(-1,128,128,3)\n",
    "X = X/255.\n",
    "y = np.array(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rare_x = open('rare_X', 'wb')\n",
    "# pickle.dump(X, rare_X)\n",
    "# rare_X.close()\n",
    "\n",
    "# rare_y = open('rare_Y', 'wb')\n",
    "# pickle.dump(y, rare_y)\n",
    "# rare_y.close()\n",
    "\n",
    "# pickle_result = open('rare_X', 'rb')\n",
    "# X = pickle.load(pickle_result)\n",
    "\n",
    "# pickle_result = open('rare_Y', 'rb')\n",
    "# y = pickle.load(pickle_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "import time\n",
    "X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1, random_state = int(time.time()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3762, 128, 128, 3)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def learning_curve(model_fit, key='acc', ylim=(0.8, 1.01)):\n",
    "    plt.figure(figsize=(12,6))\n",
    "    plt.plot(model_fit.history[key])\n",
    "    plt.plot(model_fit.history['val_' + key])\n",
    "    plt.title('Learning Curve')\n",
    "    plt.ylabel(key.title())\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.ylim(ylim)\n",
    "    plt.legend(['train', 'test'], loc='best')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fbeta_score_K(y_true, y_pred):\n",
    "    beta_squared = 4\n",
    "\n",
    "    tp = K.sum(y_true * y_pred) + K.epsilon()\n",
    "    fp = K.sum(y_pred) - tp\n",
    "    fn = K.sum(y_true) - tp\n",
    "\n",
    "    precision = tp / (tp + fp)\n",
    "    recall = tp / (tp + fn)\n",
    "\n",
    "    result = (beta_squared + 1) * (precision * recall) / (beta_squared * precision + recall + K.epsilon())\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_75\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "vgg16 (Model)                (None, 4, 4, 512)         14714688  \n",
      "_________________________________________________________________\n",
      "flatten_75 (Flatten)         (None, 8192)              0         \n",
      "_________________________________________________________________\n",
      "dense_150 (Dense)            (None, 128)               1048704   \n",
      "_________________________________________________________________\n",
      "dropout_75 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_151 (Dense)            (None, 7)                 903       \n",
      "=================================================================\n",
      "Total params: 15,764,295\n",
      "Trainable params: 1,049,607\n",
      "Non-trainable params: 14,714,688\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "optimizer = Adam(0.0001, decay=0.00001)\n",
    "base_model = VGG16(weights='imagenet', include_top=False, input_shape=(128, 128, 3))\n",
    "\n",
    "# We will create a different CNN for detecting the rare labels.\n",
    "# Note that the dimension of the dense layer at the end in 7x1(7 rare labels) NOT 17x1.\n",
    "\n",
    "for layer in base_model.layers:\n",
    "    layer.trainable = False\n",
    "    model = Sequential([\n",
    "    base_model,\n",
    "    Flatten(), \n",
    "    Dense(128, activation='relu'),\n",
    "    Dropout(0.2),\n",
    "    Dense(7, activation='sigmoid')])\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=[fbeta_score_K])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_4 False\n",
      "block1_conv1 True\n",
      "block1_conv2 True\n",
      "block1_pool True\n",
      "block2_conv1 True\n",
      "block2_conv2 True\n",
      "block2_pool True\n",
      "block3_conv1 True\n",
      "block3_conv2 True\n",
      "block3_conv3 True\n",
      "block3_pool True\n",
      "block4_conv1 True\n",
      "block4_conv2 True\n",
      "block4_conv3 True\n",
      "block4_pool True\n",
      "block5_conv1 True\n",
      "block5_conv2 True\n",
      "block5_conv3 True\n",
      "block5_pool True\n",
      "Model: \"sequential_75\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "vgg16 (Model)                (None, 4, 4, 512)         14714688  \n",
      "_________________________________________________________________\n",
      "flatten_75 (Flatten)         (None, 8192)              0         \n",
      "_________________________________________________________________\n",
      "dense_150 (Dense)            (None, 128)               1048704   \n",
      "_________________________________________________________________\n",
      "dropout_75 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_151 (Dense)            (None, 7)                 903       \n",
      "=================================================================\n",
      "Total params: 15,764,295\n",
      "Trainable params: 15,764,295\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "for layer in model.layers[0].layers[1:]:\n",
    "    layer.trainable = True\n",
    "\n",
    "for layer in model.layers[0].layers:\n",
    "    print(layer.name, layer. trainable)\n",
    "\n",
    "model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=[fbeta_score_K])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "32/32 [==============================] - 18s 557ms/step - loss: 0.1873 - fbeta_score_K: 0.3368 - val_loss: 0.1763 - val_fbeta_score_K: 0.3784\n",
      "Epoch 2/100\n",
      "32/32 [==============================] - 17s 542ms/step - loss: 0.1817 - fbeta_score_K: 0.3692 - val_loss: 0.1533 - val_fbeta_score_K: 0.3785\n",
      "Epoch 3/100\n",
      "32/32 [==============================] - 17s 546ms/step - loss: 0.1757 - fbeta_score_K: 0.3811 - val_loss: 0.1498 - val_fbeta_score_K: 0.4516\n",
      "Epoch 4/100\n",
      "32/32 [==============================] - 17s 546ms/step - loss: 0.1636 - fbeta_score_K: 0.4288 - val_loss: 0.1409 - val_fbeta_score_K: 0.4618\n",
      "Epoch 5/100\n",
      "32/32 [==============================] - 17s 546ms/step - loss: 0.1612 - fbeta_score_K: 0.4271 - val_loss: 0.1382 - val_fbeta_score_K: 0.4801\n",
      "Epoch 6/100\n",
      "32/32 [==============================] - 18s 550ms/step - loss: 0.1580 - fbeta_score_K: 0.4500 - val_loss: 0.1436 - val_fbeta_score_K: 0.4676\n",
      "Epoch 7/100\n",
      "32/32 [==============================] - 18s 550ms/step - loss: 0.1508 - fbeta_score_K: 0.4588 - val_loss: 0.1364 - val_fbeta_score_K: 0.4935\n",
      "Epoch 8/100\n",
      "32/32 [==============================] - 17s 543ms/step - loss: 0.1533 - fbeta_score_K: 0.4531 - val_loss: 0.1212 - val_fbeta_score_K: 0.5589\n",
      "Epoch 9/100\n",
      "32/32 [==============================] - 17s 546ms/step - loss: 0.1512 - fbeta_score_K: 0.4724 - val_loss: 0.1259 - val_fbeta_score_K: 0.4838\n",
      "Epoch 10/100\n",
      "32/32 [==============================] - 17s 537ms/step - loss: 0.1502 - fbeta_score_K: 0.4653 - val_loss: 0.1392 - val_fbeta_score_K: 0.5094\n",
      "Epoch 11/100\n",
      "32/32 [==============================] - 17s 547ms/step - loss: 0.1412 - fbeta_score_K: 0.4953 - val_loss: 0.1275 - val_fbeta_score_K: 0.5150\n",
      "Epoch 12/100\n",
      "32/32 [==============================] - 19s 578ms/step - loss: 0.1430 - fbeta_score_K: 0.4917 - val_loss: 0.1200 - val_fbeta_score_K: 0.5152\n",
      "Epoch 13/100\n",
      "32/32 [==============================] - 18s 576ms/step - loss: 0.1360 - fbeta_score_K: 0.5075 - val_loss: 0.1248 - val_fbeta_score_K: 0.5614\n",
      "Epoch 14/100\n",
      "32/32 [==============================] - 17s 540ms/step - loss: 0.1332 - fbeta_score_K: 0.5273 - val_loss: 0.1176 - val_fbeta_score_K: 0.5327\n",
      "Epoch 15/100\n",
      "32/32 [==============================] - 18s 548ms/step - loss: 0.1359 - fbeta_score_K: 0.5158 - val_loss: 0.1179 - val_fbeta_score_K: 0.5432\n",
      "Epoch 16/100\n",
      "32/32 [==============================] - 17s 544ms/step - loss: 0.1304 - fbeta_score_K: 0.5264 - val_loss: 0.1046 - val_fbeta_score_K: 0.5703\n",
      "Epoch 17/100\n",
      "32/32 [==============================] - 17s 543ms/step - loss: 0.1261 - fbeta_score_K: 0.5445 - val_loss: 0.1143 - val_fbeta_score_K: 0.6001\n",
      "Epoch 18/100\n",
      "32/32 [==============================] - 17s 537ms/step - loss: 0.1301 - fbeta_score_K: 0.5380 - val_loss: 0.0953 - val_fbeta_score_K: 0.6243\n",
      "Epoch 19/100\n",
      "32/32 [==============================] - 18s 549ms/step - loss: 0.1288 - fbeta_score_K: 0.5381 - val_loss: 0.1040 - val_fbeta_score_K: 0.5952\n",
      "Epoch 20/100\n",
      "32/32 [==============================] - 17s 540ms/step - loss: 0.1223 - fbeta_score_K: 0.5614 - val_loss: 0.1006 - val_fbeta_score_K: 0.6173\n",
      "Epoch 21/100\n",
      "32/32 [==============================] - 18s 547ms/step - loss: 0.1226 - fbeta_score_K: 0.5574 - val_loss: 0.0982 - val_fbeta_score_K: 0.6578\n",
      "Epoch 22/100\n",
      "32/32 [==============================] - 17s 540ms/step - loss: 0.1231 - fbeta_score_K: 0.5572 - val_loss: 0.0890 - val_fbeta_score_K: 0.6523\n",
      "Epoch 23/100\n",
      "32/32 [==============================] - 17s 545ms/step - loss: 0.1195 - fbeta_score_K: 0.5641 - val_loss: 0.0826 - val_fbeta_score_K: 0.6320\n",
      "Epoch 24/100\n",
      "32/32 [==============================] - 18s 547ms/step - loss: 0.1175 - fbeta_score_K: 0.5809 - val_loss: 0.0860 - val_fbeta_score_K: 0.6362\n",
      "Epoch 25/100\n",
      "32/32 [==============================] - 17s 546ms/step - loss: 0.1153 - fbeta_score_K: 0.5905 - val_loss: 0.0909 - val_fbeta_score_K: 0.6226\n",
      "Epoch 26/100\n",
      "32/32 [==============================] - 17s 542ms/step - loss: 0.1173 - fbeta_score_K: 0.5680 - val_loss: 0.0874 - val_fbeta_score_K: 0.6555\n",
      "Epoch 27/100\n",
      "32/32 [==============================] - 17s 543ms/step - loss: 0.1070 - fbeta_score_K: 0.6118 - val_loss: 0.0845 - val_fbeta_score_K: 0.6721\n",
      "Epoch 28/100\n",
      "32/32 [==============================] - 17s 544ms/step - loss: 0.1116 - fbeta_score_K: 0.6016 - val_loss: 0.0772 - val_fbeta_score_K: 0.6855\n",
      "Epoch 29/100\n",
      "32/32 [==============================] - 17s 546ms/step - loss: 0.1108 - fbeta_score_K: 0.5990 - val_loss: 0.0733 - val_fbeta_score_K: 0.6842\n",
      "Epoch 30/100\n",
      "32/32 [==============================] - 17s 547ms/step - loss: 0.1072 - fbeta_score_K: 0.6110 - val_loss: 0.0743 - val_fbeta_score_K: 0.6819\n",
      "Epoch 31/100\n",
      "32/32 [==============================] - 17s 540ms/step - loss: 0.1025 - fbeta_score_K: 0.6257 - val_loss: 0.0726 - val_fbeta_score_K: 0.7255\n",
      "Epoch 32/100\n",
      "32/32 [==============================] - 18s 574ms/step - loss: 0.1029 - fbeta_score_K: 0.6361 - val_loss: 0.0722 - val_fbeta_score_K: 0.6950\n",
      "Epoch 33/100\n",
      "32/32 [==============================] - 17s 538ms/step - loss: 0.1016 - fbeta_score_K: 0.6329 - val_loss: 0.0806 - val_fbeta_score_K: 0.7140\n",
      "Epoch 34/100\n",
      "32/32 [==============================] - 17s 543ms/step - loss: 0.0977 - fbeta_score_K: 0.6417 - val_loss: 0.0602 - val_fbeta_score_K: 0.7371\n",
      "Epoch 35/100\n",
      "32/32 [==============================] - 17s 540ms/step - loss: 0.1004 - fbeta_score_K: 0.6424 - val_loss: 0.0818 - val_fbeta_score_K: 0.7000\n",
      "Epoch 36/100\n",
      "32/32 [==============================] - 17s 545ms/step - loss: 0.0958 - fbeta_score_K: 0.6475 - val_loss: 0.0686 - val_fbeta_score_K: 0.6997\n",
      "Epoch 37/100\n",
      "32/32 [==============================] - 17s 546ms/step - loss: 0.1005 - fbeta_score_K: 0.6407 - val_loss: 0.0853 - val_fbeta_score_K: 0.6715\n",
      "Epoch 38/100\n",
      "32/32 [==============================] - 18s 559ms/step - loss: 0.0917 - fbeta_score_K: 0.6612 - val_loss: 0.0651 - val_fbeta_score_K: 0.7491\n",
      "Epoch 39/100\n",
      "32/32 [==============================] - 18s 555ms/step - loss: 0.0893 - fbeta_score_K: 0.6738 - val_loss: 0.0781 - val_fbeta_score_K: 0.7034\n",
      "Epoch 40/100\n",
      "32/32 [==============================] - 17s 543ms/step - loss: 0.0921 - fbeta_score_K: 0.6658 - val_loss: 0.0758 - val_fbeta_score_K: 0.7323\n",
      "Epoch 41/100\n",
      "32/32 [==============================] - 18s 550ms/step - loss: 0.0902 - fbeta_score_K: 0.6690 - val_loss: 0.0619 - val_fbeta_score_K: 0.7754\n",
      "Epoch 42/100\n",
      "32/32 [==============================] - 18s 553ms/step - loss: 0.0974 - fbeta_score_K: 0.6563 - val_loss: 0.0660 - val_fbeta_score_K: 0.7317\n",
      "Epoch 43/100\n",
      "32/32 [==============================] - 17s 543ms/step - loss: 0.0885 - fbeta_score_K: 0.6768 - val_loss: 0.0489 - val_fbeta_score_K: 0.7887\n",
      "Epoch 44/100\n",
      "32/32 [==============================] - 18s 554ms/step - loss: 0.0902 - fbeta_score_K: 0.6682 - val_loss: 0.0486 - val_fbeta_score_K: 0.7887\n",
      "Epoch 45/100\n",
      "32/32 [==============================] - 17s 542ms/step - loss: 0.0806 - fbeta_score_K: 0.7028 - val_loss: 0.0735 - val_fbeta_score_K: 0.7677\n",
      "Epoch 46/100\n",
      "32/32 [==============================] - 18s 555ms/step - loss: 0.0822 - fbeta_score_K: 0.7071 - val_loss: 0.0646 - val_fbeta_score_K: 0.7862\n",
      "Epoch 47/100\n",
      "32/32 [==============================] - 17s 546ms/step - loss: 0.0787 - fbeta_score_K: 0.7123 - val_loss: 0.0463 - val_fbeta_score_K: 0.8162\n",
      "Epoch 48/100\n",
      "32/32 [==============================] - 17s 540ms/step - loss: 0.0748 - fbeta_score_K: 0.7254 - val_loss: 0.0456 - val_fbeta_score_K: 0.8164\n",
      "Epoch 49/100\n",
      "32/32 [==============================] - 17s 544ms/step - loss: 0.0763 - fbeta_score_K: 0.7313 - val_loss: 0.0431 - val_fbeta_score_K: 0.8156\n",
      "Epoch 50/100\n",
      "32/32 [==============================] - 18s 555ms/step - loss: 0.0717 - fbeta_score_K: 0.7354 - val_loss: 0.0412 - val_fbeta_score_K: 0.8396\n",
      "Epoch 51/100\n",
      "32/32 [==============================] - 17s 546ms/step - loss: 0.0641 - fbeta_score_K: 0.7651 - val_loss: 0.0412 - val_fbeta_score_K: 0.8473\n",
      "Epoch 52/100\n",
      "32/32 [==============================] - 18s 552ms/step - loss: 0.0815 - fbeta_score_K: 0.7120 - val_loss: 0.0544 - val_fbeta_score_K: 0.7819\n",
      "Epoch 53/100\n",
      "32/32 [==============================] - 17s 547ms/step - loss: 0.0693 - fbeta_score_K: 0.7485 - val_loss: 0.0485 - val_fbeta_score_K: 0.7936\n",
      "Epoch 54/100\n",
      "32/32 [==============================] - 17s 541ms/step - loss: 0.0678 - fbeta_score_K: 0.7520 - val_loss: 0.0393 - val_fbeta_score_K: 0.8303\n",
      "Epoch 55/100\n",
      "32/32 [==============================] - 18s 550ms/step - loss: 0.0573 - fbeta_score_K: 0.7850 - val_loss: 0.0306 - val_fbeta_score_K: 0.8875\n",
      "Epoch 56/100\n",
      "32/32 [==============================] - 18s 550ms/step - loss: 0.0613 - fbeta_score_K: 0.7800 - val_loss: 0.0438 - val_fbeta_score_K: 0.8252\n",
      "Epoch 57/100\n",
      "32/32 [==============================] - 18s 547ms/step - loss: 0.0721 - fbeta_score_K: 0.7457 - val_loss: 0.0401 - val_fbeta_score_K: 0.8330\n",
      "Epoch 58/100\n",
      "32/32 [==============================] - 17s 546ms/step - loss: 0.0607 - fbeta_score_K: 0.7805 - val_loss: 0.0443 - val_fbeta_score_K: 0.8495\n",
      "Epoch 59/100\n",
      "32/32 [==============================] - 18s 553ms/step - loss: 0.0607 - fbeta_score_K: 0.7760 - val_loss: 0.0318 - val_fbeta_score_K: 0.8800\n",
      "Epoch 60/100\n",
      "32/32 [==============================] - 17s 535ms/step - loss: 0.0621 - fbeta_score_K: 0.7794 - val_loss: 0.0293 - val_fbeta_score_K: 0.8784\n",
      "Epoch 61/100\n",
      "32/32 [==============================] - 17s 521ms/step - loss: 0.0551 - fbeta_score_K: 0.7951 - val_loss: 0.0513 - val_fbeta_score_K: 0.8350\n",
      "Epoch 62/100\n",
      "32/32 [==============================] - 17s 517ms/step - loss: 0.0650 - fbeta_score_K: 0.7720 - val_loss: 0.0493 - val_fbeta_score_K: 0.8030\n",
      "Epoch 63/100\n",
      "32/32 [==============================] - 17s 532ms/step - loss: 0.0632 - fbeta_score_K: 0.7673 - val_loss: 0.0378 - val_fbeta_score_K: 0.8600\n",
      "Epoch 64/100\n",
      "32/32 [==============================] - 18s 568ms/step - loss: 0.0557 - fbeta_score_K: 0.7984 - val_loss: 0.0290 - val_fbeta_score_K: 0.8847\n",
      "Epoch 65/100\n",
      "32/32 [==============================] - 17s 543ms/step - loss: 0.0526 - fbeta_score_K: 0.8032 - val_loss: 0.0365 - val_fbeta_score_K: 0.8697\n",
      "Epoch 66/100\n",
      "32/32 [==============================] - 18s 570ms/step - loss: 0.0524 - fbeta_score_K: 0.8123 - val_loss: 0.0253 - val_fbeta_score_K: 0.8865\n",
      "Epoch 67/100\n",
      "32/32 [==============================] - 18s 567ms/step - loss: 0.0479 - fbeta_score_K: 0.8279 - val_loss: 0.0373 - val_fbeta_score_K: 0.8435\n",
      "Epoch 68/100\n",
      "32/32 [==============================] - 18s 563ms/step - loss: 0.0529 - fbeta_score_K: 0.8081 - val_loss: 0.0321 - val_fbeta_score_K: 0.8800\n",
      "Epoch 69/100\n",
      "32/32 [==============================] - 18s 564ms/step - loss: 0.0474 - fbeta_score_K: 0.8238 - val_loss: 0.0291 - val_fbeta_score_K: 0.8826\n",
      "Epoch 70/100\n",
      "32/32 [==============================] - 18s 558ms/step - loss: 0.0518 - fbeta_score_K: 0.8146 - val_loss: 0.0285 - val_fbeta_score_K: 0.8961\n",
      "Epoch 71/100\n",
      "32/32 [==============================] - 18s 572ms/step - loss: 0.0566 - fbeta_score_K: 0.8002 - val_loss: 0.0255 - val_fbeta_score_K: 0.8805\n",
      "Epoch 72/100\n",
      "32/32 [==============================] - 18s 550ms/step - loss: 0.0414 - fbeta_score_K: 0.8453 - val_loss: 0.0227 - val_fbeta_score_K: 0.9074\n",
      "Epoch 73/100\n",
      "32/32 [==============================] - 18s 552ms/step - loss: 0.0415 - fbeta_score_K: 0.8465 - val_loss: 0.0241 - val_fbeta_score_K: 0.8957\n",
      "Epoch 74/100\n",
      "32/32 [==============================] - 18s 558ms/step - loss: 0.0453 - fbeta_score_K: 0.8347 - val_loss: 0.0274 - val_fbeta_score_K: 0.8968\n",
      "Epoch 75/100\n",
      "32/32 [==============================] - 18s 547ms/step - loss: 0.0450 - fbeta_score_K: 0.8380 - val_loss: 0.0296 - val_fbeta_score_K: 0.8866\n",
      "Epoch 76/100\n",
      "32/32 [==============================] - 17s 545ms/step - loss: 0.0419 - fbeta_score_K: 0.8508 - val_loss: 0.0366 - val_fbeta_score_K: 0.8673\n",
      "Epoch 77/100\n",
      "32/32 [==============================] - 17s 542ms/step - loss: 0.0471 - fbeta_score_K: 0.8300 - val_loss: 0.0276 - val_fbeta_score_K: 0.9024\n",
      "Epoch 78/100\n",
      "32/32 [==============================] - 17s 533ms/step - loss: 0.0433 - fbeta_score_K: 0.8437 - val_loss: 0.0351 - val_fbeta_score_K: 0.8649\n",
      "Epoch 79/100\n",
      "32/32 [==============================] - 17s 536ms/step - loss: 0.0404 - fbeta_score_K: 0.8517 - val_loss: 0.0232 - val_fbeta_score_K: 0.9129\n",
      "Epoch 80/100\n",
      "32/32 [==============================] - 17s 536ms/step - loss: 0.0384 - fbeta_score_K: 0.8629 - val_loss: 0.0424 - val_fbeta_score_K: 0.8601\n",
      "Epoch 81/100\n",
      "32/32 [==============================] - 18s 548ms/step - loss: 0.0439 - fbeta_score_K: 0.8429 - val_loss: 0.0260 - val_fbeta_score_K: 0.9265\n",
      "Epoch 82/100\n",
      "32/32 [==============================] - 17s 543ms/step - loss: 0.0448 - fbeta_score_K: 0.8427 - val_loss: 0.0277 - val_fbeta_score_K: 0.8995\n",
      "Epoch 83/100\n",
      "32/32 [==============================] - 17s 535ms/step - loss: 0.0458 - fbeta_score_K: 0.8380 - val_loss: 0.0506 - val_fbeta_score_K: 0.8394\n",
      "Epoch 84/100\n",
      "32/32 [==============================] - 17s 537ms/step - loss: 0.0416 - fbeta_score_K: 0.8474 - val_loss: 0.0215 - val_fbeta_score_K: 0.9280\n",
      "Epoch 85/100\n",
      "32/32 [==============================] - 17s 539ms/step - loss: 0.0446 - fbeta_score_K: 0.8390 - val_loss: 0.0205 - val_fbeta_score_K: 0.9203\n",
      "Epoch 86/100\n",
      "32/32 [==============================] - 17s 532ms/step - loss: 0.0369 - fbeta_score_K: 0.8675 - val_loss: 0.0314 - val_fbeta_score_K: 0.8803\n",
      "Epoch 87/100\n",
      "32/32 [==============================] - 17s 523ms/step - loss: 0.0354 - fbeta_score_K: 0.8685 - val_loss: 0.0242 - val_fbeta_score_K: 0.9172\n",
      "Epoch 88/100\n",
      "32/32 [==============================] - 17s 524ms/step - loss: 0.0358 - fbeta_score_K: 0.8714 - val_loss: 0.0242 - val_fbeta_score_K: 0.9067\n",
      "Epoch 89/100\n",
      "32/32 [==============================] - 17s 525ms/step - loss: 0.0355 - fbeta_score_K: 0.8720 - val_loss: 0.0210 - val_fbeta_score_K: 0.9342\n",
      "Epoch 90/100\n",
      "32/32 [==============================] - 17s 520ms/step - loss: 0.0378 - fbeta_score_K: 0.8673 - val_loss: 0.0170 - val_fbeta_score_K: 0.9241\n",
      "Epoch 91/100\n",
      "32/32 [==============================] - 17s 527ms/step - loss: 0.0304 - fbeta_score_K: 0.8871 - val_loss: 0.0249 - val_fbeta_score_K: 0.9160\n",
      "Epoch 92/100\n",
      "32/32 [==============================] - 17s 520ms/step - loss: 0.0319 - fbeta_score_K: 0.8799 - val_loss: 0.0222 - val_fbeta_score_K: 0.9136\n",
      "Epoch 93/100\n",
      "32/32 [==============================] - 17s 532ms/step - loss: 0.0338 - fbeta_score_K: 0.8797 - val_loss: 0.0226 - val_fbeta_score_K: 0.9202\n",
      "Epoch 94/100\n",
      "32/32 [==============================] - 17s 531ms/step - loss: 0.0343 - fbeta_score_K: 0.8784 - val_loss: 0.0166 - val_fbeta_score_K: 0.9336\n",
      "Epoch 95/100\n",
      "32/32 [==============================] - 17s 533ms/step - loss: 0.0316 - fbeta_score_K: 0.8849 - val_loss: 0.0215 - val_fbeta_score_K: 0.9200\n",
      "Epoch 96/100\n",
      "32/32 [==============================] - 17s 540ms/step - loss: 0.0289 - fbeta_score_K: 0.8962 - val_loss: 0.0183 - val_fbeta_score_K: 0.9340\n",
      "Epoch 97/100\n",
      "32/32 [==============================] - 17s 541ms/step - loss: 0.0294 - fbeta_score_K: 0.8893 - val_loss: 0.0281 - val_fbeta_score_K: 0.9192\n",
      "Epoch 98/100\n",
      "32/32 [==============================] - 17s 530ms/step - loss: 0.0365 - fbeta_score_K: 0.8728 - val_loss: 0.0191 - val_fbeta_score_K: 0.9403\n",
      "Epoch 99/100\n",
      "32/32 [==============================] - 17s 532ms/step - loss: 0.0321 - fbeta_score_K: 0.8860 - val_loss: 0.0135 - val_fbeta_score_K: 0.9367\n",
      "Epoch 100/100\n",
      "32/32 [==============================] - 17s 531ms/step - loss: 0.0297 - fbeta_score_K: 0.8909 - val_loss: 0.0153 - val_fbeta_score_K: 0.9427\n"
     ]
    }
   ],
   "source": [
    "aug = keras.preprocessing.image.ImageDataGenerator(rotation_range=20, zoom_range=0.15,\n",
    "                         width_shift_range=0.2, height_shift_range=0.2, shear_range=0.15,\n",
    "                         horizontal_flip=True, fill_mode=\"nearest\")\n",
    " \n",
    "model_fit = model.fit_generator(aug.flow(X, y, batch_size=128),\n",
    "                        validation_data=(X_val, y_val), steps_per_epoch=len(X) // 128,\n",
    "                        epochs=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F beta score:  0.4963924963924964\n",
      "Error: 50.36%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/site-packages/sklearn/metrics/classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n",
      "/usr/local/lib/python3.5/site-packages/sklearn/metrics/classification.py:1137: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in samples with no true labels.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import fbeta_score\n",
    "y_pred = model.predict(X_val, batch_size=128)\n",
    "score = fbeta_score(y_val, np.array(y_pred) > 0.2, beta=2, average='samples')\n",
    "print(\"F beta score: \", score)\n",
    "print(\"Error: %.2f%%\" % (100-score*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.save('rare_model.h5')\n",
    "# model.save_weights('rare_model_weights.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_t1 = np.where(y_t > 0.2, 1, 0)\n",
    "# pd.DataFrame(y_t1, columns=rare).astype(int).to_csv('rare_pred.csv')\n",
    "# pd.DataFrame(y_val, columns=rare).astype(int).to_csv('rare_actual.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_rare, y_rare = load_data(df_train, rare)\n",
    "X_rare_train = np.array(X_rare).reshape(-1,128,128,3)\n",
    "X_rare_train = X_rare_train/255.\n",
    "y_rare_train = np.array(y_rare)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_rare_train = model.predict(X_rare_train, batch_size=128)\n",
    "# file = open('y_pred_rare_train', 'wb')\n",
    "# pickle.dump(y_pred_train, file)\n",
    "# file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_rare_train_full = np.where(y_pred_rare_train > 0.2, 1, 0)\n",
    "pd.DataFrame(y_rare_train_full, columns=rare).astype(int).to_csv('rare_pred_train.csv')\n",
    "pd.DataFrame(y_rare_train, columns=rare).astype(int).to_csv('rare_actual_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_rare_pred_train = pd.read_csv('rare_pred_train.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_name</th>\n",
       "      <th>tags</th>\n",
       "      <th>pred_tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>train_0.jpg</td>\n",
       "      <td>haze primary</td>\n",
       "      <td>clear haze primary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>train_1.jpg</td>\n",
       "      <td>agriculture clear primary water</td>\n",
       "      <td>agriculture clear partly_cloudy primary road w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>train_2.jpg</td>\n",
       "      <td>clear primary</td>\n",
       "      <td>clear primary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>train_3.jpg</td>\n",
       "      <td>clear primary</td>\n",
       "      <td>clear primary</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>train_4.jpg</td>\n",
       "      <td>agriculture clear habitation primary road</td>\n",
       "      <td>agriculture habitation partly_cloudy primary road</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    image_name                                       tags  \\\n",
       "0  train_0.jpg                               haze primary   \n",
       "1  train_1.jpg            agriculture clear primary water   \n",
       "2  train_2.jpg                              clear primary   \n",
       "3  train_3.jpg                              clear primary   \n",
       "4  train_4.jpg  agriculture clear habitation primary road   \n",
       "\n",
       "                                           pred_tags  \n",
       "0                                 clear haze primary  \n",
       "1  agriculture clear partly_cloudy primary road w...  \n",
       "2                                      clear primary  \n",
       "3                                      clear primary  \n",
       "4  agriculture habitation partly_cloudy primary road  "
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "train_predictions = pd.read_csv('2 - train_pred.csv')\n",
    "train_predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = ['agriculture', 'artisinal_mine', 'bare_ground', 'blooming', 'blow_down', 'clear', 'cloudy', 'conventional_mine', 'cultivation', 'habitation', 'haze', 'partly_cloudy', 'primary', 'road', 'selective_logging', 'slash_burn', 'water']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "def return_Binary(tags):\n",
    "    tags = tags.split(' ')\n",
    "    return [1 if x in tags else 0 for x in labels]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_name</th>\n",
       "      <th>tags</th>\n",
       "      <th>pred_tags</th>\n",
       "      <th>pred_binary</th>\n",
       "      <th>actual_binary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>train_0.jpg</td>\n",
       "      <td>haze primary</td>\n",
       "      <td>clear haze primary</td>\n",
       "      <td>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, ...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>train_1.jpg</td>\n",
       "      <td>agriculture clear primary water</td>\n",
       "      <td>agriculture clear partly_cloudy primary road w...</td>\n",
       "      <td>[1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, ...</td>\n",
       "      <td>[1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>train_2.jpg</td>\n",
       "      <td>clear primary</td>\n",
       "      <td>clear primary</td>\n",
       "      <td>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>train_3.jpg</td>\n",
       "      <td>clear primary</td>\n",
       "      <td>clear primary</td>\n",
       "      <td>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...</td>\n",
       "      <td>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>train_4.jpg</td>\n",
       "      <td>agriculture clear habitation primary road</td>\n",
       "      <td>agriculture habitation partly_cloudy primary road</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, ...</td>\n",
       "      <td>[1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    image_name                                       tags  \\\n",
       "0  train_0.jpg                               haze primary   \n",
       "1  train_1.jpg            agriculture clear primary water   \n",
       "2  train_2.jpg                              clear primary   \n",
       "3  train_3.jpg                              clear primary   \n",
       "4  train_4.jpg  agriculture clear habitation primary road   \n",
       "\n",
       "                                           pred_tags  \\\n",
       "0                                 clear haze primary   \n",
       "1  agriculture clear partly_cloudy primary road w...   \n",
       "2                                      clear primary   \n",
       "3                                      clear primary   \n",
       "4  agriculture habitation partly_cloudy primary road   \n",
       "\n",
       "                                         pred_binary  \\\n",
       "0  [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, ...   \n",
       "1  [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, ...   \n",
       "2  [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...   \n",
       "3  [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...   \n",
       "4  [1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, ...   \n",
       "\n",
       "                                       actual_binary  \n",
       "0  [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, ...  \n",
       "1  [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...  \n",
       "2  [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...  \n",
       "3  [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, ...  \n",
       "4  [1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, ...  "
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_predictions['pred_binary'] = train_predictions['pred_tags'].map(return_Binary)\n",
    "train_predictions['actual_binary'] = train_predictions['tags'].map(return_Binary)\n",
    "train_predictions.head()\n",
    "pred_toArray = np.array(list(train_predictions['pred_binary']))\n",
    "actual_toArray = np.array(list(train_predictions['actual_binary']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                   precision    recall  f1-score   support\n",
      "\n",
      "      agriculture       0.89      0.63      0.74     17287\n",
      "   artisinal_mine       0.00      0.14      0.01         7\n",
      "      bare_ground       0.04      0.16      0.07       232\n",
      "         blooming       0.20      0.10      0.14       649\n",
      "        blow_down       0.00      0.00      0.00        29\n",
      "            clear       0.71      0.95      0.81     21278\n",
      "           cloudy       0.87      0.75      0.80      2436\n",
      "conventional_mine       0.00      0.00      0.00         0\n",
      "      cultivation       0.06      0.62      0.11       445\n",
      "       habitation       0.74      0.48      0.58      5580\n",
      "             haze       0.65      0.63      0.64      2782\n",
      "    partly_cloudy       0.99      0.34      0.51     21332\n",
      "          primary       1.00      0.96      0.98     38899\n",
      "             road       0.85      0.66      0.75     10324\n",
      "selective_logging       0.00      0.00      0.00         7\n",
      "       slash_burn       0.00      0.00      0.00         0\n",
      "            water       0.90      0.43      0.59     15310\n",
      "\n",
      "      avg / total       0.89      0.70      0.75    136597\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/site-packages/sklearn/metrics/classification.py:1137: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "# Previos Matrix without Rare label prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classification Report after the rare label training\n",
    "### Imporved F1 scores for the rare labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                   precision    recall  f1-score   support\n",
      "\n",
      "      agriculture       0.89      0.63      0.74     17287\n",
      "   artisinal_mine       1.00      0.65      0.78       525\n",
      "      bare_ground       1.00      0.15      0.25      5915\n",
      "         blooming       0.94      0.15      0.26      2046\n",
      "        blow_down       0.99      0.10      0.18       951\n",
      "            clear       0.71      0.95      0.81     21278\n",
      "           cloudy       0.87      0.75      0.80      2436\n",
      "conventional_mine       1.00      0.57      0.73       175\n",
      "      cultivation       0.06      0.62      0.11       445\n",
      "       habitation       0.74      0.48      0.58      5580\n",
      "             haze       0.65      0.63      0.64      2782\n",
      "    partly_cloudy       0.99      0.34      0.51     21332\n",
      "          primary       1.00      0.96      0.98     38899\n",
      "             road       0.85      0.66      0.75     10324\n",
      "selective_logging       0.94      0.23      0.37      1401\n",
      "       slash_burn       1.00      0.12      0.21      1765\n",
      "            water       0.90      0.43      0.59     15310\n",
      "\n",
      "      avg / total       0.90      0.66      0.72    148451\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_predBinary = pd.DataFrame(pred_toArray, columns=labels)\n",
    "df_actualBinary = pd.DataFrame(actual_toArray, columns=labels)\n",
    "for tag in rare:\n",
    "    df_predBinary[tag] = df_rare_pred_train[tag]\n",
    "print(classification_report(df_predBinary, df_actualBinary, target_names=labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEST_PATH = 'test-jpg/test-jpg'\n",
    "TEST_CSV_PATH = 'sample_submission_v2.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data_test(df_train):\n",
    "    X = []\n",
    "\n",
    "    for img_name, tags in df_train.values:\n",
    "        try:\n",
    "            img = cv2.imread('test-jpg/test-jpg/{}'.format(img_name + '.jpg'))\n",
    "            X.append(cv2.resize(img, (128,128)))\n",
    "        \n",
    "        except Exception as e:\n",
    "            print('Issue with loading', img_name)\n",
    "            \n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test = pd.read_csv(TEST_CSV_PATH)\n",
    "X_test = load_data_test(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = np.array(X_test).reshape(-1,128,128,3)\n",
    "X_test = X_test/255."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing the results from existing model\n",
    "result = pd.read_csv('results_rerun2.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rare label prediction with test data\n",
    "y_pred_rare_test = model.predict(X_test, batch_size=128)\n",
    "# file = open('y_pred_rare_test', 'wb')\n",
    "# pickle.dump(y_pred_rare_test, file)\n",
    "# file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Applying threshold of 0.2\n",
    "y_pred_rare_test_thresh2 = np.where(y_pred_rare_test > 0.2, 1, 0)\n",
    "df_pred_rare_test_thresh2 = pd.DataFrame(y_pred_rare_test_thresh2, columns=rare).astype(int)\n",
    "\n",
    "# df_pred_rare_test_thresh2.to_csv('rare_pred_test_thresh2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We will get the probabilities from the rare label detection and use it to scale up the rare label probabilities of the existing model. \n",
    "for tag in rare:\n",
    "    result[tag] = df_pred_rare_test[tag]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:3: FutureWarning: \n",
      ".ix is deprecated. Please use\n",
      ".loc for label based indexing or\n",
      ".iloc for positional indexing\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#ix-indexer-is-deprecated\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "preds = []\n",
    "for i in range(result.shape[0]):\n",
    "    a = result.ix[[i]]\n",
    "    a = a.apply(lambda x: x > THRES, axis=1)\n",
    "    a = a.transpose()\n",
    "    a = a.loc[a[i] == True]\n",
    "    ' '.join(list(a.index))\n",
    "    preds.append(' '.join(list(a.index)))\n",
    "    \n",
    "df_test['tags'] = preds\n",
    "df_test['image_name'] = df_test['image_name'].astype(str).str.slice(stop=-4)\n",
    "df_test.to_csv('rare_test-submit.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kaggle score - 0.91599"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# result.to_csv('results_rare_test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Threshold 0.5 for rare"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rare label prediction with test data\n",
    "y_pred_rare_test = model.predict(X_test, batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred_rare_test_thresh5 = np.where(y_pred_rare_test > 0.5, 1, 0)\n",
    "df_pred_rare_test_thresh5 = pd.DataFrame(y_pred_rare_test_thresh5, columns=rare).astype(int)\n",
    "df_pred_rare_test_thresh5.to_csv('rare_pred_test_thresh5.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "for tag in rare:\n",
    "    result[tag] = df_pred_rare_test_thresh5[tag]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:3: FutureWarning: \n",
      ".ix is deprecated. Please use\n",
      ".loc for label based indexing or\n",
      ".iloc for positional indexing\n",
      "\n",
      "See the documentation here:\n",
      "http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#ix-indexer-is-deprecated\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "preds = []\n",
    "for i in range(result.shape[0]):\n",
    "    a = result.ix[[i]]\n",
    "    a = a.apply(lambda x: x > THRES, axis=1)\n",
    "    a = a.transpose()\n",
    "    a = a.loc[a[i] == True]\n",
    "    ' '.join(list(a.index))\n",
    "    preds.append(' '.join(list(a.index)))\n",
    "    \n",
    "df_test['tags'] = preds\n",
    "# df_test['image_name'] = df_test['image_name'].astype(str).str.slice(stop=-4)\n",
    "df_test.to_csv('rare_test-submit_thresh5.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kaggle Score: 0.92006\n",
    "### Here the score increased, but it penalized the rare label accuracy. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}