Dataset_Modifications.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Description\n",
    "In this notebook we created a utility to fight class imbalance. For this reason, using Gaussian Blur over images that are imbalanced we are imputing the dataset to create ~38k samples overall."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import pandas as pd\n",
    "from shutil import copyfile\n",
    "from shutil import rmtree\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import cv2\n",
    "import numpy as np\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating directory structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "src = \"data/train_set/\"\n",
    "dst = \"data/custom_split/\"\n",
    "\n",
    "os.makedirs(dst,exist_ok=True) # <---- creating custom destination directories\n",
    "os.makedirs(dst+'train',exist_ok=True)\n",
    "os.makedirs(dst+'validation',exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(\"data/train_labels.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique labels: 80\n",
      "[21 29 17 50  8 42 68 10  9 39 11 44 49  5 65 61 56 43 54 40 23 64 58 33\n",
      " 45  3 12 46 59 77 14 13 18 69 47 63 52 66 80 30 55 53 19 67 78 24  2 25\n",
      " 37 57 36  6  1 72 27 60 31 48 74 51 15 26 75 62  4 22 41 79 34 73  7 71\n",
      " 35 38 76 28 70 16 32 20]\n",
      "      img_name  label\n",
      "0  train_1.jpg     21\n",
      "1  train_2.jpg     29\n",
      "2  train_3.jpg     17\n",
      "3  train_4.jpg     21\n",
      "4  train_5.jpg     50\n"
     ]
    }
   ],
   "source": [
    "print(\"Unique labels:\",len(train_df.label.unique()))\n",
    "print(train_df.label.unique())\n",
    "print(train_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SELECTED UNIQUE: [21 29 17 50  8 42 68 10  9 39 11 44 49  5 65 61 56 43 54 40 23 64 58 33\n",
      " 45  3 12 46 59 77 14 13 18 69 47 63 52 66 80 30 55 53 19 67 78 24  2 25\n",
      " 37 57 36  6  1 72 27 60 31 48 74 51 15 26 75 62  4 22 41 79 34 73  7 71\n",
      " 35 38 76 28 70 16 32 20] 80\n"
     ]
    }
   ],
   "source": [
    "output_train = pd.DataFrame()\n",
    "output_validation = pd.DataFrame()\n",
    "output_test = pd.DataFrame()\n",
    "\n",
    "first_n_labels = train_df.label.unique()[:80] # <------------------- edit here first n of unique labels\n",
    "print(\"SELECTED UNIQUE:\",first_n_labels, len(first_n_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### WARNING ######\n",
    "##### CLEAN ENV #####\n",
    "\n",
    "rmtree(dst+'train')\n",
    "rmtree(dst+'validation')\n",
    "\n",
    "os.makedirs(dst+'train',exist_ok=True)\n",
    "os.makedirs(dst+'validation',exist_ok=True)\n",
    "os.remove(dst+\"train_desc.csv\")\n",
    "os.remove(dst+\"validation_desc.csv\")\n",
    "\n",
    "##### WARNING ######\n",
    "##### CLEAN ENV #####"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Manipulation\n",
    "Separating data into PyTorch directory structure and imputing missing data using blur"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "filenames_train = []\n",
    "labels_train = []\n",
    "\n",
    "filenames_val = []\n",
    "labels_val = []\n",
    "\n",
    "\n",
    "for lbl in first_n_labels:\n",
    "    training = 450 # <--------------------------- EDIT HERE NUM OF TRAIN SAMPLES PER LABEL\n",
    "    validation = 40 # <--------------------------- EDIT HERE NUM OF VALIDATION SAMPLES PER LABEL\n",
    "    \n",
    "    os.makedirs(dst+'train/'+str(lbl),exist_ok=True)\n",
    "    os.makedirs(dst+'validation/'+str(lbl),exist_ok=True)\n",
    "    os.makedirs(dst+'test/'+str(lbl),exist_ok=True)\n",
    "    \n",
    "    for index, row in train_df.iterrows():\n",
    "        \n",
    "        img_name = row['img_name']\n",
    "        img_lbl = row['label']\n",
    "        \n",
    "        if lbl == img_lbl and validation > 0 and lbl != '20':\n",
    "            copyfile(src+img_name, dst+\"validation/\"+str(lbl)+\"/\"+img_name)\n",
    "            filenames_val.append(img_name)\n",
    "            labels_val.append(img_lbl)\n",
    "            validation -= 1\n",
    "        elif lbl == img_lbl and training > 0 and validation < 1:\n",
    "            copyfile(src+img_name, dst+\"train/\"+str(lbl)+\"/\"+img_name)\n",
    "            filenames_train.append(img_name)\n",
    "            labels_train.append(img_lbl)\n",
    "            training -= 1\n",
    "        elif validation < 1 and training < 1:\n",
    "            break        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(80):\n",
    "    #special exception case for very underrepresented class 20.\n",
    "    if i+1 != 20:\n",
    "        file_num = len([iq for iq in os.scandir('data/custom_split/train/'+str(i+1))])\n",
    "        if(file_num != 450):\n",
    "            for iq in os.scandir('data/custom_split/train/'+str(i+1)):\n",
    "                if file_num == 450:\n",
    "                    break\n",
    "                else:\n",
    "                    image = cv2.imread('data/custom_split/train/'+str(i+1)+\"/\"+iq.name)\n",
    "                    image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) \n",
    "                    figure_size = 9 \n",
    "                    new_image = cv2.GaussianBlur(image, (figure_size, figure_size),0)\n",
    "                    img_to_write = cv2.cvtColor(new_image, cv2.COLOR_RGB2BGR)\n",
    "                    cv2.imwrite('data/custom_split/train/'+str(i+1)+'/g_'+iq.name,img_to_write)\n",
    "                    file_num+=1            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Single image blur demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAIN: (100, 2)\n",
      "TEST: (0, 0)\n",
      "VALIDATION: (40, 2)\n"
     ]
    }
   ],
   "source": [
    "output_train['img_name'] = filenames_train\n",
    "output_train['label'] = labels_train\n",
    "\n",
    "output_validation['img_name'] = filenames_val\n",
    "output_validation['label'] = labels_val\n",
    "\n",
    "output_train.to_csv(dst+\"train_desc.csv\")\n",
    "output_validation.to_csv(dst+\"validation_desc.csv\")\n",
    "\n",
    "print(\"TRAIN:\", output_train.shape)\n",
    "print(\"TEST:\", output_test.shape)\n",
    "print(\"VALIDATION:\", output_validation.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "rgb_img = plt.imread('data/train_set/train_1.jpg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image = cv2.imread('data/train_set/train_2.jpg')\n",
    "image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) \n",
    "figure_size = 9 \n",
    "\n",
    "new_image = cv2.GaussianBlur(image, (figure_size, figure_size),0)\n",
    "\n",
    "img_to_write = cv2.cvtColor(new_image, cv2.COLOR_RGB2BGR)\n",
    "cv2.imwrite('data/demo/train_2_g.jpg',img_to_write)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}