1. File Preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kita Pisahkan datanya menjadi 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- train (data latih)\n",
    "- validation (validasi untuk data latih)\n",
    "- test (data testing untuk menguji model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:36:44.577679Z",
     "start_time": "2020-04-27T13:36:44.576110Z"
    }
   },
   "outputs": [],
   "source": [
    "# extract melalui notebook jika diperlukan\n",
    "# !unzip flowers-recognition.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:20:25.152791Z",
     "start_time": "2020-04-27T13:20:25.151197Z"
    }
   },
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:20:25.411819Z",
     "start_time": "2020-04-27T13:20:25.410118Z"
    }
   },
   "outputs": [],
   "source": [
    "mypath= 'flowers/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:20:26.610071Z",
     "start_time": "2020-04-27T13:20:26.587117Z"
    }
   },
   "outputs": [],
   "source": [
    "file_name = []\n",
    "tag = []\n",
    "full_path = []\n",
    "for path, subdirs, files in os.walk(mypath):\n",
    "    for name in files:\n",
    "        full_path.append(os.path.join(path, name)) \n",
    "        tag.append(path.split('/')[-1])        \n",
    "        file_name.append(name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:20:35.931129Z",
     "start_time": "2020-04-27T13:20:35.721341Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:24:26.890142Z",
     "start_time": "2020-04-27T13:24:26.883846Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tag\n",
       "daisy        1538\n",
       "dandelion    2110\n",
       "rose         1568\n",
       "sunflower    1468\n",
       "tulip        1968\n",
       "dtype: int64"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# memasukan variabel yang sudah dikumpulkan pada looping di atas menjadi sebuah dataframe agar rapih\n",
    "df = pd.DataFrame({\"path\":full_path,'file_name':file_name,\"tag\":tag})\n",
    "df.groupby(['tag']).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:24:27.173074Z",
     "start_time": "2020-04-27T13:24:27.167878Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>file_name</th>\n",
       "      <th>tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>flowers/daisy/100080576_f52e8ee070_n.jpg</td>\n",
       "      <td>100080576_f52e8ee070_n.jpg</td>\n",
       "      <td>daisy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>flowers/daisy/10140303196_b88d3d6cec.jpg</td>\n",
       "      <td>10140303196_b88d3d6cec.jpg</td>\n",
       "      <td>daisy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>flowers/daisy/10172379554_b296050f82_n.jpg</td>\n",
       "      <td>10172379554_b296050f82_n.jpg</td>\n",
       "      <td>daisy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>flowers/daisy/10172567486_2748826a8b.jpg</td>\n",
       "      <td>10172567486_2748826a8b.jpg</td>\n",
       "      <td>daisy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>flowers/daisy/10172636503_21bededa75_n.jpg</td>\n",
       "      <td>10172636503_21bededa75_n.jpg</td>\n",
       "      <td>daisy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         path                     file_name  \\\n",
       "0    flowers/daisy/100080576_f52e8ee070_n.jpg    100080576_f52e8ee070_n.jpg   \n",
       "1    flowers/daisy/10140303196_b88d3d6cec.jpg    10140303196_b88d3d6cec.jpg   \n",
       "2  flowers/daisy/10172379554_b296050f82_n.jpg  10172379554_b296050f82_n.jpg   \n",
       "3    flowers/daisy/10172567486_2748826a8b.jpg    10172567486_2748826a8b.jpg   \n",
       "4  flowers/daisy/10172636503_21bededa75_n.jpg  10172636503_21bededa75_n.jpg   \n",
       "\n",
       "     tag  \n",
       "0  daisy  \n",
       "1  daisy  \n",
       "2  daisy  \n",
       "3  daisy  \n",
       "4  daisy  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#cek sample datanya\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:24:27.608099Z",
     "start_time": "2020-04-27T13:24:27.606378Z"
    }
   },
   "outputs": [],
   "source": [
    "#load library untuk train test split\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:24:38.449924Z",
     "start_time": "2020-04-27T13:24:38.447907Z"
    }
   },
   "outputs": [],
   "source": [
    "#variabel yang digunakan pada pemisahan data ini\n",
    "X= df['path']\n",
    "y= df['tag']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:24:38.787271Z",
     "start_time": "2020-04-27T13:24:38.781487Z"
    }
   },
   "outputs": [],
   "source": [
    "# split dataset awal menjadi data train dan test\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.20, random_state=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:24:38.946665Z",
     "start_time": "2020-04-27T13:24:38.942709Z"
    }
   },
   "outputs": [],
   "source": [
    "# kemudian data test dibagi menjadi 2 sehingga menjadi data test dan data validation.\n",
    "X_test, X_val, y_test, y_val = train_test_split(\n",
    "    X_test, y_test, test_size=0.5, random_state=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:27:35.790033Z",
     "start_time": "2020-04-27T13:27:35.784677Z"
    }
   },
   "outputs": [],
   "source": [
    "# menyatukan kedalam masing-masing dataframe\n",
    "\n",
    "df_tr = pd.DataFrame({'path':X_train\n",
    "              ,'tag':y_train\n",
    "             ,'set':'train'})\n",
    "\n",
    "df_te = pd.DataFrame({'path':X_test\n",
    "              ,'tag':y_test\n",
    "             ,'set':'test'})\n",
    "\n",
    "df_val = pd.DataFrame({'path':X_val\n",
    "              ,'tag':y_val\n",
    "             ,'set':'validation'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:27:36.006934Z",
     "start_time": "2020-04-27T13:27:36.004779Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train size 6921\n",
      "val size 865\n",
      "test size 866\n"
     ]
    }
   ],
   "source": [
    "print('train size', len(df_tr))\n",
    "print('val size', len(df_te))\n",
    "print('test size', len(df_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:45:04.988035Z",
     "start_time": "2020-04-27T13:45:04.977060Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================================================== \n",
      "\n",
      "set         tag      \n",
      "test        daisy         157\n",
      "            dandelion     212\n",
      "            rose          176\n",
      "            sunflower     135\n",
      "            tulip         185\n",
      "train       daisy        1223\n",
      "            dandelion    1683\n",
      "            rose         1232\n",
      "            sunflower    1189\n",
      "            tulip        1594\n",
      "validation  daisy         158\n",
      "            dandelion     215\n",
      "            rose          160\n",
      "            sunflower     144\n",
      "            tulip         189\n",
      "dtype: int64 \n",
      "\n",
      "===================================================== \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>tag</th>\n",
       "      <th>set</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6386</th>\n",
       "      <td>flowers/dandelion/34578947551_863af3acb3_n.jpg</td>\n",
       "      <td>dandelion</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>648</th>\n",
       "      <td>flowers/flowers/tulip/7166612682_711e680027_n.jpg</td>\n",
       "      <td>tulip</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7109</th>\n",
       "      <td>flowers/flowers/daisy/5794835_d15905c7c8_n.jpg</td>\n",
       "      <td>daisy</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   path        tag    set\n",
       "6386     flowers/dandelion/34578947551_863af3acb3_n.jpg  dandelion  train\n",
       "648   flowers/flowers/tulip/7166612682_711e680027_n.jpg      tulip  train\n",
       "7109     flowers/flowers/daisy/5794835_d15905c7c8_n.jpg      daisy   test"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# melihat proporsi pada masing masing set apakah sudah ok atau masih ada yang ingin diubah\n",
    "df_all = df_tr.append([df_te,df_val]).reset_index(drop=1)\\\n",
    "\n",
    "print('===================================================== \\n')\n",
    "print(df_all.groupby(['set','tag']).size(),'\\n')\n",
    "\n",
    "print('===================================================== \\n')\n",
    "\n",
    "#cek sample datanya\n",
    "df_all.sample(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Merapihkan ke folder set masing-masing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:45:15.341975Z",
     "start_time": "2020-04-27T13:45:15.340307Z"
    }
   },
   "outputs": [],
   "source": [
    "# menghapus folder dataset jika diperlukan\n",
    "#!rm -rf dataset/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:45:15.523556Z",
     "start_time": "2020-04-27T13:45:15.521907Z"
    }
   },
   "outputs": [],
   "source": [
    "import shutil\n",
    "from tqdm.notebook import tqdm as tq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:45:17.315452Z",
     "start_time": "2020-04-27T13:45:17.313905Z"
    }
   },
   "outputs": [],
   "source": [
    "datasource_path = \"flowers/\"\n",
    "dataset_path = \"dataset/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-04-27T13:45:21.624008Z",
     "start_time": "2020-04-27T13:45:20.037843Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b71b97c318ea44d6bb8ce8af63004d44",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for index, row in tq(df_all.iterrows()):\n",
    "    \n",
    "    #detect filepath\n",
    "    file_path = row['path']\n",
    "    if os.path.exists(file_path) == False:\n",
    "            file_path = os.path.join(datasource_path,row['tag'],row['image'].split('.')[0])            \n",
    "    \n",
    "    #make folder destination dirs\n",
    "    if os.path.exists(os.path.join(dataset_path,row['set'],row['tag'])) == False:\n",
    "        os.makedirs(os.path.join(dataset_path,row['set'],row['tag']))\n",
    "    \n",
    "    #define file dest\n",
    "    destination_file_name = file_path.split('/')[-1]\n",
    "    file_dest = os.path.join(dataset_path,row['set'],row['tag'],destination_file_name)\n",
    "    \n",
    "    #copy file from source to dest\n",
    "    if os.path.exists(file_dest) == False:\n",
    "        shutil.copy2(file_path,file_dest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Finish! Dataset siap digunakan "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tf2]",
   "language": "python",
   "name": "conda-env-tf2-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}