2. wep_classifier_tuning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Hyperparameter Tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functions.tuning import Tuner\n",
    "\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run once if permission error when training\n",
    "# torch.hub.list('pytorch/vision:v0.8.0', force_reload=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set hyperparameters\n",
    "EPOCHS = 1000\n",
    "LEARNING_RATE = 0.001\n",
    "BATCH_SIZES = [32, 64, 128] # samples per batch to load\n",
    "SPLIT_SIZE = 0.15 # validation & test dataset size\n",
    "H_LAYERS = [[256, 256], [512, 256], [512, 512], [1024, 516]]\n",
    "N_CLASSES = 35 # number of classes\n",
    "N_PREDS = 5 # number of model predictions (e.g. top-5 error rate)\n",
    "MAIN_FILEPATH = 'dataset/resized'\n",
    "SEED = 1\n",
    "MODEL_NAMES = [\"GoogLeNet\", \"MobileNet-V2\", \"ResNet-34\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CUDA is available. Device has been set to GPU.\n"
     ]
    }
   ],
   "source": [
    "# Set tune class object\n",
    "tune = Tuner()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transform the dataset\n",
    "dataset = tune.set_data(MAIN_FILEPATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GoogleNet:\n",
      "5,937,091 total parameters\n",
      "337,187 training parameters\n",
      "\n",
      "MobileNet-V2:\n",
      "2,626,595 total parameters\n",
      "402,723 training parameters\n",
      "\n",
      "ResNet-34:\n",
      "21,490,787 total parameters\n",
      "206,115 training parameters\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Calculate trainable parameters\n",
    "tune.calc_params(MODEL_NAMES, N_CLASSES, H_LAYERS[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training: GoogleNet_32_256_256\n",
      "Epoch: 1/1000 Step: 180/361 Training Loss: 2.879 Validation Loss: 1.841 Accuracy: 0.486\n",
      "Validation loss decreased (inf -> 1.841). Saving model...\n",
      "Epoch: 1/1000 Step: 360/361 Training Loss: 1.867 Validation Loss: 1.294 Accuracy: 0.632\n",
      "Validation loss decreased (1.841 -> 1.294). Saving model...\n",
      "Epoch: 2/1000 Step: 180/361 Training Loss: 1.571 Validation Loss: 1.184 Accuracy: 0.645\n",
      "Validation loss decreased (1.294 -> 1.184). Saving model...\n",
      "Epoch: 2/1000 Step: 360/361 Training Loss: 1.422 Validation Loss: 1.056 Accuracy: 0.690\n",
      "Validation loss decreased (1.184 -> 1.056). Saving model...\n",
      "Epoch: 3/1000 Step: 180/361 Training Loss: 1.382 Validation Loss: 1.032 Accuracy: 0.694\n",
      "Validation loss decreased (1.056 -> 1.032). Saving model...\n",
      "Epoch: 3/1000 Step: 360/361 Training Loss: 1.287 Validation Loss: 0.999 Accuracy: 0.702\n",
      "Validation loss decreased (1.032 -> 0.999). Saving model...\n",
      "Epoch: 4/1000 Step: 180/361 Training Loss: 1.259 Validation Loss: 0.926 Accuracy: 0.735\n",
      "Validation loss decreased (0.999 -> 0.926). Saving model...\n",
      "Epoch: 4/1000 Step: 360/361 Training Loss: 1.250 Validation Loss: 0.875 Accuracy: 0.756\n",
      "Validation loss decreased (0.926 -> 0.875). Saving model...\n",
      "Epoch: 5/1000 Step: 180/361 Training Loss: 1.189 Validation Loss: 0.931 Accuracy: 0.722\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 5/1000 Step: 360/361 Training Loss: 1.184 Validation Loss: 0.894 Accuracy: 0.743\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 6/1000 Step: 180/361 Training Loss: 1.119 Validation Loss: 0.843 Accuracy: 0.751\n",
      "Validation loss decreased (0.875 -> 0.843). Saving model...\n",
      "Epoch: 6/1000 Step: 360/361 Training Loss: 1.127 Validation Loss: 0.833 Accuracy: 0.756\n",
      "Validation loss decreased (0.843 -> 0.833). Saving model...\n",
      "Epoch: 7/1000 Step: 180/361 Training Loss: 1.142 Validation Loss: 0.834 Accuracy: 0.752\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 7/1000 Step: 360/361 Training Loss: 1.108 Validation Loss: 0.841 Accuracy: 0.754\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 8/1000 Step: 180/361 Training Loss: 1.085 Validation Loss: 0.861 Accuracy: 0.743\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 8/1000 Step: 360/361 Training Loss: 1.084 Validation Loss: 0.822 Accuracy: 0.756\n",
      "Validation loss decreased (0.833 -> 0.822). Saving model...\n",
      "Epoch: 9/1000 Step: 180/361 Training Loss: 1.080 Validation Loss: 0.769 Accuracy: 0.775\n",
      "Validation loss decreased (0.822 -> 0.769). Saving model...\n",
      "Epoch: 9/1000 Step: 360/361 Training Loss: 1.053 Validation Loss: 0.790 Accuracy: 0.767\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 10/1000 Step: 180/361 Training Loss: 1.047 Validation Loss: 0.781 Accuracy: 0.765\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 10/1000 Step: 360/361 Training Loss: 1.018 Validation Loss: 0.782 Accuracy: 0.770\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 11/1000 Step: 180/361 Training Loss: 1.041 Validation Loss: 0.772 Accuracy: 0.776\n",
      "Early stopping counter: 4/5.\n",
      "Epoch: 11/1000 Step: 360/361 Training Loss: 1.032 Validation Loss: 0.754 Accuracy: 0.771\n",
      "Validation loss decreased (0.769 -> 0.754). Saving model...\n",
      "Epoch: 12/1000 Step: 180/361 Training Loss: 1.007 Validation Loss: 0.747 Accuracy: 0.787\n",
      "Validation loss decreased (0.754 -> 0.747). Saving model...\n",
      "Epoch: 12/1000 Step: 360/361 Training Loss: 1.031 Validation Loss: 0.743 Accuracy: 0.784\n",
      "Validation loss decreased (0.747 -> 0.743). Saving model...\n",
      "Epoch: 13/1000 Step: 180/361 Training Loss: 0.993 Validation Loss: 0.745 Accuracy: 0.781\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 13/1000 Step: 360/361 Training Loss: 1.001 Validation Loss: 0.739 Accuracy: 0.785\n",
      "Validation loss decreased (0.743 -> 0.739). Saving model...\n",
      "Epoch: 14/1000 Step: 180/361 Training Loss: 0.992 Validation Loss: 0.732 Accuracy: 0.783\n",
      "Validation loss decreased (0.739 -> 0.732). Saving model...\n",
      "Epoch: 14/1000 Step: 360/361 Training Loss: 0.993 Validation Loss: 0.727 Accuracy: 0.789\n",
      "Validation loss decreased (0.732 -> 0.727). Saving model...\n",
      "Epoch: 15/1000 Step: 180/361 Training Loss: 1.004 Validation Loss: 0.763 Accuracy: 0.767\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 15/1000 Step: 360/361 Training Loss: 0.977 Validation Loss: 0.728 Accuracy: 0.783\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 16/1000 Step: 180/361 Training Loss: 0.969 Validation Loss: 0.756 Accuracy: 0.773\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 16/1000 Step: 360/361 Training Loss: 0.968 Validation Loss: 0.706 Accuracy: 0.791\n",
      "Validation loss decreased (0.727 -> 0.706). Saving model...\n",
      "Epoch: 17/1000 Step: 180/361 Training Loss: 0.984 Validation Loss: 0.734 Accuracy: 0.784\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 17/1000 Step: 360/361 Training Loss: 0.977 Validation Loss: 0.726 Accuracy: 0.783\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 18/1000 Step: 180/361 Training Loss: 0.969 Validation Loss: 0.715 Accuracy: 0.788\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 18/1000 Step: 360/361 Training Loss: 0.964 Validation Loss: 0.735 Accuracy: 0.774\n",
      "Early stopping counter: 4/5.\n",
      "Epoch: 19/1000 Step: 180/361 Training Loss: 0.969 Validation Loss: 0.728 Accuracy: 0.791\n",
      "Early stopping counter: 5/5.\n",
      "Early stopping limit reached. Training terminated.\n",
      "Total training time: 1.00 hrs 5.00 mins 2.53 secs\n",
      "\n",
      "Training: MobileNet-V2_32_256_256\n",
      "Epoch: 1/1000 Step: 180/361 Training Loss: 2.505 Validation Loss: 1.525 Accuracy: 0.563\n",
      "Validation loss decreased (inf -> 1.525). Saving model...\n",
      "Epoch: 1/1000 Step: 360/361 Training Loss: 1.438 Validation Loss: 0.961 Accuracy: 0.727\n",
      "Validation loss decreased (1.525 -> 0.961). Saving model...\n",
      "Epoch: 2/1000 Step: 180/361 Training Loss: 1.212 Validation Loss: 0.804 Accuracy: 0.765\n",
      "Validation loss decreased (0.961 -> 0.804). Saving model...\n",
      "Epoch: 2/1000 Step: 360/361 Training Loss: 1.060 Validation Loss: 0.792 Accuracy: 0.777\n",
      "Validation loss decreased (0.804 -> 0.792). Saving model...\n",
      "Epoch: 3/1000 Step: 180/361 Training Loss: 1.008 Validation Loss: 0.727 Accuracy: 0.788\n",
      "Validation loss decreased (0.792 -> 0.727). Saving model...\n",
      "Epoch: 3/1000 Step: 360/361 Training Loss: 0.977 Validation Loss: 0.704 Accuracy: 0.799\n",
      "Validation loss decreased (0.727 -> 0.704). Saving model...\n",
      "Epoch: 4/1000 Step: 180/361 Training Loss: 0.930 Validation Loss: 0.676 Accuracy: 0.808\n",
      "Validation loss decreased (0.704 -> 0.676). Saving model...\n",
      "Epoch: 4/1000 Step: 360/361 Training Loss: 0.896 Validation Loss: 0.663 Accuracy: 0.801\n",
      "Validation loss decreased (0.676 -> 0.663). Saving model...\n",
      "Epoch: 5/1000 Step: 180/361 Training Loss: 0.896 Validation Loss: 0.620 Accuracy: 0.820\n",
      "Validation loss decreased (0.663 -> 0.620). Saving model...\n",
      "Epoch: 5/1000 Step: 360/361 Training Loss: 0.822 Validation Loss: 0.629 Accuracy: 0.813\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 6/1000 Step: 180/361 Training Loss: 0.813 Validation Loss: 0.626 Accuracy: 0.816\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 6/1000 Step: 360/361 Training Loss: 0.821 Validation Loss: 0.594 Accuracy: 0.824\n",
      "Validation loss decreased (0.620 -> 0.594). Saving model...\n",
      "Epoch: 7/1000 Step: 180/361 Training Loss: 0.807 Validation Loss: 0.617 Accuracy: 0.818\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 7/1000 Step: 360/361 Training Loss: 0.789 Validation Loss: 0.592 Accuracy: 0.828\n",
      "Validation loss decreased (0.594 -> 0.592). Saving model...\n",
      "Epoch: 8/1000 Step: 180/361 Training Loss: 0.761 Validation Loss: 0.609 Accuracy: 0.820\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 8/1000 Step: 360/361 Training Loss: 0.766 Validation Loss: 0.559 Accuracy: 0.837\n",
      "Validation loss decreased (0.592 -> 0.559). Saving model...\n",
      "Epoch: 9/1000 Step: 180/361 Training Loss: 0.769 Validation Loss: 0.554 Accuracy: 0.843\n",
      "Validation loss decreased (0.559 -> 0.554). Saving model...\n",
      "Epoch: 9/1000 Step: 360/361 Training Loss: 0.721 Validation Loss: 0.563 Accuracy: 0.832\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 10/1000 Step: 180/361 Training Loss: 0.720 Validation Loss: 0.595 Accuracy: 0.827\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 10/1000 Step: 360/361 Training Loss: 0.751 Validation Loss: 0.558 Accuracy: 0.835\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 11/1000 Step: 180/361 Training Loss: 0.713 Validation Loss: 0.586 Accuracy: 0.818\n",
      "Early stopping counter: 4/5.\n",
      "Epoch: 11/1000 Step: 360/361 Training Loss: 0.749 Validation Loss: 0.555 Accuracy: 0.838\n",
      "Early stopping counter: 5/5.\n",
      "Early stopping limit reached. Training terminated.\n",
      "Total training time: 0.00 hrs 40.00 mins 41.43 secs\n",
      "\n",
      "Training: ResNet-34_32_256_256\n",
      "Epoch: 1/1000 Step: 180/361 Training Loss: 2.669 Validation Loss: 1.517 Accuracy: 0.582\n",
      "Validation loss decreased (inf -> 1.517). Saving model...\n",
      "Epoch: 1/1000 Step: 360/361 Training Loss: 1.554 Validation Loss: 1.089 Accuracy: 0.683\n",
      "Validation loss decreased (1.517 -> 1.089). Saving model...\n",
      "Epoch: 2/1000 Step: 180/361 Training Loss: 1.348 Validation Loss: 0.936 Accuracy: 0.730\n",
      "Validation loss decreased (1.089 -> 0.936). Saving model...\n",
      "Epoch: 2/1000 Step: 360/361 Training Loss: 1.178 Validation Loss: 0.839 Accuracy: 0.752\n",
      "Validation loss decreased (0.936 -> 0.839). Saving model...\n",
      "Epoch: 3/1000 Step: 180/361 Training Loss: 1.101 Validation Loss: 0.877 Accuracy: 0.740\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 3/1000 Step: 360/361 Training Loss: 1.070 Validation Loss: 0.823 Accuracy: 0.746\n",
      "Validation loss decreased (0.839 -> 0.823). Saving model...\n",
      "Epoch: 4/1000 Step: 180/361 Training Loss: 1.019 Validation Loss: 0.737 Accuracy: 0.780\n",
      "Validation loss decreased (0.823 -> 0.737). Saving model...\n",
      "Epoch: 4/1000 Step: 360/361 Training Loss: 1.014 Validation Loss: 0.762 Accuracy: 0.769\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 5/1000 Step: 180/361 Training Loss: 0.975 Validation Loss: 0.730 Accuracy: 0.786\n",
      "Validation loss decreased (0.737 -> 0.730). Saving model...\n",
      "Epoch: 5/1000 Step: 360/361 Training Loss: 0.904 Validation Loss: 0.711 Accuracy: 0.787\n",
      "Validation loss decreased (0.730 -> 0.711). Saving model...\n",
      "Epoch: 6/1000 Step: 180/361 Training Loss: 0.953 Validation Loss: 0.759 Accuracy: 0.771\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 6/1000 Step: 360/361 Training Loss: 0.901 Validation Loss: 0.731 Accuracy: 0.789\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 7/1000 Step: 180/361 Training Loss: 0.861 Validation Loss: 0.671 Accuracy: 0.806\n",
      "Validation loss decreased (0.711 -> 0.671). Saving model...\n",
      "Epoch: 7/1000 Step: 360/361 Training Loss: 0.868 Validation Loss: 0.650 Accuracy: 0.805\n",
      "Validation loss decreased (0.671 -> 0.650). Saving model...\n",
      "Epoch: 8/1000 Step: 180/361 Training Loss: 0.837 Validation Loss: 0.629 Accuracy: 0.812\n",
      "Validation loss decreased (0.650 -> 0.629). Saving model...\n",
      "Epoch: 8/1000 Step: 360/361 Training Loss: 0.860 Validation Loss: 0.667 Accuracy: 0.803\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 9/1000 Step: 180/361 Training Loss: 0.802 Validation Loss: 0.663 Accuracy: 0.801\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 9/1000 Step: 360/361 Training Loss: 0.854 Validation Loss: 0.649 Accuracy: 0.803\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 10/1000 Step: 180/361 Training Loss: 0.847 Validation Loss: 0.654 Accuracy: 0.797\n",
      "Early stopping counter: 4/5.\n",
      "Epoch: 10/1000 Step: 360/361 Training Loss: 0.825 Validation Loss: 0.641 Accuracy: 0.808\n",
      "Early stopping counter: 5/5.\n",
      "Early stopping limit reached. Training terminated.\n",
      "Total training time: 0.00 hrs 37.00 mins 37.17 secs\n",
      "\n",
      "Training: GoogleNet_32_512_256\n",
      "Epoch: 1/1000 Step: 180/361 Training Loss: 2.838 Validation Loss: 1.757 Accuracy: 0.511\n",
      "Validation loss decreased (inf -> 1.757). Saving model...\n",
      "Epoch: 1/1000 Step: 360/361 Training Loss: 1.737 Validation Loss: 1.317 Accuracy: 0.616\n",
      "Validation loss decreased (1.757 -> 1.317). Saving model...\n",
      "Epoch: 2/1000 Step: 180/361 Training Loss: 1.517 Validation Loss: 1.193 Accuracy: 0.645\n",
      "Validation loss decreased (1.317 -> 1.193). Saving model...\n",
      "Epoch: 2/1000 Step: 360/361 Training Loss: 1.392 Validation Loss: 1.066 Accuracy: 0.692\n",
      "Validation loss decreased (1.193 -> 1.066). Saving model...\n",
      "Epoch: 3/1000 Step: 180/361 Training Loss: 1.258 Validation Loss: 1.012 Accuracy: 0.706\n",
      "Validation loss decreased (1.066 -> 1.012). Saving model...\n",
      "Epoch: 3/1000 Step: 360/361 Training Loss: 1.237 Validation Loss: 0.962 Accuracy: 0.721\n",
      "Validation loss decreased (1.012 -> 0.962). Saving model...\n",
      "Epoch: 4/1000 Step: 180/361 Training Loss: 1.215 Validation Loss: 0.917 Accuracy: 0.736\n",
      "Validation loss decreased (0.962 -> 0.917). Saving model...\n",
      "Epoch: 4/1000 Step: 360/361 Training Loss: 1.164 Validation Loss: 0.893 Accuracy: 0.746\n",
      "Validation loss decreased (0.917 -> 0.893). Saving model...\n",
      "Epoch: 5/1000 Step: 180/361 Training Loss: 1.157 Validation Loss: 0.868 Accuracy: 0.746\n",
      "Validation loss decreased (0.893 -> 0.868). Saving model...\n",
      "Epoch: 5/1000 Step: 360/361 Training Loss: 1.105 Validation Loss: 0.823 Accuracy: 0.751\n",
      "Validation loss decreased (0.868 -> 0.823). Saving model...\n",
      "Epoch: 6/1000 Step: 180/361 Training Loss: 1.120 Validation Loss: 0.805 Accuracy: 0.769\n",
      "Validation loss decreased (0.823 -> 0.805). Saving model...\n",
      "Epoch: 6/1000 Step: 360/361 Training Loss: 1.040 Validation Loss: 0.826 Accuracy: 0.768\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 7/1000 Step: 180/361 Training Loss: 1.035 Validation Loss: 0.785 Accuracy: 0.777\n",
      "Validation loss decreased (0.805 -> 0.785). Saving model...\n",
      "Epoch: 7/1000 Step: 360/361 Training Loss: 1.040 Validation Loss: 0.808 Accuracy: 0.764\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 8/1000 Step: 180/361 Training Loss: 1.017 Validation Loss: 0.809 Accuracy: 0.762\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 8/1000 Step: 360/361 Training Loss: 1.003 Validation Loss: 0.775 Accuracy: 0.769\n",
      "Validation loss decreased (0.785 -> 0.775). Saving model...\n",
      "Epoch: 9/1000 Step: 180/361 Training Loss: 0.974 Validation Loss: 0.770 Accuracy: 0.783\n",
      "Validation loss decreased (0.775 -> 0.770). Saving model...\n",
      "Epoch: 9/1000 Step: 360/361 Training Loss: 0.991 Validation Loss: 0.745 Accuracy: 0.774\n",
      "Validation loss decreased (0.770 -> 0.745). Saving model...\n",
      "Epoch: 10/1000 Step: 180/361 Training Loss: 0.970 Validation Loss: 0.753 Accuracy: 0.774\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 10/1000 Step: 360/361 Training Loss: 0.974 Validation Loss: 0.754 Accuracy: 0.775\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 11/1000 Step: 180/361 Training Loss: 0.977 Validation Loss: 0.724 Accuracy: 0.779\n",
      "Validation loss decreased (0.745 -> 0.724). Saving model...\n",
      "Epoch: 11/1000 Step: 360/361 Training Loss: 0.931 Validation Loss: 0.735 Accuracy: 0.787\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 12/1000 Step: 180/361 Training Loss: 0.950 Validation Loss: 0.732 Accuracy: 0.800\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 12/1000 Step: 360/361 Training Loss: 0.918 Validation Loss: 0.779 Accuracy: 0.766\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 13/1000 Step: 180/361 Training Loss: 0.915 Validation Loss: 0.732 Accuracy: 0.785\n",
      "Early stopping counter: 4/5.\n",
      "Epoch: 13/1000 Step: 360/361 Training Loss: 0.935 Validation Loss: 0.736 Accuracy: 0.785\n",
      "Early stopping counter: 5/5.\n",
      "Early stopping limit reached. Training terminated.\n",
      "Total training time: 0.00 hrs 42.00 mins 48.27 secs\n",
      "\n",
      "Training: MobileNet-V2_32_512_256\n",
      "Epoch: 1/1000 Step: 180/361 Training Loss: 2.367 Validation Loss: 1.221 Accuracy: 0.647\n",
      "Validation loss decreased (inf -> 1.221). Saving model...\n",
      "Epoch: 1/1000 Step: 360/361 Training Loss: 1.291 Validation Loss: 0.866 Accuracy: 0.750\n",
      "Validation loss decreased (1.221 -> 0.866). Saving model...\n",
      "Epoch: 2/1000 Step: 180/361 Training Loss: 1.090 Validation Loss: 0.836 Accuracy: 0.746\n",
      "Validation loss decreased (0.866 -> 0.836). Saving model...\n",
      "Epoch: 2/1000 Step: 360/361 Training Loss: 0.991 Validation Loss: 0.735 Accuracy: 0.781\n",
      "Validation loss decreased (0.836 -> 0.735). Saving model...\n",
      "Epoch: 3/1000 Step: 180/361 Training Loss: 0.919 Validation Loss: 0.699 Accuracy: 0.800\n",
      "Validation loss decreased (0.735 -> 0.699). Saving model...\n",
      "Epoch: 3/1000 Step: 360/361 Training Loss: 0.907 Validation Loss: 0.700 Accuracy: 0.791\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 4/1000 Step: 180/361 Training Loss: 0.831 Validation Loss: 0.710 Accuracy: 0.793\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 4/1000 Step: 360/361 Training Loss: 0.829 Validation Loss: 0.672 Accuracy: 0.810\n",
      "Validation loss decreased (0.699 -> 0.672). Saving model...\n",
      "Epoch: 5/1000 Step: 180/361 Training Loss: 0.791 Validation Loss: 0.592 Accuracy: 0.823\n",
      "Validation loss decreased (0.672 -> 0.592). Saving model...\n",
      "Epoch: 5/1000 Step: 360/361 Training Loss: 0.794 Validation Loss: 0.568 Accuracy: 0.831\n",
      "Validation loss decreased (0.592 -> 0.568). Saving model...\n",
      "Epoch: 6/1000 Step: 180/361 Training Loss: 0.723 Validation Loss: 0.604 Accuracy: 0.826\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 6/1000 Step: 360/361 Training Loss: 0.742 Validation Loss: 0.542 Accuracy: 0.849\n",
      "Validation loss decreased (0.568 -> 0.542). Saving model...\n",
      "Epoch: 7/1000 Step: 180/361 Training Loss: 0.712 Validation Loss: 0.617 Accuracy: 0.819\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 7/1000 Step: 360/361 Training Loss: 0.719 Validation Loss: 0.605 Accuracy: 0.826\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 8/1000 Step: 180/361 Training Loss: 0.691 Validation Loss: 0.559 Accuracy: 0.838\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 8/1000 Step: 360/361 Training Loss: 0.695 Validation Loss: 0.591 Accuracy: 0.829\n",
      "Early stopping counter: 4/5.\n",
      "Epoch: 9/1000 Step: 180/361 Training Loss: 0.688 Validation Loss: 0.552 Accuracy: 0.834\n",
      "Early stopping counter: 5/5.\n",
      "Early stopping limit reached. Training terminated.\n",
      "Total training time: 0.00 hrs 29.00 mins 48.48 secs\n",
      "\n",
      "Training: ResNet-34_32_512_256\n",
      "Epoch: 1/1000 Step: 180/361 Training Loss: 2.522 Validation Loss: 1.432 Accuracy: 0.574\n",
      "Validation loss decreased (inf -> 1.432). Saving model...\n",
      "Epoch: 1/1000 Step: 360/361 Training Loss: 1.469 Validation Loss: 1.055 Accuracy: 0.697\n",
      "Validation loss decreased (1.432 -> 1.055). Saving model...\n",
      "Epoch: 2/1000 Step: 180/361 Training Loss: 1.241 Validation Loss: 0.889 Accuracy: 0.731\n",
      "Validation loss decreased (1.055 -> 0.889). Saving model...\n",
      "Epoch: 2/1000 Step: 360/361 Training Loss: 1.121 Validation Loss: 0.845 Accuracy: 0.754\n",
      "Validation loss decreased (0.889 -> 0.845). Saving model...\n",
      "Epoch: 3/1000 Step: 180/361 Training Loss: 1.026 Validation Loss: 0.795 Accuracy: 0.761\n",
      "Validation loss decreased (0.845 -> 0.795). Saving model...\n",
      "Epoch: 3/1000 Step: 360/361 Training Loss: 0.986 Validation Loss: 0.766 Accuracy: 0.773\n",
      "Validation loss decreased (0.795 -> 0.766). Saving model...\n",
      "Epoch: 4/1000 Step: 180/361 Training Loss: 0.944 Validation Loss: 0.732 Accuracy: 0.787\n",
      "Validation loss decreased (0.766 -> 0.732). Saving model...\n",
      "Epoch: 4/1000 Step: 360/361 Training Loss: 0.907 Validation Loss: 0.759 Accuracy: 0.764\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 5/1000 Step: 180/361 Training Loss: 0.866 Validation Loss: 0.710 Accuracy: 0.786\n",
      "Validation loss decreased (0.732 -> 0.710). Saving model...\n",
      "Epoch: 5/1000 Step: 360/361 Training Loss: 0.893 Validation Loss: 0.697 Accuracy: 0.786\n",
      "Validation loss decreased (0.710 -> 0.697). Saving model...\n",
      "Epoch: 6/1000 Step: 180/361 Training Loss: 0.847 Validation Loss: 0.676 Accuracy: 0.802\n",
      "Validation loss decreased (0.697 -> 0.676). Saving model...\n",
      "Epoch: 6/1000 Step: 360/361 Training Loss: 0.832 Validation Loss: 0.677 Accuracy: 0.803\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 7/1000 Step: 180/361 Training Loss: 0.817 Validation Loss: 0.661 Accuracy: 0.806\n",
      "Validation loss decreased (0.676 -> 0.661). Saving model...\n",
      "Epoch: 7/1000 Step: 360/361 Training Loss: 0.808 Validation Loss: 0.639 Accuracy: 0.803\n",
      "Validation loss decreased (0.661 -> 0.639). Saving model...\n",
      "Epoch: 8/1000 Step: 180/361 Training Loss: 0.813 Validation Loss: 0.648 Accuracy: 0.811\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 8/1000 Step: 360/361 Training Loss: 0.785 Validation Loss: 0.631 Accuracy: 0.803\n",
      "Validation loss decreased (0.639 -> 0.631). Saving model...\n",
      "Epoch: 9/1000 Step: 180/361 Training Loss: 0.765 Validation Loss: 0.605 Accuracy: 0.819\n",
      "Validation loss decreased (0.631 -> 0.605). Saving model...\n",
      "Epoch: 9/1000 Step: 360/361 Training Loss: 0.767 Validation Loss: 0.639 Accuracy: 0.814\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 10/1000 Step: 180/361 Training Loss: 0.727 Validation Loss: 0.655 Accuracy: 0.799\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 10/1000 Step: 360/361 Training Loss: 0.745 Validation Loss: 0.613 Accuracy: 0.820\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 11/1000 Step: 180/361 Training Loss: 0.734 Validation Loss: 0.641 Accuracy: 0.806\n",
      "Early stopping counter: 4/5.\n",
      "Epoch: 11/1000 Step: 360/361 Training Loss: 0.751 Validation Loss: 0.612 Accuracy: 0.822\n",
      "Early stopping counter: 5/5.\n",
      "Early stopping limit reached. Training terminated.\n",
      "Total training time: 0.00 hrs 40.00 mins 28.14 secs\n",
      "\n",
      "Training: GoogleNet_32_512_512\n",
      "Epoch: 1/1000 Step: 180/361 Training Loss: 2.588 Validation Loss: 1.488 Accuracy: 0.580\n",
      "Validation loss decreased (inf -> 1.488). Saving model...\n",
      "Epoch: 1/1000 Step: 360/361 Training Loss: 1.655 Validation Loss: 1.232 Accuracy: 0.633\n",
      "Validation loss decreased (1.488 -> 1.232). Saving model...\n",
      "Epoch: 2/1000 Step: 180/361 Training Loss: 1.420 Validation Loss: 1.037 Accuracy: 0.696\n",
      "Validation loss decreased (1.232 -> 1.037). Saving model...\n",
      "Epoch: 2/1000 Step: 360/361 Training Loss: 1.310 Validation Loss: 1.036 Accuracy: 0.692\n",
      "Validation loss decreased (1.037 -> 1.036). Saving model...\n",
      "Epoch: 3/1000 Step: 180/361 Training Loss: 1.266 Validation Loss: 0.939 Accuracy: 0.733\n",
      "Validation loss decreased (1.036 -> 0.939). Saving model...\n",
      "Epoch: 3/1000 Step: 360/361 Training Loss: 1.180 Validation Loss: 0.937 Accuracy: 0.715\n",
      "Validation loss decreased (0.939 -> 0.937). Saving model...\n",
      "Epoch: 4/1000 Step: 180/361 Training Loss: 1.145 Validation Loss: 0.834 Accuracy: 0.757\n",
      "Validation loss decreased (0.937 -> 0.834). Saving model...\n",
      "Epoch: 4/1000 Step: 360/361 Training Loss: 1.130 Validation Loss: 0.866 Accuracy: 0.739\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 5/1000 Step: 180/361 Training Loss: 1.105 Validation Loss: 0.887 Accuracy: 0.733\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 5/1000 Step: 360/361 Training Loss: 1.047 Validation Loss: 0.820 Accuracy: 0.755\n",
      "Validation loss decreased (0.834 -> 0.820). Saving model...\n",
      "Epoch: 6/1000 Step: 180/361 Training Loss: 1.065 Validation Loss: 0.837 Accuracy: 0.755\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 6/1000 Step: 360/361 Training Loss: 1.034 Validation Loss: 0.795 Accuracy: 0.763\n",
      "Validation loss decreased (0.820 -> 0.795). Saving model...\n",
      "Epoch: 7/1000 Step: 180/361 Training Loss: 1.022 Validation Loss: 0.803 Accuracy: 0.761\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 7/1000 Step: 360/361 Training Loss: 1.014 Validation Loss: 0.775 Accuracy: 0.766\n",
      "Validation loss decreased (0.795 -> 0.775). Saving model...\n",
      "Epoch: 8/1000 Step: 180/361 Training Loss: 0.983 Validation Loss: 0.732 Accuracy: 0.783\n",
      "Validation loss decreased (0.775 -> 0.732). Saving model...\n",
      "Epoch: 8/1000 Step: 360/361 Training Loss: 0.995 Validation Loss: 0.760 Accuracy: 0.774\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 9/1000 Step: 180/361 Training Loss: 0.967 Validation Loss: 0.784 Accuracy: 0.769\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 9/1000 Step: 360/361 Training Loss: 0.959 Validation Loss: 0.756 Accuracy: 0.776\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 10/1000 Step: 180/361 Training Loss: 0.946 Validation Loss: 0.741 Accuracy: 0.776\n",
      "Early stopping counter: 4/5.\n",
      "Epoch: 10/1000 Step: 360/361 Training Loss: 0.931 Validation Loss: 0.767 Accuracy: 0.776\n",
      "Early stopping counter: 5/5.\n",
      "Early stopping limit reached. Training terminated.\n",
      "Total training time: 0.00 hrs 36.00 mins 2.11 secs\n",
      "\n",
      "Training: MobileNet-V2_32_512_512\n",
      "Epoch: 1/1000 Step: 180/361 Training Loss: 2.228 Validation Loss: 1.178 Accuracy: 0.664\n",
      "Validation loss decreased (inf -> 1.178). Saving model...\n",
      "Epoch: 1/1000 Step: 360/361 Training Loss: 1.223 Validation Loss: 0.861 Accuracy: 0.752\n",
      "Validation loss decreased (1.178 -> 0.861). Saving model...\n",
      "Epoch: 2/1000 Step: 180/361 Training Loss: 1.046 Validation Loss: 0.746 Accuracy: 0.774\n",
      "Validation loss decreased (0.861 -> 0.746). Saving model...\n",
      "Epoch: 2/1000 Step: 360/361 Training Loss: 1.009 Validation Loss: 0.731 Accuracy: 0.778\n",
      "Validation loss decreased (0.746 -> 0.731). Saving model...\n",
      "Epoch: 3/1000 Step: 180/361 Training Loss: 0.915 Validation Loss: 0.751 Accuracy: 0.767\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 3/1000 Step: 360/361 Training Loss: 0.868 Validation Loss: 0.631 Accuracy: 0.809\n",
      "Validation loss decreased (0.731 -> 0.631). Saving model...\n",
      "Epoch: 4/1000 Step: 180/361 Training Loss: 0.826 Validation Loss: 0.620 Accuracy: 0.826\n",
      "Validation loss decreased (0.631 -> 0.620). Saving model...\n",
      "Epoch: 4/1000 Step: 360/361 Training Loss: 0.797 Validation Loss: 0.601 Accuracy: 0.818\n",
      "Validation loss decreased (0.620 -> 0.601). Saving model...\n",
      "Epoch: 5/1000 Step: 180/361 Training Loss: 0.764 Validation Loss: 0.607 Accuracy: 0.817\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 5/1000 Step: 360/361 Training Loss: 0.759 Validation Loss: 0.621 Accuracy: 0.814\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 6/1000 Step: 180/361 Training Loss: 0.734 Validation Loss: 0.564 Accuracy: 0.839\n",
      "Validation loss decreased (0.601 -> 0.564). Saving model...\n",
      "Epoch: 6/1000 Step: 360/361 Training Loss: 0.704 Validation Loss: 0.603 Accuracy: 0.828\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 7/1000 Step: 180/361 Training Loss: 0.713 Validation Loss: 0.579 Accuracy: 0.834\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 7/1000 Step: 360/361 Training Loss: 0.683 Validation Loss: 0.567 Accuracy: 0.829\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 8/1000 Step: 180/361 Training Loss: 0.684 Validation Loss: 0.548 Accuracy: 0.839\n",
      "Validation loss decreased (0.564 -> 0.548). Saving model...\n",
      "Epoch: 8/1000 Step: 360/361 Training Loss: 0.688 Validation Loss: 0.548 Accuracy: 0.840\n",
      "Validation loss decreased (0.548 -> 0.548). Saving model...\n",
      "Epoch: 9/1000 Step: 180/361 Training Loss: 0.644 Validation Loss: 0.572 Accuracy: 0.834\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 9/1000 Step: 360/361 Training Loss: 0.670 Validation Loss: 0.600 Accuracy: 0.823\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 10/1000 Step: 180/361 Training Loss: 0.641 Validation Loss: 0.548 Accuracy: 0.843\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 10/1000 Step: 360/361 Training Loss: 0.676 Validation Loss: 0.521 Accuracy: 0.851\n",
      "Validation loss decreased (0.548 -> 0.521). Saving model...\n",
      "Epoch: 11/1000 Step: 180/361 Training Loss: 0.636 Validation Loss: 0.550 Accuracy: 0.838\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 11/1000 Step: 360/361 Training Loss: 0.650 Validation Loss: 0.546 Accuracy: 0.840\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 12/1000 Step: 180/361 Training Loss: 0.609 Validation Loss: 0.536 Accuracy: 0.834\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 12/1000 Step: 360/361 Training Loss: 0.649 Validation Loss: 0.566 Accuracy: 0.836\n",
      "Early stopping counter: 4/5.\n",
      "Epoch: 13/1000 Step: 180/361 Training Loss: 0.598 Validation Loss: 0.534 Accuracy: 0.843\n",
      "Early stopping counter: 5/5.\n",
      "Early stopping limit reached. Training terminated.\n",
      "Total training time: 0.00 hrs 42.00 mins 2.45 secs\n",
      "\n",
      "Training: ResNet-34_32_512_512\n",
      "Epoch: 1/1000 Step: 180/361 Training Loss: 2.331 Validation Loss: 1.347 Accuracy: 0.602\n",
      "Validation loss decreased (inf -> 1.347). Saving model...\n",
      "Epoch: 1/1000 Step: 360/361 Training Loss: 1.374 Validation Loss: 0.968 Accuracy: 0.715\n",
      "Validation loss decreased (1.347 -> 0.968). Saving model...\n",
      "Epoch: 2/1000 Step: 180/361 Training Loss: 1.173 Validation Loss: 0.894 Accuracy: 0.742\n",
      "Validation loss decreased (0.968 -> 0.894). Saving model...\n",
      "Epoch: 2/1000 Step: 360/361 Training Loss: 1.033 Validation Loss: 0.788 Accuracy: 0.757\n",
      "Validation loss decreased (0.894 -> 0.788). Saving model...\n",
      "Epoch: 3/1000 Step: 180/361 Training Loss: 1.031 Validation Loss: 0.804 Accuracy: 0.754\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 3/1000 Step: 360/361 Training Loss: 0.980 Validation Loss: 0.761 Accuracy: 0.780\n",
      "Validation loss decreased (0.788 -> 0.761). Saving model...\n",
      "Epoch: 4/1000 Step: 180/361 Training Loss: 0.885 Validation Loss: 0.725 Accuracy: 0.783\n",
      "Validation loss decreased (0.761 -> 0.725). Saving model...\n",
      "Epoch: 4/1000 Step: 360/361 Training Loss: 0.939 Validation Loss: 0.735 Accuracy: 0.775\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 5/1000 Step: 180/361 Training Loss: 0.871 Validation Loss: 0.664 Accuracy: 0.797\n",
      "Validation loss decreased (0.725 -> 0.664). Saving model...\n",
      "Epoch: 5/1000 Step: 360/361 Training Loss: 0.873 Validation Loss: 0.682 Accuracy: 0.790\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 6/1000 Step: 180/361 Training Loss: 0.820 Validation Loss: 0.686 Accuracy: 0.800\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 6/1000 Step: 360/361 Training Loss: 0.850 Validation Loss: 0.660 Accuracy: 0.803\n",
      "Validation loss decreased (0.664 -> 0.660). Saving model...\n",
      "Epoch: 7/1000 Step: 180/361 Training Loss: 0.796 Validation Loss: 0.654 Accuracy: 0.799\n",
      "Validation loss decreased (0.660 -> 0.654). Saving model...\n",
      "Epoch: 7/1000 Step: 360/361 Training Loss: 0.801 Validation Loss: 0.653 Accuracy: 0.804\n",
      "Validation loss decreased (0.654 -> 0.653). Saving model...\n",
      "Epoch: 8/1000 Step: 180/361 Training Loss: 0.773 Validation Loss: 0.652 Accuracy: 0.803\n",
      "Validation loss decreased (0.653 -> 0.652). Saving model...\n",
      "Epoch: 8/1000 Step: 360/361 Training Loss: 0.799 Validation Loss: 0.698 Accuracy: 0.791\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 9/1000 Step: 180/361 Training Loss: 0.750 Validation Loss: 0.610 Accuracy: 0.815\n",
      "Validation loss decreased (0.652 -> 0.610). Saving model...\n",
      "Epoch: 9/1000 Step: 360/361 Training Loss: 0.789 Validation Loss: 0.647 Accuracy: 0.814\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 10/1000 Step: 180/361 Training Loss: 0.734 Validation Loss: 0.656 Accuracy: 0.808\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 10/1000 Step: 360/361 Training Loss: 0.756 Validation Loss: 0.621 Accuracy: 0.810\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 11/1000 Step: 180/361 Training Loss: 0.742 Validation Loss: 0.609 Accuracy: 0.814\n",
      "Validation loss decreased (0.610 -> 0.609). Saving model...\n",
      "Epoch: 11/1000 Step: 360/361 Training Loss: 0.716 Validation Loss: 0.651 Accuracy: 0.808\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 12/1000 Step: 180/361 Training Loss: 0.720 Validation Loss: 0.610 Accuracy: 0.821\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 12/1000 Step: 360/361 Training Loss: 0.715 Validation Loss: 0.591 Accuracy: 0.830\n",
      "Validation loss decreased (0.609 -> 0.591). Saving model...\n",
      "Epoch: 13/1000 Step: 180/361 Training Loss: 0.705 Validation Loss: 0.669 Accuracy: 0.817\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 13/1000 Step: 360/361 Training Loss: 0.718 Validation Loss: 0.642 Accuracy: 0.812\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 14/1000 Step: 180/361 Training Loss: 0.707 Validation Loss: 0.598 Accuracy: 0.827\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 14/1000 Step: 360/361 Training Loss: 0.665 Validation Loss: 0.624 Accuracy: 0.817\n",
      "Early stopping counter: 4/5.\n",
      "Epoch: 15/1000 Step: 180/361 Training Loss: 0.673 Validation Loss: 0.631 Accuracy: 0.813\n",
      "Early stopping counter: 5/5.\n",
      "Early stopping limit reached. Training terminated.\n",
      "Total training time: 0.00 hrs 49.00 mins 58.53 secs\n",
      "\n",
      "Training: GoogleNet_32_1024_516\n",
      "Epoch: 1/1000 Step: 180/361 Training Loss: 2.537 Validation Loss: 1.502 Accuracy: 0.572\n",
      "Validation loss decreased (inf -> 1.502). Saving model...\n",
      "Epoch: 1/1000 Step: 360/361 Training Loss: 1.594 Validation Loss: 1.165 Accuracy: 0.652\n",
      "Validation loss decreased (1.502 -> 1.165). Saving model...\n",
      "Epoch: 2/1000 Step: 180/361 Training Loss: 1.374 Validation Loss: 1.101 Accuracy: 0.668\n",
      "Validation loss decreased (1.165 -> 1.101). Saving model...\n",
      "Epoch: 2/1000 Step: 360/361 Training Loss: 1.272 Validation Loss: 0.983 Accuracy: 0.715\n",
      "Validation loss decreased (1.101 -> 0.983). Saving model...\n",
      "Epoch: 3/1000 Step: 180/361 Training Loss: 1.205 Validation Loss: 0.974 Accuracy: 0.708\n",
      "Validation loss decreased (0.983 -> 0.974). Saving model...\n",
      "Epoch: 3/1000 Step: 360/361 Training Loss: 1.153 Validation Loss: 0.910 Accuracy: 0.731\n",
      "Validation loss decreased (0.974 -> 0.910). Saving model...\n",
      "Epoch: 4/1000 Step: 180/361 Training Loss: 1.128 Validation Loss: 0.910 Accuracy: 0.730\n",
      "Validation loss decreased (0.910 -> 0.910). Saving model...\n",
      "Epoch: 4/1000 Step: 360/361 Training Loss: 1.070 Validation Loss: 0.837 Accuracy: 0.751\n",
      "Validation loss decreased (0.910 -> 0.837). Saving model...\n",
      "Epoch: 5/1000 Step: 180/361 Training Loss: 1.054 Validation Loss: 0.833 Accuracy: 0.752\n",
      "Validation loss decreased (0.837 -> 0.833). Saving model...\n",
      "Epoch: 5/1000 Step: 360/361 Training Loss: 1.028 Validation Loss: 0.791 Accuracy: 0.763\n",
      "Validation loss decreased (0.833 -> 0.791). Saving model...\n",
      "Epoch: 6/1000 Step: 180/361 Training Loss: 1.016 Validation Loss: 0.851 Accuracy: 0.742\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 6/1000 Step: 360/361 Training Loss: 1.004 Validation Loss: 0.807 Accuracy: 0.754\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 7/1000 Step: 180/361 Training Loss: 0.986 Validation Loss: 0.769 Accuracy: 0.768\n",
      "Validation loss decreased (0.791 -> 0.769). Saving model...\n",
      "Epoch: 7/1000 Step: 360/361 Training Loss: 0.965 Validation Loss: 0.789 Accuracy: 0.770\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 8/1000 Step: 180/361 Training Loss: 0.939 Validation Loss: 0.774 Accuracy: 0.773\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 8/1000 Step: 360/361 Training Loss: 0.952 Validation Loss: 0.779 Accuracy: 0.761\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 9/1000 Step: 180/361 Training Loss: 0.960 Validation Loss: 0.757 Accuracy: 0.779\n",
      "Validation loss decreased (0.769 -> 0.757). Saving model...\n",
      "Epoch: 9/1000 Step: 360/361 Training Loss: 0.911 Validation Loss: 0.746 Accuracy: 0.777\n",
      "Validation loss decreased (0.757 -> 0.746). Saving model...\n",
      "Epoch: 10/1000 Step: 180/361 Training Loss: 0.933 Validation Loss: 0.709 Accuracy: 0.796\n",
      "Validation loss decreased (0.746 -> 0.709). Saving model...\n",
      "Epoch: 10/1000 Step: 360/361 Training Loss: 0.923 Validation Loss: 0.710 Accuracy: 0.787\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 11/1000 Step: 180/361 Training Loss: 0.947 Validation Loss: 0.765 Accuracy: 0.776\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 11/1000 Step: 360/361 Training Loss: 0.923 Validation Loss: 0.729 Accuracy: 0.790\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 12/1000 Step: 180/361 Training Loss: 0.874 Validation Loss: 0.711 Accuracy: 0.792\n",
      "Early stopping counter: 4/5.\n",
      "Epoch: 12/1000 Step: 360/361 Training Loss: 0.880 Validation Loss: 0.681 Accuracy: 0.807\n",
      "Validation loss decreased (0.709 -> 0.681). Saving model...\n",
      "Epoch: 13/1000 Step: 180/361 Training Loss: 0.848 Validation Loss: 0.728 Accuracy: 0.784\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 13/1000 Step: 360/361 Training Loss: 0.871 Validation Loss: 0.741 Accuracy: 0.778\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 14/1000 Step: 180/361 Training Loss: 0.878 Validation Loss: 0.728 Accuracy: 0.789\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 14/1000 Step: 360/361 Training Loss: 0.835 Validation Loss: 0.687 Accuracy: 0.801\n",
      "Early stopping counter: 4/5.\n",
      "Epoch: 15/1000 Step: 180/361 Training Loss: 0.834 Validation Loss: 0.686 Accuracy: 0.796\n",
      "Early stopping counter: 5/5.\n",
      "Early stopping limit reached. Training terminated.\n",
      "Total training time: 0.00 hrs 49.00 mins 45.23 secs\n",
      "\n",
      "Training: MobileNet-V2_32_1024_516\n",
      "Epoch: 1/1000 Step: 180/361 Training Loss: 2.231 Validation Loss: 1.188 Accuracy: 0.664\n",
      "Validation loss decreased (inf -> 1.188). Saving model...\n",
      "Epoch: 1/1000 Step: 360/361 Training Loss: 1.235 Validation Loss: 0.924 Accuracy: 0.722\n",
      "Validation loss decreased (1.188 -> 0.924). Saving model...\n",
      "Epoch: 2/1000 Step: 180/361 Training Loss: 1.020 Validation Loss: 0.817 Accuracy: 0.750\n",
      "Validation loss decreased (0.924 -> 0.817). Saving model...\n",
      "Epoch: 2/1000 Step: 360/361 Training Loss: 0.964 Validation Loss: 0.710 Accuracy: 0.791\n",
      "Validation loss decreased (0.817 -> 0.710). Saving model...\n",
      "Epoch: 3/1000 Step: 180/361 Training Loss: 0.893 Validation Loss: 0.712 Accuracy: 0.781\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 3/1000 Step: 360/361 Training Loss: 0.853 Validation Loss: 0.663 Accuracy: 0.816\n",
      "Validation loss decreased (0.710 -> 0.663). Saving model...\n",
      "Epoch: 4/1000 Step: 180/361 Training Loss: 0.802 Validation Loss: 0.633 Accuracy: 0.812\n",
      "Validation loss decreased (0.663 -> 0.633). Saving model...\n",
      "Epoch: 4/1000 Step: 360/361 Training Loss: 0.780 Validation Loss: 0.615 Accuracy: 0.819\n",
      "Validation loss decreased (0.633 -> 0.615). Saving model...\n",
      "Epoch: 5/1000 Step: 180/361 Training Loss: 0.720 Validation Loss: 0.630 Accuracy: 0.821\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 5/1000 Step: 360/361 Training Loss: 0.760 Validation Loss: 0.606 Accuracy: 0.817\n",
      "Validation loss decreased (0.615 -> 0.606). Saving model...\n",
      "Epoch: 6/1000 Step: 180/361 Training Loss: 0.730 Validation Loss: 0.577 Accuracy: 0.832\n",
      "Validation loss decreased (0.606 -> 0.577). Saving model...\n",
      "Epoch: 6/1000 Step: 360/361 Training Loss: 0.685 Validation Loss: 0.595 Accuracy: 0.828\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 7/1000 Step: 180/361 Training Loss: 0.690 Validation Loss: 0.597 Accuracy: 0.827\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 7/1000 Step: 360/361 Training Loss: 0.700 Validation Loss: 0.607 Accuracy: 0.815\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 8/1000 Step: 180/361 Training Loss: 0.660 Validation Loss: 0.607 Accuracy: 0.820\n",
      "Early stopping counter: 4/5.\n",
      "Epoch: 8/1000 Step: 360/361 Training Loss: 0.673 Validation Loss: 0.585 Accuracy: 0.833\n",
      "Early stopping counter: 5/5.\n",
      "Early stopping limit reached. Training terminated.\n",
      "Total training time: 0.00 hrs 28.00 mins 21.52 secs\n",
      "\n",
      "Training: ResNet-34_32_1024_516\n",
      "Epoch: 1/1000 Step: 180/361 Training Loss: 2.241 Validation Loss: 1.253 Accuracy: 0.616\n",
      "Validation loss decreased (inf -> 1.253). Saving model...\n",
      "Epoch: 1/1000 Step: 360/361 Training Loss: 1.322 Validation Loss: 1.094 Accuracy: 0.664\n",
      "Validation loss decreased (1.253 -> 1.094). Saving model...\n",
      "Epoch: 2/1000 Step: 180/361 Training Loss: 1.111 Validation Loss: 0.826 Accuracy: 0.746\n",
      "Validation loss decreased (1.094 -> 0.826). Saving model...\n",
      "Epoch: 2/1000 Step: 360/361 Training Loss: 1.057 Validation Loss: 0.797 Accuracy: 0.761\n",
      "Validation loss decreased (0.826 -> 0.797). Saving model...\n",
      "Epoch: 3/1000 Step: 180/361 Training Loss: 0.983 Validation Loss: 0.767 Accuracy: 0.765\n",
      "Validation loss decreased (0.797 -> 0.767). Saving model...\n",
      "Epoch: 3/1000 Step: 360/361 Training Loss: 0.943 Validation Loss: 0.752 Accuracy: 0.776\n",
      "Validation loss decreased (0.767 -> 0.752). Saving model...\n",
      "Epoch: 4/1000 Step: 180/361 Training Loss: 0.898 Validation Loss: 0.727 Accuracy: 0.775\n",
      "Validation loss decreased (0.752 -> 0.727). Saving model...\n",
      "Epoch: 4/1000 Step: 360/361 Training Loss: 0.889 Validation Loss: 0.724 Accuracy: 0.783\n",
      "Validation loss decreased (0.727 -> 0.724). Saving model...\n",
      "Epoch: 5/1000 Step: 180/361 Training Loss: 0.869 Validation Loss: 0.772 Accuracy: 0.782\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 5/1000 Step: 360/361 Training Loss: 0.834 Validation Loss: 0.732 Accuracy: 0.781\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 6/1000 Step: 180/361 Training Loss: 0.818 Validation Loss: 0.655 Accuracy: 0.796\n",
      "Validation loss decreased (0.724 -> 0.655). Saving model...\n",
      "Epoch: 6/1000 Step: 360/361 Training Loss: 0.778 Validation Loss: 0.651 Accuracy: 0.805\n",
      "Validation loss decreased (0.655 -> 0.651). Saving model...\n",
      "Epoch: 7/1000 Step: 180/361 Training Loss: 0.765 Validation Loss: 0.655 Accuracy: 0.807\n",
      "Early stopping counter: 1/5.\n",
      "Epoch: 7/1000 Step: 360/361 Training Loss: 0.774 Validation Loss: 0.689 Accuracy: 0.797\n",
      "Early stopping counter: 2/5.\n",
      "Epoch: 8/1000 Step: 180/361 Training Loss: 0.756 Validation Loss: 0.716 Accuracy: 0.799\n",
      "Early stopping counter: 3/5.\n",
      "Epoch: 8/1000 Step: 360/361 Training Loss: 0.758 Validation Loss: 0.710 Accuracy: 0.799\n",
      "Early stopping counter: 4/5.\n",
      "Epoch: 9/1000 Step: 180/361 Training Loss: 0.745 Validation Loss: 0.656 Accuracy: 0.797\n",
      "Early stopping counter: 5/5.\n",
      "Early stopping limit reached. Training terminated.\n",
      "Total training time: 0.00 hrs 31.00 mins 52.06 secs\n",
      "All models trained on 32 batches.\n",
      "\n",
      "GoogleNet_64_256_256 already trained.\n",
      "MobileNet-V2_64_256_256 already trained.\n",
      "ResNet-34_64_256_256 already trained.\n",
      "GoogleNet_64_512_256 already trained.\n",
      "MobileNet-V2_64_512_256 already trained.\n",
      "ResNet-34_64_512_256 already trained.\n",
      "GoogleNet_64_512_512 already trained.\n",
      "MobileNet-V2_64_512_512 already trained.\n",
      "ResNet-34_64_512_512 already trained.\n",
      "GoogleNet_64_1024_516 already trained.\n",
      "MobileNet-V2_64_1024_516 already trained.\n",
      "ResNet-34_64_1024_516 already trained.\n",
      "All models trained on 64 batches.\n",
      "\n",
      "GoogleNet_128_256_256 already trained.\n",
      "MobileNet-V2_128_256_256 already trained.\n",
      "ResNet-34_128_256_256 already trained.\n",
      "GoogleNet_128_512_256 already trained.\n",
      "MobileNet-V2_128_512_256 already trained.\n",
      "ResNet-34_128_512_256 already trained.\n",
      "GoogleNet_128_512_512 already trained.\n",
      "MobileNet-V2_128_512_512 already trained.\n",
      "ResNet-34_128_512_512 already trained.\n",
      "GoogleNet_128_1024_516 already trained.\n",
      "MobileNet-V2_128_1024_516 already trained.\n",
      "ResNet-34_128_1024_516 already trained.\n",
      "All models trained on 128 batches.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(BATCH_SIZES)):\n",
    "    # Split data\n",
    "    train_loader, valid_loader, test_loader = tune.utils.split_data(dataset, \n",
    "                                                                    BATCH_SIZES[i], \n",
    "                                                                    SPLIT_SIZE, SEED)\n",
    "    \n",
    "    # Tune models\n",
    "    tune.tune_model(model_names, BATCH_SIZES[i], train_loader, valid_loader, \n",
    "                    N_CLASSES, H_LAYERS, LEARNING_RATE, EPOCHS)\n",
    "    print(f\"All models trained on {BATCH_SIZES[i]} batches.\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Performance Evaluation\n",
    "Please refer to the Jupyter Notebook `3. visualise_results.ipynb` for this section."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "wepc",
   "language": "python",
   "name": "wepc"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}