examples/08_Optimizing_AUROC_Loss_with_DenseNet121_on_Melanoma.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "08.Optimizing_AUROC_Loss_with_DenseNet121_on_Melanoma.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ud6jh8GWr_qT"
      },
      "source": [
        "* Author: Zhuoning Yuan\n",
        "* Project: https://github.com/Optimization-AI/LibAUC\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dcqqRTWMsFKY"
      },
      "source": [
        "# **Installing LibAUC**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ter36EFG2k2g"
      },
      "source": [
        "!pip install libauc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rZL_V9NutBio"
      },
      "source": [
        "# **Downloading Melanoma Dataset**\n",
        " \n",
        "*  You can download dataset from here: https://www.kaggle.com/cdeotte/jpeg-melanoma-256x256. \n",
        "*  In this tutorial, I am going to use JPEG Melanoma 256x256.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gXlAZOYRs_rf"
      },
      "source": [
        "!cp /content/drive/MyDrive/LibAUC/melanoma_256x256.zip /content/\n",
        "!unzip melanoma_224x224.zip -d /content/melanoma/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SWwE3v_R5Oku"
      },
      "source": [
        "\n",
        "# **Importing LibAUC**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fDfVpPkp4o9O"
      },
      "source": [
        "from libauc.losses import AUCMLoss\n",
        "from libauc.optimizers import PESG\n",
        "from libauc.models import DenseNet121, DenseNet169\n",
        "from libauc.datasets import Melanoma\n",
        "from libauc.utils import auroc\n",
        "\n",
        "import torch \n",
        "from PIL import Image\n",
        "import numpy as np\n",
        "import torchvision.transforms as transforms\n",
        "from torch.utils.data import Dataset"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l2swK5Mo7Kca"
      },
      "source": [
        "# **Reproducibility**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OiiT5oEp7J3C"
      },
      "source": [
        "def set_all_seeds(SEED):\n",
        "    # REPRODUCIBILITY\n",
        "    torch.manual_seed(SEED)\n",
        "    np.random.seed(SEED)\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "    torch.backends.cudnn.benchmark = False"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G-StcokOURv9"
      },
      "source": [
        "# **Data Augmentation**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sQlAJDeoUTtR"
      },
      "source": [
        "import albumentations as A\n",
        "from albumentations.pytorch.transforms import ToTensor\n",
        "\n",
        "def augmentations(image_size=256, is_test=True):\n",
        "    # https://www.kaggle.com/vishnus/a-simple-pytorch-starter-code-single-fold-93\n",
        "    imagenet_stats = {'mean':[0.485, 0.456, 0.406], 'std':[0.229, 0.224, 0.225]}\n",
        "    train_tfms = A.Compose([\n",
        "        A.Cutout(p=0.5),\n",
        "        A.RandomRotate90(p=0.5),\n",
        "        A.Flip(p=0.5),\n",
        "        A.OneOf([\n",
        "            A.RandomBrightnessContrast(brightness_limit=0.2,\n",
        "                                       contrast_limit=0.2,\n",
        "                                       ),\n",
        "            A.HueSaturationValue(\n",
        "                hue_shift_limit=20,\n",
        "                sat_shift_limit=50,\n",
        "                val_shift_limit=50)\n",
        "        ], p=0.5),\n",
        "        A.OneOf([\n",
        "            A.IAAAdditiveGaussianNoise(),\n",
        "            A.GaussNoise(),\n",
        "        ], p=0.5),\n",
        "        A.OneOf([\n",
        "            A.MotionBlur(p=0.2),\n",
        "            A.MedianBlur(blur_limit=3, p=0.1),\n",
        "            A.Blur(blur_limit=3, p=0.1),\n",
        "        ], p=0.5),\n",
        "        A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.5),\n",
        "        A.OneOf([\n",
        "            A.OpticalDistortion(p=0.3),\n",
        "            A.GridDistortion(p=0.1),\n",
        "            A.IAAPiecewiseAffine(p=0.3),\n",
        "        ], p=0.5), \n",
        "        ToTensor(normalize=imagenet_stats)\n",
        "        ])\n",
        "    \n",
        "    test_tfms = A.Compose([ToTensor(normalize=imagenet_stats)])\n",
        "    if is_test:\n",
        "        return test_tfms\n",
        "    else:\n",
        "        return train_tfms"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fP4adNO97YBV"
      },
      "source": [
        "# **Optimizing AUCM Loss**\n",
        "* Installation of `albumentations` is required!"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 519
        },
        "id": "Xx7j0UYe6V0J",
        "outputId": "3bce73dc-ea0a-4639-b3ba-888b276a7d97"
      },
      "source": [
        "# dataset\n",
        "trainSet = Melanoma(root='./melanoma/', is_test=False, test_size=0.2, transforms=augmentations)\n",
        "testSet = Melanoma(root='./melanoma/', is_test=True, test_size=0.2, transforms=augmentations)\n",
        "\n",
        "# paramaters\n",
        "SEED = 123\n",
        "BATCH_SIZE = 64\n",
        "lr = 0.1 \n",
        "gamma = 500\n",
        "imratio = trainSet.imratio\n",
        "weight_decay = 1e-5\n",
        "margin = 1.0\n",
        "\n",
        "# model\n",
        "set_all_seeds(SEED)\n",
        "model = DenseNet121(pretrained=True, last_activation=None, activations='relu', num_classes=1)\n",
        "model = model.cuda()\n",
        "\n",
        "trainloader = torch.utils.data.DataLoader(trainSet, batch_size=BATCH_SIZE, num_workers=2, shuffle=True)\n",
        "testloader =  torch.utils.data.DataLoader(testSet, batch_size=BATCH_SIZE, num_workers=2, shuffle=False)\n",
        "\n",
        "# load your own pretrained model here\n",
        "#  PATH = 'ce_pretrained_model.pth' \n",
        "#  state_dict = torch.load(PATH)\n",
        "#  state_dict.pop('classifier.weight', None)\n",
        "#  state_dict.pop('classifier.bias', None) \n",
        "#  model.load_state_dict(state_dict, strict=False)\n",
        "\n",
        "# define loss & optimizer\n",
        "Loss = AUCMLoss(imratio=imratio)\n",
        "optimizer = PESG(model, \n",
        "                 a=Loss.a, \n",
        "                 b=Loss.b, \n",
        "                 alpha=Loss.alpha, \n",
        "                 lr=lr, \n",
        "                 gamma=gamma, \n",
        "                 margin=margin, \n",
        "                 weight_decay=weight_decay)\n",
        "\n",
        "total_epochs = 16\n",
        "best_val_auc = 0\n",
        "for epoch in range(total_epochs):\n",
        "\n",
        "  # reset stages \n",
        "  if epoch== int(total_epochs*0.5) or epoch== int(total_epochs*0.75):\n",
        "     optimizer.update_regularizer(decay_factor=10) \n",
        "\n",
        "  # training \n",
        "  for idx, data in enumerate(trainloader):\n",
        "      train_data, train_labels = data\n",
        "      train_data, train_labels = train_data.cuda(), train_labels.cuda()\n",
        "      y_pred = model(train_data)\n",
        "      y_pred = torch.sigmoid(y_pred)\n",
        "      loss = Loss(y_pred, train_labels)\n",
        "      optimizer.zero_grad()\n",
        "      loss.backward()\n",
        "      optimizer.step()\n",
        "\n",
        "  # validation\n",
        "  model.eval()\n",
        "  with torch.no_grad():    \n",
        "       test_pred = []\n",
        "       test_true = [] \n",
        "       for jdx, data in enumerate(testloader):\n",
        "           test_data, test_label = data\n",
        "           test_data = test_data.cuda()\n",
        "           y_pred = model(test_data)\n",
        "           y_pred = torch.sigmoid(y_pred)\n",
        "           test_pred.append(y_pred.cpu().detach().numpy())\n",
        "           test_true.append(test_label.numpy())\n",
        "              \n",
        "       test_true = np.concatenate(test_true)\n",
        "       test_pred = np.concatenate(test_pred)\n",
        "       val_auc =  auroc(test_true, test_pred) \n",
        "       model.train()\n",
        "\n",
        "       if best_val_auc < val_auc:\n",
        "          best_val_auc = val_auc\n",
        "              \n",
        "       print ('Epoch=%s, Loss=%.4f, Val_AUC=%.4f, lr=%.4f'%(epoch, loss, val_auc, optimizer.lr))\n",
        "\n",
        "print ('Best Val_AUC is %.4f'%best_val_auc)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "test set split is 0.2\n",
            "Found 26134 image in total, 464 postive images, 25670 negative images.\n",
            "test set split is 0.2\n",
            "Found 6558 image in total, 117 postive images, 6441 negative images.\n",
            "Epoch=0, Loss=0.0238, Val_AUC=0.8411, lr=0.1000\n",
            "Epoch=1, Loss=0.0245, Val_AUC=0.8697, lr=0.1000\n",
            "Epoch=2, Loss=0.0213, Val_AUC=0.8460, lr=0.1000\n",
            "Epoch=3, Loss=-0.0234, Val_AUC=0.8783, lr=0.1000\n"
          ]
        }
      ]
    }
  ]
}