notebooks/resnet56_cifar10.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "# ResNet56 calibration experiment on CIFAR10 Dataset\n",
    "\n",
    "We compare multiple calibration algorithms with their reduced, \n",
    "class-wise and class-wise reduced counterparts.\n",
    "\n",
    "We work with a pre-trained ResNet56 classifier trained on the [CIFAR10 Dataset](https://www.cs.toronto.edu/~kriz/cifar.html),\n",
    "a multi-class classification dataset consisting of 60000 images split evenly across 10 classes\n",
    "\n",
    "The model achieves an accuracy of roughly 93% of the test set.\n",
    "\n",
    "Since the model's accuracy is pretty high it is, as expected, well calibrated\n",
    "(pre-calibration ECE ≈ 0.046, post-calibration ECE <= 0.015).\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import os\n",
    "import sys\n",
    "from pathlib import Path\n",
    "from typing import List\n",
    "\n",
    "# This import is needed to avoid a circular import error\n",
    "import kyle.calibration.calibration_methods\n",
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from kyle.datasets import get_cifar10_dataloader\n",
    "from kyle.evaluation import EvalStats\n",
    "from kyle.models.resnet import resnet56\n",
    "from sklearn.metrics import accuracy_score\n",
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is needed for notebooks in case jupyter is started directly in the notebooks directory\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_working_directory = Path(\".\").resolve()\n",
    "if current_working_directory.name == \"notebooks\":\n",
    "    sys.path.insert(0, os.fspath(current_working_directory.parent))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.constants import DATA_DIR, OUTPUT_DIR, RANDOM_SEED\n",
    "from src.data_and_models.resnet import download_resnet_models\n",
    "from src.utils import (\n",
    "    configure_plots,\n",
    "    perform_default_evaluation,\n",
    "    plot_evaluation_results_from_dataframe,\n",
    "    set_random_seed,\n",
    ")\n",
    "from src.utils.evaluation import combined_results_into_dataframe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Constants\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir = OUTPUT_DIR / \"resnet56_cifar10\"\n",
    "output_dir.mkdir(exist_ok=True)\n",
    "output_file = output_dir / \"results.csv\"\n",
    "\n",
    "resnet_dir = DATA_DIR / \"resnet56_cifar10\"\n",
    "resnet56_model_file = resnet_dir / \"resnet56.th\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_classes = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Configuration\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_random_seed(RANDOM_SEED)\n",
    "configure_plots()\n",
    "\n",
    "# Required to avoid `RuntimeError: Too many open files.`\n",
    "# Refer to this issue for for information: https://github.com/pytorch/pytorch/issues/11201\n",
    "torch.multiprocessing.set_sharing_strategy(\"file_system\")\n",
    "\n",
    "logger = logging.getLogger(__name__)\n",
    "logging.basicConfig(level=logging.INFO, format=\"%(message)s\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_loader = get_cifar10_dataloader(os.fspath(resnet_dir), train=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "download_resnet_models(resnet_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = resnet56()\n",
    "model = torch.nn.DataParallel(model)\n",
    "check_point = torch.load(resnet56_model_file, map_location=torch.device(\"cpu\"))\n",
    "model.load_state_dict(check_point[\"state_dict\"])\n",
    "model = model.to(device)\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating Calibration\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger.info(\"Generating model predictions on test set\")\n",
    "\n",
    "logits = []\n",
    "true_labels = []\n",
    "\n",
    "with torch.no_grad():\n",
    "    for features, labels in tqdm(data_loader, total=len(data_loader)):\n",
    "        features = features.to(device)\n",
    "        true_labels.append(labels)\n",
    "        output = model(features)\n",
    "        output = output.to(\"cpu\")\n",
    "        logits.append(output)\n",
    "\n",
    "uncalibrated_confidences = F.softmax(torch.cat(logits), dim=1).numpy()\n",
    "y_true = torch.cat(true_labels).numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = np.argmax(uncalibrated_confidences, axis=1)\n",
    "model_accuracy = accuracy_score(y_true, y_pred)\n",
    "logger.info(f\"Model accuracy: {model_accuracy*100}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_stats = EvalStats(y_true, uncalibrated_confidences, bins=25)\n",
    "logger.info(f\"ECE before calibration: {eval_stats.expected_calibration_error()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Recalibration\n",
    "We evaluate reduction wrappers on multiple metrics with different calibration algorithms\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Balanced\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger.info(\"Performing evaluation\")\n",
    "\n",
    "eval_results = perform_default_evaluation(\n",
    "    confidences=uncalibrated_confidences,\n",
    "    gt_labels=y_true,\n",
    "    cv=4,\n",
    "    bins=20,\n",
    ")\n",
    "\n",
    "results_df = combined_results_into_dataframe(\n",
    "    eval_results,\n",
    "    model_name=\"ResNet56\",\n",
    "    dataset_name=\"CIFAR10\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduction_methods_order: List[str] = results_df[\"Reduction Method\"].unique().tolist()\n",
    "reduction_methods_order = [reduction_methods_order[0]] + sorted(\n",
    "    reduction_methods_order[1:], key=len\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save Results\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger.info(\"Saving results\")\n",
    "results_df.to_csv(output_file, sep=\";\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plots\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger.info(\"Plotting results\")\n",
    "\n",
    "results_df = results_df.query(\"(Metric != 'condition') & (Metric != 'weak_condition')\")\n",
    "\n",
    "plot_evaluation_results_from_dataframe(\n",
    "    results_df,\n",
    "    hue_order=reduction_methods_order,\n",
    "    output_file=(output_dir / \"evaluation_ECE_resnet56_cifar10.eps\"),\n",
    "    show=False,\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}