lib_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Demo for MANIQA metric setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run theese commands in benchmark's root directory to setup MANIQA metric from it's source directory.\n",
    "# You can also use your custom metric if it matches framework's call interface. An example implementation can be found in the model.py file in subjects/maniqa/\n",
    "\n",
    "# !git submodule update --init --recursive\n",
    "# !cd subjects/maniqa/src && git apply ../patches/maniqa.patch -v\n",
    "# !cd subjects/maniqa && cp model.py src"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load MANIQA weights\n",
    "# !wget https://github.com/IIGROUP/MANIQA/releases/download/Koniq10k/ckpt_koniq10k.pt -P subjects/maniqa/\n",
    "# Install MANIQA requirements\n",
    "# !pip install -r subjects/maniqa/src/requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Change these parameters if you are using other metric. \n",
    "# 'bounds' in metric_cfg specify minimum and maximum possible metric values if such limits exist, approximate range of metric's values otherwise.\n",
    "metric_path = './subjects/maniqa/src/'\n",
    "path_to_weights = './subjects/maniqa/ckpt_koniq10k.pt'\n",
    "metric_cfg = {'is_fr': False, 'bounds':{'low':0, 'high':1}, 'name':'maniqa'}\n",
    "device = 'cuda:0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(metric_path)\n",
    "from model import MetricModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MetricModel(model_path=path_to_weights, device=device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run benchmark attacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import robustness_benchmark as rb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['ifgsm', 'mifgsm', 'amifgsm', 'std-fgsm', 'korhonen-et-al', 'madc', 'cumulative-uap', 'generative-uap', 'uap'])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Specify path to dataset and it's name. It must be a directory containing only images (or only videos)\n",
    "dataset_path = './test_dataset/'\n",
    "dataset_name = 'TEST'\n",
    "\n",
    "# Specify directory where results of attacks will be stored. \n",
    "# Results will be organized as directories with names similar to the attacks.\n",
    "# Files with names *metric_name*.csv will be saved in these directories.\n",
    "result_save_dir = './test_results/'\n",
    "\n",
    "# attacks_to_run = rb.interface.iterative_attacks\n",
    "# attacks_to_run = rb.interface.uap_attacks\n",
    "attacks_to_run = rb.interface.all_default_attacks\n",
    "rb.interface.all_default_attacks.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# You can also add your custom attack \n",
    "# Make sure that it follows benchmark call interface.\n",
    "# You can find default attacks implementation in robustness_benchmark.methods.<attack name>\n",
    "# attacks_to_run['attack name'] = <attack callable>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Results saved in this notebook's cell outputs are not representative, attacks were launched on small debug set of only 5 images.\n",
    "rb.interface.run_attacks(\n",
    "                        attacks_dict=attacks_to_run,\n",
    "                        device=device, \n",
    "                        metric_model=model, \n",
    "                        metric_cfg=metric_cfg, \n",
    "                        dataset_names=[dataset_name], \n",
    "                        dataset_paths=[dataset_path], \n",
    "                        save_dir=result_save_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Collect results from csv files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>maniqa_clear</th>\n",
       "      <th>maniqa_attacked</th>\n",
       "      <th>maniqa_ssim</th>\n",
       "      <th>maniqa_psnr</th>\n",
       "      <th>maniqa_mse</th>\n",
       "      <th>attack</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.497564</td>\n",
       "      <td>1.108695</td>\n",
       "      <td>0.809924</td>\n",
       "      <td>30.214003</td>\n",
       "      <td>0.000952</td>\n",
       "      <td>amifgsm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.568488</td>\n",
       "      <td>1.090278</td>\n",
       "      <td>0.906829</td>\n",
       "      <td>29.778993</td>\n",
       "      <td>0.001052</td>\n",
       "      <td>amifgsm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.335649</td>\n",
       "      <td>1.060856</td>\n",
       "      <td>0.812977</td>\n",
       "      <td>30.184957</td>\n",
       "      <td>0.000958</td>\n",
       "      <td>amifgsm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.510699</td>\n",
       "      <td>1.091831</td>\n",
       "      <td>0.793638</td>\n",
       "      <td>30.128886</td>\n",
       "      <td>0.000971</td>\n",
       "      <td>amifgsm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.485868</td>\n",
       "      <td>0.956575</td>\n",
       "      <td>0.905383</td>\n",
       "      <td>29.416244</td>\n",
       "      <td>0.001144</td>\n",
       "      <td>amifgsm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.497564</td>\n",
       "      <td>0.313744</td>\n",
       "      <td>0.567957</td>\n",
       "      <td>24.812781</td>\n",
       "      <td>0.003302</td>\n",
       "      <td>default-uap_VOC2012_amp0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.568488</td>\n",
       "      <td>0.353059</td>\n",
       "      <td>0.783253</td>\n",
       "      <td>24.812781</td>\n",
       "      <td>0.003302</td>\n",
       "      <td>default-uap_VOC2012_amp0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.335649</td>\n",
       "      <td>0.202528</td>\n",
       "      <td>0.578624</td>\n",
       "      <td>24.812781</td>\n",
       "      <td>0.003302</td>\n",
       "      <td>default-uap_VOC2012_amp0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.510699</td>\n",
       "      <td>0.289065</td>\n",
       "      <td>0.572215</td>\n",
       "      <td>24.812781</td>\n",
       "      <td>0.003302</td>\n",
       "      <td>default-uap_VOC2012_amp0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.485868</td>\n",
       "      <td>0.338082</td>\n",
       "      <td>0.799441</td>\n",
       "      <td>24.812781</td>\n",
       "      <td>0.003302</td>\n",
       "      <td>default-uap_VOC2012_amp0.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>285 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   dataset  maniqa_clear  maniqa_attacked  maniqa_ssim  maniqa_psnr  \\\n",
       "0     TEST      0.497564         1.108695     0.809924    30.214003   \n",
       "1     TEST      0.568488         1.090278     0.906829    29.778993   \n",
       "2     TEST      0.335649         1.060856     0.812977    30.184957   \n",
       "3     TEST      0.510699         1.091831     0.793638    30.128886   \n",
       "4     TEST      0.485868         0.956575     0.905383    29.416244   \n",
       "..     ...           ...              ...          ...          ...   \n",
       "47    TEST      0.497564         0.313744     0.567957    24.812781   \n",
       "50    TEST      0.568488         0.353059     0.783253    24.812781   \n",
       "53    TEST      0.335649         0.202528     0.578624    24.812781   \n",
       "56    TEST      0.510699         0.289065     0.572215    24.812781   \n",
       "59    TEST      0.485868         0.338082     0.799441    24.812781   \n",
       "\n",
       "    maniqa_mse                      attack  \n",
       "0     0.000952                     amifgsm  \n",
       "1     0.001052                     amifgsm  \n",
       "2     0.000958                     amifgsm  \n",
       "3     0.000971                     amifgsm  \n",
       "4     0.001144                     amifgsm  \n",
       "..         ...                         ...  \n",
       "47    0.003302  default-uap_VOC2012_amp0.8  \n",
       "50    0.003302  default-uap_VOC2012_amp0.8  \n",
       "53    0.003302  default-uap_VOC2012_amp0.8  \n",
       "56    0.003302  default-uap_VOC2012_amp0.8  \n",
       "59    0.003302  default-uap_VOC2012_amp0.8  \n",
       "\n",
       "[285 rows x 7 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_df = rb.interface.collect_results(result_save_dir, metric_cfg=metric_cfg)\n",
    "results_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Domain transformation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: domain transformation (based on Neural Optimal Transport, described ) currently only works with metrics used in benchmark, for which domain transformation is pretrained and stored in ```robustness_benchmark/models/models_to_mdtvsfa.pth```.\\\n",
    "For other metrics, you can still use ```evaluate_robustness()``` on results_df."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00, 12.58it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current metric:  maniqa\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>maniqa_clear</th>\n",
       "      <th>maniqa_attacked</th>\n",
       "      <th>maniqa_ssim</th>\n",
       "      <th>maniqa_psnr</th>\n",
       "      <th>maniqa_mse</th>\n",
       "      <th>attack</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.593020</td>\n",
       "      <td>1.385626</td>\n",
       "      <td>0.809924</td>\n",
       "      <td>30.214003</td>\n",
       "      <td>0.000952</td>\n",
       "      <td>amifgsm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.691731</td>\n",
       "      <td>1.360014</td>\n",
       "      <td>0.906829</td>\n",
       "      <td>29.778993</td>\n",
       "      <td>0.001052</td>\n",
       "      <td>amifgsm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.316637</td>\n",
       "      <td>1.326889</td>\n",
       "      <td>0.812977</td>\n",
       "      <td>30.184957</td>\n",
       "      <td>0.000958</td>\n",
       "      <td>amifgsm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.605928</td>\n",
       "      <td>1.363978</td>\n",
       "      <td>0.793638</td>\n",
       "      <td>30.128886</td>\n",
       "      <td>0.000971</td>\n",
       "      <td>amifgsm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.571077</td>\n",
       "      <td>1.199576</td>\n",
       "      <td>0.905383</td>\n",
       "      <td>29.416244</td>\n",
       "      <td>0.001144</td>\n",
       "      <td>amifgsm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.593067</td>\n",
       "      <td>0.286116</td>\n",
       "      <td>0.567957</td>\n",
       "      <td>24.812781</td>\n",
       "      <td>0.003302</td>\n",
       "      <td>default-uap_VOC2012_amp0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.699335</td>\n",
       "      <td>0.339540</td>\n",
       "      <td>0.783253</td>\n",
       "      <td>24.812781</td>\n",
       "      <td>0.003302</td>\n",
       "      <td>default-uap_VOC2012_amp0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.321193</td>\n",
       "      <td>0.115863</td>\n",
       "      <td>0.578624</td>\n",
       "      <td>24.812781</td>\n",
       "      <td>0.003302</td>\n",
       "      <td>default-uap_VOC2012_amp0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.616140</td>\n",
       "      <td>0.231154</td>\n",
       "      <td>0.572215</td>\n",
       "      <td>24.812781</td>\n",
       "      <td>0.003302</td>\n",
       "      <td>default-uap_VOC2012_amp0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>TEST</td>\n",
       "      <td>0.572159</td>\n",
       "      <td>0.319131</td>\n",
       "      <td>0.799441</td>\n",
       "      <td>24.812781</td>\n",
       "      <td>0.003302</td>\n",
       "      <td>default-uap_VOC2012_amp0.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>285 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   dataset  maniqa_clear  maniqa_attacked  maniqa_ssim  maniqa_psnr  \\\n",
       "0     TEST      0.593020         1.385626     0.809924    30.214003   \n",
       "1     TEST      0.691731         1.360014     0.906829    29.778993   \n",
       "2     TEST      0.316637         1.326889     0.812977    30.184957   \n",
       "3     TEST      0.605928         1.363978     0.793638    30.128886   \n",
       "4     TEST      0.571077         1.199576     0.905383    29.416244   \n",
       "..     ...           ...              ...          ...          ...   \n",
       "47    TEST      0.593067         0.286116     0.567957    24.812781   \n",
       "50    TEST      0.699335         0.339540     0.783253    24.812781   \n",
       "53    TEST      0.321193         0.115863     0.578624    24.812781   \n",
       "56    TEST      0.616140         0.231154     0.572215    24.812781   \n",
       "59    TEST      0.572159         0.319131     0.799441    24.812781   \n",
       "\n",
       "    maniqa_mse                      attack  \n",
       "0     0.000952                     amifgsm  \n",
       "1     0.001052                     amifgsm  \n",
       "2     0.000958                     amifgsm  \n",
       "3     0.000971                     amifgsm  \n",
       "4     0.001144                     amifgsm  \n",
       "..         ...                         ...  \n",
       "47    0.003302  default-uap_VOC2012_amp0.8  \n",
       "50    0.003302  default-uap_VOC2012_amp0.8  \n",
       "53    0.003302  default-uap_VOC2012_amp0.8  \n",
       "56    0.003302  default-uap_VOC2012_amp0.8  \n",
       "59    0.003302  default-uap_VOC2012_amp0.8  \n",
       "\n",
       "[285 rows x 7 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transformed_results_df = rb.interface.domain_transform(results_df, metrics=[metric_cfg['name']], batch_size=1000, device=device)\n",
    "transformed_results_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Metric robustness evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metric</th>\n",
       "      <th>attack</th>\n",
       "      <th>energy_distance_score</th>\n",
       "      <th>normalized_absolute_gain</th>\n",
       "      <th>normalized_relative_gain</th>\n",
       "      <th>relative_gain_classic</th>\n",
       "      <th>robustness_score</th>\n",
       "      <th>wasserstein_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>amifgsm</td>\n",
       "      <td>1.879</td>\n",
       "      <td>2.016 (1.848, 2.184)</td>\n",
       "      <td>1.364 (1.055, 1.674)</td>\n",
       "      <td>1.563 (1.171, 1.956)</td>\n",
       "      <td>-0.387 (-0.422, -0.351)</td>\n",
       "      <td>2.016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>cumulative-uap_COCO_amp0.2</td>\n",
       "      <td>-0.063</td>\n",
       "      <td>-0.011 (-0.021, -0.001)</td>\n",
       "      <td>-0.007 (-0.012, -0.001)</td>\n",
       "      <td>-0.007 (-0.013, -0.001)</td>\n",
       "      <td>2.101 (1.594, 2.609)</td>\n",
       "      <td>-0.012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>cumulative-uap_VOC2012_amp0.2</td>\n",
       "      <td>-0.325</td>\n",
       "      <td>-0.163 (-0.229, -0.096)</td>\n",
       "      <td>-0.099 (-0.134, -0.064)</td>\n",
       "      <td>-0.112 (-0.150, -0.074)</td>\n",
       "      <td>0.766 (0.567, 0.966)</td>\n",
       "      <td>-0.163</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>cumulative-uap_COCO_amp0.4</td>\n",
       "      <td>-0.187</td>\n",
       "      <td>-0.057 (-0.093, -0.022)</td>\n",
       "      <td>-0.034 (-0.053, -0.014)</td>\n",
       "      <td>-0.038 (-0.059, -0.016)</td>\n",
       "      <td>1.289 (1.010, 1.567)</td>\n",
       "      <td>-0.057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>cumulative-uap_VOC2012_amp0.4</td>\n",
       "      <td>-0.654</td>\n",
       "      <td>-0.440 (-0.532, -0.349)</td>\n",
       "      <td>-0.276 (-0.328, -0.224)</td>\n",
       "      <td>-0.315 (-0.379, -0.252)</td>\n",
       "      <td>0.284 (0.183, 0.385)</td>\n",
       "      <td>-0.440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>cumulative-uap_COCO_amp0.8</td>\n",
       "      <td>-0.253</td>\n",
       "      <td>-0.051 (-0.138, 0.037)</td>\n",
       "      <td>-0.016 (-0.082, 0.050)</td>\n",
       "      <td>-0.015 (-0.093, 0.063)</td>\n",
       "      <td>0.960 (0.760, 1.160)</td>\n",
       "      <td>-0.108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>cumulative-uap_VOC2012_amp0.8</td>\n",
       "      <td>-1.083</td>\n",
       "      <td>-0.856 (-1.003, -0.709)</td>\n",
       "      <td>-0.525 (-0.564, -0.486)</td>\n",
       "      <td>-0.589 (-0.633, -0.545)</td>\n",
       "      <td>-0.007 (-0.126, 0.112)</td>\n",
       "      <td>-0.856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>generative-uap_COCO_amp0.2</td>\n",
       "      <td>-0.112</td>\n",
       "      <td>-0.034 (-0.050, -0.018)</td>\n",
       "      <td>-0.022 (-0.034, -0.011)</td>\n",
       "      <td>-0.025 (-0.038, -0.012)</td>\n",
       "      <td>1.518 (1.231, 1.806)</td>\n",
       "      <td>-0.034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>generative-uap_VOC2012_amp0.2</td>\n",
       "      <td>-0.107</td>\n",
       "      <td>-0.034 (-0.060, -0.008)</td>\n",
       "      <td>-0.023 (-0.039, -0.007)</td>\n",
       "      <td>-0.026 (-0.045, -0.008)</td>\n",
       "      <td>1.413 (1.130, 1.696)</td>\n",
       "      <td>-0.034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>generative-uap_COCO_amp0.4</td>\n",
       "      <td>-0.241</td>\n",
       "      <td>-0.106 (-0.156, -0.057)</td>\n",
       "      <td>-0.066 (-0.093, -0.039)</td>\n",
       "      <td>-0.074 (-0.105, -0.043)</td>\n",
       "      <td>1.043 (0.706, 1.379)</td>\n",
       "      <td>-0.106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>generative-uap_VOC2012_amp0.4</td>\n",
       "      <td>-0.300</td>\n",
       "      <td>-0.142 (-0.210, -0.074)</td>\n",
       "      <td>-0.087 (-0.125, -0.050)</td>\n",
       "      <td>-0.099 (-0.140, -0.057)</td>\n",
       "      <td>0.872 (0.605, 1.139)</td>\n",
       "      <td>-0.142</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>generative-uap_COCO_amp0.8</td>\n",
       "      <td>-0.437</td>\n",
       "      <td>-0.272 (-0.405, -0.138)</td>\n",
       "      <td>-0.166 (-0.238, -0.095)</td>\n",
       "      <td>-0.185 (-0.265, -0.106)</td>\n",
       "      <td>0.735 (0.235, 1.236)</td>\n",
       "      <td>-0.272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>generative-uap_VOC2012_amp0.8</td>\n",
       "      <td>-0.581</td>\n",
       "      <td>-0.340 (-0.458, -0.223)</td>\n",
       "      <td>-0.198 (-0.255, -0.140)</td>\n",
       "      <td>-0.220 (-0.281, -0.159)</td>\n",
       "      <td>0.455 (0.210, 0.700)</td>\n",
       "      <td>-0.340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>ifgsm</td>\n",
       "      <td>2.151</td>\n",
       "      <td>2.965 (2.470, 3.459)</td>\n",
       "      <td>2.011 (1.484, 2.538)</td>\n",
       "      <td>2.308 (1.656, 2.960)</td>\n",
       "      <td>-0.535 (-0.604, -0.465)</td>\n",
       "      <td>2.965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>korhonen-et-al</td>\n",
       "      <td>1.887</td>\n",
       "      <td>2.040 (1.849, 2.231)</td>\n",
       "      <td>1.374 (1.016, 1.733)</td>\n",
       "      <td>1.599 (1.131, 2.068)</td>\n",
       "      <td>-0.392 (-0.436, -0.349)</td>\n",
       "      <td>2.040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>madc</td>\n",
       "      <td>0.988</td>\n",
       "      <td>0.776 (0.671, 0.882)</td>\n",
       "      <td>0.529 (0.369, 0.689)</td>\n",
       "      <td>0.611 (0.408, 0.813)</td>\n",
       "      <td>0.035 (-0.017, 0.086)</td>\n",
       "      <td>0.776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>mifgsm</td>\n",
       "      <td>1.875</td>\n",
       "      <td>2.006 (1.838, 2.174)</td>\n",
       "      <td>1.360 (1.050, 1.670)</td>\n",
       "      <td>1.565 (1.169, 1.960)</td>\n",
       "      <td>-0.386 (-0.423, -0.350)</td>\n",
       "      <td>2.006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>std-fgsm</td>\n",
       "      <td>1.142</td>\n",
       "      <td>0.935 (0.793, 1.078)</td>\n",
       "      <td>0.640 (0.445, 0.836)</td>\n",
       "      <td>0.739 (0.492, 0.985)</td>\n",
       "      <td>-0.042 (-0.120, 0.037)</td>\n",
       "      <td>0.935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>default-uap_COCO_amp0.2</td>\n",
       "      <td>-0.267</td>\n",
       "      <td>-0.128 (-0.190, -0.067)</td>\n",
       "      <td>-0.081 (-0.116, -0.045)</td>\n",
       "      <td>-0.091 (-0.131, -0.051)</td>\n",
       "      <td>0.927 (0.650, 1.203)</td>\n",
       "      <td>-0.128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>default-uap_VOC2012_amp0.2</td>\n",
       "      <td>-0.303</td>\n",
       "      <td>-0.151 (-0.215, -0.087)</td>\n",
       "      <td>-0.092 (-0.125, -0.058)</td>\n",
       "      <td>-0.104 (-0.141, -0.067)</td>\n",
       "      <td>0.797 (0.611, 0.984)</td>\n",
       "      <td>-0.151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>default-uap_COCO_amp0.4</td>\n",
       "      <td>-0.585</td>\n",
       "      <td>-0.389 (-0.491, -0.286)</td>\n",
       "      <td>-0.247 (-0.316, -0.179)</td>\n",
       "      <td>-0.284 (-0.368, -0.200)</td>\n",
       "      <td>0.348 (0.249, 0.448)</td>\n",
       "      <td>-0.389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>default-uap_VOC2012_amp0.4</td>\n",
       "      <td>-0.627</td>\n",
       "      <td>-0.421 (-0.523, -0.319)</td>\n",
       "      <td>-0.265 (-0.328, -0.203)</td>\n",
       "      <td>-0.302 (-0.377, -0.226)</td>\n",
       "      <td>0.310 (0.211, 0.409)</td>\n",
       "      <td>-0.421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>default-uap_COCO_amp0.8</td>\n",
       "      <td>-0.985</td>\n",
       "      <td>-0.761 (-0.888, -0.635)</td>\n",
       "      <td>-0.470 (-0.517, -0.422)</td>\n",
       "      <td>-0.531 (-0.592, -0.470)</td>\n",
       "      <td>0.045 (-0.049, 0.140)</td>\n",
       "      <td>-0.761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>default-uap_VOC2012_amp0.8</td>\n",
       "      <td>-1.005</td>\n",
       "      <td>-0.781 (-0.913, -0.649)</td>\n",
       "      <td>-0.482 (-0.527, -0.437)</td>\n",
       "      <td>-0.545 (-0.602, -0.487)</td>\n",
       "      <td>0.032 (-0.066, 0.130)</td>\n",
       "      <td>-0.781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>all</td>\n",
       "      <td>0.574</td>\n",
       "      <td>0.502 (0.364, 0.640)</td>\n",
       "      <td>0.351 (0.250, 0.451)</td>\n",
       "      <td>0.412 (0.292, 0.532)</td>\n",
       "      <td>0.379 (0.291, 0.466)</td>\n",
       "      <td>0.691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>iterative</td>\n",
       "      <td>1.581</td>\n",
       "      <td>1.846 (1.673, 2.019)</td>\n",
       "      <td>1.251 (1.089, 1.412)</td>\n",
       "      <td>1.457 (1.256, 1.658)</td>\n",
       "      <td>-0.303 (-0.347, -0.260)</td>\n",
       "      <td>1.846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>maniqa</td>\n",
       "      <td>uap</td>\n",
       "      <td>-0.390</td>\n",
       "      <td>-0.281 (-0.323, -0.239)</td>\n",
       "      <td>-0.173 (-0.198, -0.147)</td>\n",
       "      <td>-0.198 (-0.227, -0.169)</td>\n",
       "      <td>0.776 (0.679, 0.873)</td>\n",
       "      <td>-0.281</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    metric                         attack energy_distance_score  \\\n",
       "0   maniqa                        amifgsm                 1.879   \n",
       "1   maniqa     cumulative-uap_COCO_amp0.2                -0.063   \n",
       "2   maniqa  cumulative-uap_VOC2012_amp0.2                -0.325   \n",
       "3   maniqa     cumulative-uap_COCO_amp0.4                -0.187   \n",
       "4   maniqa  cumulative-uap_VOC2012_amp0.4                -0.654   \n",
       "5   maniqa     cumulative-uap_COCO_amp0.8                -0.253   \n",
       "6   maniqa  cumulative-uap_VOC2012_amp0.8                -1.083   \n",
       "7   maniqa     generative-uap_COCO_amp0.2                -0.112   \n",
       "8   maniqa  generative-uap_VOC2012_amp0.2                -0.107   \n",
       "9   maniqa     generative-uap_COCO_amp0.4                -0.241   \n",
       "10  maniqa  generative-uap_VOC2012_amp0.4                -0.300   \n",
       "11  maniqa     generative-uap_COCO_amp0.8                -0.437   \n",
       "12  maniqa  generative-uap_VOC2012_amp0.8                -0.581   \n",
       "13  maniqa                          ifgsm                 2.151   \n",
       "14  maniqa                 korhonen-et-al                 1.887   \n",
       "15  maniqa                           madc                 0.988   \n",
       "16  maniqa                         mifgsm                 1.875   \n",
       "17  maniqa                       std-fgsm                 1.142   \n",
       "18  maniqa        default-uap_COCO_amp0.2                -0.267   \n",
       "19  maniqa     default-uap_VOC2012_amp0.2                -0.303   \n",
       "20  maniqa        default-uap_COCO_amp0.4                -0.585   \n",
       "21  maniqa     default-uap_VOC2012_amp0.4                -0.627   \n",
       "22  maniqa        default-uap_COCO_amp0.8                -0.985   \n",
       "23  maniqa     default-uap_VOC2012_amp0.8                -1.005   \n",
       "24  maniqa                            all                 0.574   \n",
       "25  maniqa                      iterative                 1.581   \n",
       "26  maniqa                            uap                -0.390   \n",
       "\n",
       "   normalized_absolute_gain normalized_relative_gain    relative_gain_classic  \\\n",
       "0      2.016 (1.848, 2.184)     1.364 (1.055, 1.674)     1.563 (1.171, 1.956)   \n",
       "1   -0.011 (-0.021, -0.001)  -0.007 (-0.012, -0.001)  -0.007 (-0.013, -0.001)   \n",
       "2   -0.163 (-0.229, -0.096)  -0.099 (-0.134, -0.064)  -0.112 (-0.150, -0.074)   \n",
       "3   -0.057 (-0.093, -0.022)  -0.034 (-0.053, -0.014)  -0.038 (-0.059, -0.016)   \n",
       "4   -0.440 (-0.532, -0.349)  -0.276 (-0.328, -0.224)  -0.315 (-0.379, -0.252)   \n",
       "5    -0.051 (-0.138, 0.037)   -0.016 (-0.082, 0.050)   -0.015 (-0.093, 0.063)   \n",
       "6   -0.856 (-1.003, -0.709)  -0.525 (-0.564, -0.486)  -0.589 (-0.633, -0.545)   \n",
       "7   -0.034 (-0.050, -0.018)  -0.022 (-0.034, -0.011)  -0.025 (-0.038, -0.012)   \n",
       "8   -0.034 (-0.060, -0.008)  -0.023 (-0.039, -0.007)  -0.026 (-0.045, -0.008)   \n",
       "9   -0.106 (-0.156, -0.057)  -0.066 (-0.093, -0.039)  -0.074 (-0.105, -0.043)   \n",
       "10  -0.142 (-0.210, -0.074)  -0.087 (-0.125, -0.050)  -0.099 (-0.140, -0.057)   \n",
       "11  -0.272 (-0.405, -0.138)  -0.166 (-0.238, -0.095)  -0.185 (-0.265, -0.106)   \n",
       "12  -0.340 (-0.458, -0.223)  -0.198 (-0.255, -0.140)  -0.220 (-0.281, -0.159)   \n",
       "13     2.965 (2.470, 3.459)     2.011 (1.484, 2.538)     2.308 (1.656, 2.960)   \n",
       "14     2.040 (1.849, 2.231)     1.374 (1.016, 1.733)     1.599 (1.131, 2.068)   \n",
       "15     0.776 (0.671, 0.882)     0.529 (0.369, 0.689)     0.611 (0.408, 0.813)   \n",
       "16     2.006 (1.838, 2.174)     1.360 (1.050, 1.670)     1.565 (1.169, 1.960)   \n",
       "17     0.935 (0.793, 1.078)     0.640 (0.445, 0.836)     0.739 (0.492, 0.985)   \n",
       "18  -0.128 (-0.190, -0.067)  -0.081 (-0.116, -0.045)  -0.091 (-0.131, -0.051)   \n",
       "19  -0.151 (-0.215, -0.087)  -0.092 (-0.125, -0.058)  -0.104 (-0.141, -0.067)   \n",
       "20  -0.389 (-0.491, -0.286)  -0.247 (-0.316, -0.179)  -0.284 (-0.368, -0.200)   \n",
       "21  -0.421 (-0.523, -0.319)  -0.265 (-0.328, -0.203)  -0.302 (-0.377, -0.226)   \n",
       "22  -0.761 (-0.888, -0.635)  -0.470 (-0.517, -0.422)  -0.531 (-0.592, -0.470)   \n",
       "23  -0.781 (-0.913, -0.649)  -0.482 (-0.527, -0.437)  -0.545 (-0.602, -0.487)   \n",
       "24     0.502 (0.364, 0.640)     0.351 (0.250, 0.451)     0.412 (0.292, 0.532)   \n",
       "25     1.846 (1.673, 2.019)     1.251 (1.089, 1.412)     1.457 (1.256, 1.658)   \n",
       "26  -0.281 (-0.323, -0.239)  -0.173 (-0.198, -0.147)  -0.198 (-0.227, -0.169)   \n",
       "\n",
       "           robustness_score wasserstein_score  \n",
       "0   -0.387 (-0.422, -0.351)             2.016  \n",
       "1      2.101 (1.594, 2.609)            -0.012  \n",
       "2      0.766 (0.567, 0.966)            -0.163  \n",
       "3      1.289 (1.010, 1.567)            -0.057  \n",
       "4      0.284 (0.183, 0.385)            -0.440  \n",
       "5      0.960 (0.760, 1.160)            -0.108  \n",
       "6    -0.007 (-0.126, 0.112)            -0.856  \n",
       "7      1.518 (1.231, 1.806)            -0.034  \n",
       "8      1.413 (1.130, 1.696)            -0.034  \n",
       "9      1.043 (0.706, 1.379)            -0.106  \n",
       "10     0.872 (0.605, 1.139)            -0.142  \n",
       "11     0.735 (0.235, 1.236)            -0.272  \n",
       "12     0.455 (0.210, 0.700)            -0.340  \n",
       "13  -0.535 (-0.604, -0.465)             2.965  \n",
       "14  -0.392 (-0.436, -0.349)             2.040  \n",
       "15    0.035 (-0.017, 0.086)             0.776  \n",
       "16  -0.386 (-0.423, -0.350)             2.006  \n",
       "17   -0.042 (-0.120, 0.037)             0.935  \n",
       "18     0.927 (0.650, 1.203)            -0.128  \n",
       "19     0.797 (0.611, 0.984)            -0.151  \n",
       "20     0.348 (0.249, 0.448)            -0.389  \n",
       "21     0.310 (0.211, 0.409)            -0.421  \n",
       "22    0.045 (-0.049, 0.140)            -0.761  \n",
       "23    0.032 (-0.066, 0.130)            -0.781  \n",
       "24     0.379 (0.291, 0.466)             0.691  \n",
       "25  -0.303 (-0.347, -0.260)             1.846  \n",
       "26     0.776 (0.679, 0.873)            -0.281  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# W/o domain transformation\n",
    "# rb.interface.evaluate_robustness(results_df, attacks=list(transformed_results_df.attack.unique()) + ['all', 'iterative', 'uap'])\n",
    "\n",
    "rb.interface.evaluate_robustness(transformed_results_df, attacks=list(transformed_results_df.attack.unique()) + ['all', 'iterative', 'uap'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}