demo_scoring.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "#import seaborn as sns\n",
    "#from matplotlib import pyplot as plt\n",
    "#from tqdm import tqdm\n",
    "import scipy.stats as st\n",
    "import os\n",
    "from pathlib import Path \n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from robustness_benchmark.NOT import transform_full_df, load_models\n",
    "from robustness_benchmark.score_methods import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get attack name from filename\n",
    "def get_atk(filename, metric):\n",
    "    pattern = f\"(.+)_{metric}\"\n",
    "    match = re.search(pattern, filename)\n",
    "    if match:\n",
    "        return match.group(1)\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "\n",
    "# parse results for one metric\n",
    "def parse_results(path='./res', metric='maniqa'):\n",
    "    required_cols = ['clear', 'attacked', 'ssim', 'psnr', 'mse']\n",
    "    uap_amps = [0.2, 0.4, 0.8]\n",
    "    uap_train_dsets = ['VOC2012', 'COCO']\n",
    "    result_df = pd.DataFrame()\n",
    "    for file in Path(path).iterdir():\n",
    "        if not str(file).endswith('test.csv'):\n",
    "            continue\n",
    "        atk_name = get_atk(str(file.stem), metric)\n",
    "        if atk_name is None:\n",
    "            continue\n",
    "        \n",
    "        #print(atk_name)\n",
    "        cur_df = pd.read_csv(file).rename(columns={'test_dataset': 'dataset'})\n",
    "\n",
    "        if 'uap' not in str(file.stem):\n",
    "            data_to_add = cur_df[['dataset'] + required_cols].copy()\n",
    "            data_to_add = data_to_add.rename(columns={x: f'{metric}_{x}' for x in required_cols})\n",
    "            data_to_add['attack'] = atk_name\n",
    "            data_to_add.dataset.replace(to_replace={'NIPS2017': 'NIPS'}, inplace=True)\n",
    "            result_df = pd.concat([result_df, data_to_add], axis=0)\n",
    "        else:\n",
    "            for amp in uap_amps:\n",
    "                for train_set in uap_train_dsets:\n",
    "                    cur_atk_name = atk_name\n",
    "                    if atk_name == 'uap':\n",
    "                        cur_atk_name = 'default-uap'\n",
    "                    cur_atk_name += f'_{train_set}_amp{str(amp)}'\n",
    "                    data_to_add = cur_df[cur_df['amplitude'] == amp][cur_df['train_dataset'] == train_set][['dataset'] + required_cols].copy()\n",
    "                    data_to_add = data_to_add.rename(columns={x: f'{metric}_{x}' for x in required_cols})\n",
    "                    data_to_add['attack'] = cur_atk_name\n",
    "                    data_to_add.dataset.replace(to_replace={'NIPS2017': 'NIPS'}, inplace=True)\n",
    "                    result_df = pd.concat([result_df, data_to_add], axis=0)\n",
    "    return result_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "metric_to_evaluate = 'maniqa'\n",
    "\n",
    "# parse results from demo directory after running attacks\n",
    "#data = parse_results(path='./res', metric=metric_to_evaluate)\n",
    "\n",
    "# results as of the time of writing\n",
    "# available at https://calypso.gml-team.ru:5001/sharing/NFLRz05g9 (password: 'neurips_benchmark_2023')\n",
    "path_to_results = './results.ft'\n",
    "data = pd.read_feather(path_to_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>attack</th>\n",
       "      <th>maniqa_clear</th>\n",
       "      <th>maniqa_attacked</th>\n",
       "      <th>maniqa_ssim</th>\n",
       "      <th>maniqa_psnr</th>\n",
       "      <th>maniqa_mse</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NIPS</td>\n",
       "      <td>amifgsm</td>\n",
       "      <td>0.497564</td>\n",
       "      <td>1.109380</td>\n",
       "      <td>0.810342</td>\n",
       "      <td>30.222343</td>\n",
       "      <td>0.000950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NIPS</td>\n",
       "      <td>amifgsm</td>\n",
       "      <td>0.531079</td>\n",
       "      <td>1.077701</td>\n",
       "      <td>0.875497</td>\n",
       "      <td>29.726215</td>\n",
       "      <td>0.001065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NIPS</td>\n",
       "      <td>amifgsm</td>\n",
       "      <td>0.668703</td>\n",
       "      <td>1.114359</td>\n",
       "      <td>0.855742</td>\n",
       "      <td>29.728792</td>\n",
       "      <td>0.001064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NIPS</td>\n",
       "      <td>amifgsm</td>\n",
       "      <td>0.568488</td>\n",
       "      <td>1.090326</td>\n",
       "      <td>0.906851</td>\n",
       "      <td>29.779349</td>\n",
       "      <td>0.001052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NIPS</td>\n",
       "      <td>amifgsm</td>\n",
       "      <td>0.659713</td>\n",
       "      <td>1.073918</td>\n",
       "      <td>0.747340</td>\n",
       "      <td>30.407143</td>\n",
       "      <td>0.000911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77275</th>\n",
       "      <td>VIMEO</td>\n",
       "      <td>default-uap_COCO_amp0.8</td>\n",
       "      <td>0.520685</td>\n",
       "      <td>0.149498</td>\n",
       "      <td>0.384374</td>\n",
       "      <td>24.441853</td>\n",
       "      <td>0.003596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77276</th>\n",
       "      <td>VIMEO</td>\n",
       "      <td>default-uap_COCO_amp0.8</td>\n",
       "      <td>0.411854</td>\n",
       "      <td>0.204567</td>\n",
       "      <td>0.546673</td>\n",
       "      <td>24.441853</td>\n",
       "      <td>0.003596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77277</th>\n",
       "      <td>VIMEO</td>\n",
       "      <td>default-uap_COCO_amp0.8</td>\n",
       "      <td>0.370396</td>\n",
       "      <td>0.178519</td>\n",
       "      <td>0.506352</td>\n",
       "      <td>24.441853</td>\n",
       "      <td>0.003596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77278</th>\n",
       "      <td>VIMEO</td>\n",
       "      <td>default-uap_COCO_amp0.8</td>\n",
       "      <td>0.420029</td>\n",
       "      <td>0.203137</td>\n",
       "      <td>0.391060</td>\n",
       "      <td>24.441853</td>\n",
       "      <td>0.003596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77279</th>\n",
       "      <td>VIMEO</td>\n",
       "      <td>default-uap_COCO_amp0.8</td>\n",
       "      <td>0.404197</td>\n",
       "      <td>0.168623</td>\n",
       "      <td>0.506051</td>\n",
       "      <td>24.441853</td>\n",
       "      <td>0.003596</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>77280 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      dataset                   attack  maniqa_clear  maniqa_attacked  \\\n",
       "0        NIPS                  amifgsm      0.497564         1.109380   \n",
       "1        NIPS                  amifgsm      0.531079         1.077701   \n",
       "2        NIPS                  amifgsm      0.668703         1.114359   \n",
       "3        NIPS                  amifgsm      0.568488         1.090326   \n",
       "4        NIPS                  amifgsm      0.659713         1.073918   \n",
       "...       ...                      ...           ...              ...   \n",
       "77275   VIMEO  default-uap_COCO_amp0.8      0.520685         0.149498   \n",
       "77276   VIMEO  default-uap_COCO_amp0.8      0.411854         0.204567   \n",
       "77277   VIMEO  default-uap_COCO_amp0.8      0.370396         0.178519   \n",
       "77278   VIMEO  default-uap_COCO_amp0.8      0.420029         0.203137   \n",
       "77279   VIMEO  default-uap_COCO_amp0.8      0.404197         0.168623   \n",
       "\n",
       "       maniqa_ssim  maniqa_psnr  maniqa_mse  \n",
       "0         0.810342    30.222343    0.000950  \n",
       "1         0.875497    29.726215    0.001065  \n",
       "2         0.855742    29.728792    0.001064  \n",
       "3         0.906851    29.779349    0.001052  \n",
       "4         0.747340    30.407143    0.000911  \n",
       "...            ...          ...         ...  \n",
       "77275     0.384374    24.441853    0.003596  \n",
       "77276     0.546673    24.441853    0.003596  \n",
       "77277     0.506352    24.441853    0.003596  \n",
       "77278     0.391060    24.441853    0.003596  \n",
       "77279     0.506051    24.441853    0.003596  \n",
       "\n",
       "[77280 rows x 7 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_to_evaluate = data\\\n",
    "    [[\n",
    "    'dataset',\n",
    "    'attack',\n",
    "    f'{metric_to_evaluate}_clear',\n",
    "    f'{metric_to_evaluate}_attacked',\n",
    "    f'{metric_to_evaluate}_ssim',\n",
    "    f'{metric_to_evaluate}_psnr',\n",
    "    f'{metric_to_evaluate}_mse'\n",
    "    ]].reset_index(drop=True)\n",
    "data_to_evaluate"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Domain transformation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 78/78 [00:13<00:00,  5.99it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>attack</th>\n",
       "      <th>maniqa_clear</th>\n",
       "      <th>maniqa_attacked</th>\n",
       "      <th>maniqa_ssim</th>\n",
       "      <th>maniqa_psnr</th>\n",
       "      <th>maniqa_mse</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NIPS</td>\n",
       "      <td>amifgsm</td>\n",
       "      <td>0.592009</td>\n",
       "      <td>1.385288</td>\n",
       "      <td>0.810342</td>\n",
       "      <td>30.222343</td>\n",
       "      <td>0.000950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NIPS</td>\n",
       "      <td>amifgsm</td>\n",
       "      <td>0.641324</td>\n",
       "      <td>1.346277</td>\n",
       "      <td>0.875497</td>\n",
       "      <td>29.726215</td>\n",
       "      <td>0.001065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NIPS</td>\n",
       "      <td>amifgsm</td>\n",
       "      <td>0.838071</td>\n",
       "      <td>1.389977</td>\n",
       "      <td>0.855742</td>\n",
       "      <td>29.728792</td>\n",
       "      <td>0.001064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NIPS</td>\n",
       "      <td>amifgsm</td>\n",
       "      <td>0.698492</td>\n",
       "      <td>1.362370</td>\n",
       "      <td>0.906851</td>\n",
       "      <td>29.779349</td>\n",
       "      <td>0.001052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NIPS</td>\n",
       "      <td>amifgsm</td>\n",
       "      <td>0.829641</td>\n",
       "      <td>1.340757</td>\n",
       "      <td>0.747340</td>\n",
       "      <td>30.407143</td>\n",
       "      <td>0.000911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77275</th>\n",
       "      <td>VIMEO</td>\n",
       "      <td>default-uap_COCO_amp0.8</td>\n",
       "      <td>0.624594</td>\n",
       "      <td>0.099642</td>\n",
       "      <td>0.384374</td>\n",
       "      <td>24.441853</td>\n",
       "      <td>0.003596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77276</th>\n",
       "      <td>VIMEO</td>\n",
       "      <td>default-uap_COCO_amp0.8</td>\n",
       "      <td>0.440777</td>\n",
       "      <td>0.116113</td>\n",
       "      <td>0.546673</td>\n",
       "      <td>24.441853</td>\n",
       "      <td>0.003596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77277</th>\n",
       "      <td>VIMEO</td>\n",
       "      <td>default-uap_COCO_amp0.8</td>\n",
       "      <td>0.371981</td>\n",
       "      <td>0.110458</td>\n",
       "      <td>0.506352</td>\n",
       "      <td>24.441853</td>\n",
       "      <td>0.003596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77278</th>\n",
       "      <td>VIMEO</td>\n",
       "      <td>default-uap_COCO_amp0.8</td>\n",
       "      <td>0.456308</td>\n",
       "      <td>0.117133</td>\n",
       "      <td>0.391060</td>\n",
       "      <td>24.441853</td>\n",
       "      <td>0.003596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77279</th>\n",
       "      <td>VIMEO</td>\n",
       "      <td>default-uap_COCO_amp0.8</td>\n",
       "      <td>0.434788</td>\n",
       "      <td>0.109997</td>\n",
       "      <td>0.506051</td>\n",
       "      <td>24.441853</td>\n",
       "      <td>0.003596</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>77280 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      dataset                   attack  maniqa_clear  maniqa_attacked  \\\n",
       "0        NIPS                  amifgsm      0.592009         1.385288   \n",
       "1        NIPS                  amifgsm      0.641324         1.346277   \n",
       "2        NIPS                  amifgsm      0.838071         1.389977   \n",
       "3        NIPS                  amifgsm      0.698492         1.362370   \n",
       "4        NIPS                  amifgsm      0.829641         1.340757   \n",
       "...       ...                      ...           ...              ...   \n",
       "77275   VIMEO  default-uap_COCO_amp0.8      0.624594         0.099642   \n",
       "77276   VIMEO  default-uap_COCO_amp0.8      0.440777         0.116113   \n",
       "77277   VIMEO  default-uap_COCO_amp0.8      0.371981         0.110458   \n",
       "77278   VIMEO  default-uap_COCO_amp0.8      0.456308         0.117133   \n",
       "77279   VIMEO  default-uap_COCO_amp0.8      0.434788         0.109997   \n",
       "\n",
       "       maniqa_ssim  maniqa_psnr  maniqa_mse  \n",
       "0         0.810342    30.222343    0.000950  \n",
       "1         0.875497    29.726215    0.001065  \n",
       "2         0.855742    29.728792    0.001064  \n",
       "3         0.906851    29.779349    0.001052  \n",
       "4         0.747340    30.407143    0.000911  \n",
       "...            ...          ...         ...  \n",
       "77275     0.384374    24.441853    0.003596  \n",
       "77276     0.546673    24.441853    0.003596  \n",
       "77277     0.506352    24.441853    0.003596  \n",
       "77278     0.391060    24.441853    0.003596  \n",
       "77279     0.506051    24.441853    0.003596  \n",
       "\n",
       "[77280 rows x 7 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "# affects VRAM usage if run on GPU\n",
    "CHUNK_SIZE = 1000\n",
    "def chunker(df, size):\n",
    "    return [df.iloc[pos:pos + size] for pos in range(0, len(df), size)]\n",
    "\n",
    "\n",
    "# Run domain transform by chunks\n",
    "data_transformed = pd.DataFrame()\n",
    "for cur_df in tqdm(chunker(data_to_evaluate, 1000)):\n",
    "    cur_data_transformed = transform_full_df(df=cur_df, metrics=[metric_to_evaluate], path_to_models='./models', domain='mdtvsfa', dev='cuda:0')\n",
    "    data_transformed = pd.concat([data_transformed, cur_data_transformed])\n",
    "data_transformed    "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate results"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Overall scores (on all datasets and attacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Absolute gain: 0.094 (0.091, 0.097)\n",
      "Relative gain: 0.073 (0.070, 0.075)\n",
      "Robustness score: 0.645 (0.641, 0.650)\n",
      "Wasserstein score: 0.203\n",
      "Energy score: 0.238\n"
     ]
    }
   ],
   "source": [
    "eval_df = data_transformed.copy()\n",
    "\n",
    "# W/o domain transform:\n",
    "# eval_df = data.copy()\n",
    "\n",
    "abs_gain_scores = normalized_absolute_gain(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])\n",
    "low, high = st.t.interval(0.95, len(abs_gain_scores)-1, loc=np.mean(abs_gain_scores), scale=st.sem(abs_gain_scores))\n",
    "print('Absolute gain: {:.3f} ({:.3f}, {:.3f})'.format(abs_gain_scores.mean(), low, high))\n",
    "\n",
    "rel_gain_scores = normalized_relative_gain(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])\n",
    "low, high = st.t.interval(0.95, len(rel_gain_scores)-1, loc=np.mean(rel_gain_scores), scale=st.sem(rel_gain_scores))\n",
    "print('Relative gain: {:.3f} ({:.3f}, {:.3f})'.format(rel_gain_scores.mean(), low, high))\n",
    "\n",
    "rob_scores = robustness_score(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])\n",
    "low, high = st.t.interval(0.95, len(rob_scores)-1, loc=np.mean(rob_scores), scale=st.sem(rob_scores))\n",
    "print('Robustness score: {:.3f} ({:.3f}, {:.3f})'.format(rob_scores.mean(), low, high))\n",
    "\n",
    "print('Wasserstein score: {:.3f}'.format(calc_wasserstein_score(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])))\n",
    "print('Energy score: {:.3f}'.format(energy_distance_score(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['amifgsm', 'cumulative-uap_VOC2012_amp0.2',\n",
       "       'cumulative-uap_COCO_amp0.2', 'cumulative-uap_VOC2012_amp0.4',\n",
       "       'cumulative-uap_COCO_amp0.4', 'cumulative-uap_VOC2012_amp0.8',\n",
       "       'cumulative-uap_COCO_amp0.8', 'generative-uap_VOC2012_amp0.2',\n",
       "       'generative-uap_COCO_amp0.2', 'generative-uap_VOC2012_amp0.4',\n",
       "       'generative-uap_COCO_amp0.4', 'generative-uap_VOC2012_amp0.8',\n",
       "       'generative-uap_COCO_amp0.8', 'ifgsm', 'korhonen-et-al', 'madc',\n",
       "       'mifgsm', 'std-fgsm', 'default-uap_VOC2012_amp0.2',\n",
       "       'default-uap_COCO_amp0.2', 'default-uap_VOC2012_amp0.4',\n",
       "       'default-uap_COCO_amp0.4', 'default-uap_VOC2012_amp0.8',\n",
       "       'default-uap_COCO_amp0.8'], dtype=object)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Available attacks\n",
    "data.attack.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['NIPS', 'DERF', 'VIMEO'], dtype=object)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Available datasets\n",
    "data.dataset.unique()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scores on selected dataset\\attack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Absolute gain: 0.297 (0.288, 0.305)\n",
      "Relative gain: 0.200 (0.193, 0.207)\n",
      "Robustness score: 0.399 (0.380, 0.419)\n",
      "Wasserstein score: 0.297\n",
      "Energy score: 0.530\n"
     ]
    }
   ],
   "source": [
    "dataset = 'NIPS'  # \"NIPS\", \"DERF\", \"VIMEO\" or \"all\"\n",
    "attack = 'std-fgsm'  # attack name or \"all\" or \"uap-based\" or \"iterative\"\n",
    "uap_attacks = [ x for x in data_transformed.attack.unique() if 'uap' in x]\n",
    "\n",
    "eval_df = data_transformed.copy()\n",
    "\n",
    "# W/o domain transform\n",
    "# eval_df = data.copy()\n",
    "\n",
    "\n",
    "if dataset != 'all':\n",
    "    eval_df = eval_df[eval_df.dataset == dataset]\n",
    "if attack != 'all' and attack != 'uap-based' and attack != 'iterative':\n",
    "    eval_df = eval_df[eval_df.attack == attack]\n",
    "elif attack == 'uap-based':\n",
    "    eval_df = eval_df[eval_df.attack.isin(uap_attacks)]\n",
    "elif attack == 'iterative':\n",
    "    eval_df = eval_df[~eval_df.attack.isin(uap_attacks)]\n",
    "\n",
    "abs_gain_scores = normalized_absolute_gain(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])\n",
    "low, high = st.t.interval(0.95, len(abs_gain_scores)-1, loc=np.mean(abs_gain_scores), scale=st.sem(abs_gain_scores))\n",
    "print('Absolute gain: {:.3f} ({:.3f}, {:.3f})'.format(abs_gain_scores.mean(), low, high))\n",
    "\n",
    "rel_gain_scores = normalized_relative_gain(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])\n",
    "low, high = st.t.interval(0.95, len(rel_gain_scores)-1, loc=np.mean(rel_gain_scores), scale=st.sem(rel_gain_scores))\n",
    "print('Relative gain: {:.3f} ({:.3f}, {:.3f})'.format(rel_gain_scores.mean(), low, high))\n",
    "\n",
    "rob_scores = robustness_score(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])\n",
    "low, high = st.t.interval(0.95, len(rob_scores)-1, loc=np.mean(rob_scores), scale=st.sem(rob_scores))\n",
    "print('Robustness score: {:.3f} ({:.3f}, {:.3f})'.format(rob_scores.mean(), low, high))\n",
    "\n",
    "print('Wasserstein score: {:.3f}'.format(calc_wasserstein_score(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])))\n",
    "print('Energy score: {:.3f}'.format(energy_distance_score(eval_df[f'{metric_to_evaluate}_clear'], eval_df[f'{metric_to_evaluate}_attacked'])))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}