FinalResults-updated-with-PRAUC.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext rpy2.ipython\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import glob\n",
    "from sklearn import metrics\n",
    "from scipy.stats import wilcoxon\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "wavenet_preds_dir = \"preds/test_preds_wavenet/preds_test_WaveNet_\"\n",
    "wavenet_targets_dir = \"targets/targets_wavenet/targets_test_WaveNet_\"\n",
    "wavenet_labels_dir = \"labels/test_labels_wavenet/label_list_WaveNet_\"\n",
    "\n",
    "cae_preds_dir = \"preds/test_preds_cae/preds_test_CAE_\"\n",
    "cae_targets_dir = \"targets/targets_cae/targets_test_CAE_\"\n",
    "cae_labels_dir = \"labels/test_labels_cae/label_list_CAE_\"\n",
    "scenes = (\n",
    "    \"beach\",\n",
    "    \"bus\",\n",
    "    \"cafe_restaurant\",\n",
    "    \"car\",\n",
    "    \"city_center\",\n",
    "    \"forest_path\",\n",
    "    \"grocery_store\",\n",
    "    \"home\",\n",
    "    \"library\",\n",
    "    \"metro_station\",\n",
    "    \"office\",\n",
    "    \"park\",\n",
    "    \"residential_area\",\n",
    "    \"train\",\n",
    "    \"tram\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def results(scene_name, preds_dir, targets_dir, labels_dir):\n",
    "    preds = np.load(\"{}{}.npy\".format(preds_dir,scene_name))\n",
    "    preds = np.squeeze(preds)\n",
    "    targets =np.load(\"{}{}.npy\".format(targets_dir,scene_name))\n",
    "    targets = np.squeeze(targets)\n",
    "    labels =np.load(\"{}{}.npy\".format(labels_dir,scene_name))\n",
    "    return preds, targets, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WaveNet Results:\n",
      "0.72 beach (0.7249212353262138)\n",
      "0.83 bus (0.8287052631578948)\n",
      "0.76 cafe_restaurant (0.7561466229663864)\n",
      "0.82 car (0.8217299387792277)\n",
      "0.82 city_center (0.8236671102689241)\n",
      "0.72 forest_path (0.7175636189877798)\n",
      "0.77 grocery_store (0.7716451089573134)\n",
      "0.69 home (0.6905454425988069)\n",
      "0.67 library (0.6662959154254813)\n",
      "0.79 metro_station (0.790010652017652)\n",
      "0.78 office (0.7841212117390216)\n",
      "0.80 park (0.8038107366043231)\n",
      "0.78 residential_area (0.777699619439862)\n",
      "0.84 train (0.8382276824461414)\n",
      "0.87 tram (0.8695021100364341)\n"
     ]
    }
   ],
   "source": [
    "print('WaveNet Results:')\n",
    "wavenet_df_list = []\n",
    "for scene in scenes:\n",
    "    wn_preds, wn_targets, wn_labels = results(scene,wavenet_preds_dir, wavenet_targets_dir, wavenet_labels_dir )\n",
    "    wn_SE = (wn_preds - wn_targets) ** 2\n",
    "    wn_mse_col = wn_SE.mean(axis=0)\n",
    "    wn_mse = wn_SE.mean(axis=1)\n",
    "    wn_fpr, wn_tpr, wn_thresholds = metrics.roc_curve(wn_labels,wn_mse)\n",
    "    wn_roc_auc_mse = metrics.auc(wn_fpr, wn_tpr)\n",
    "    df = pd.DataFrame()\n",
    "    df['mse'] =  wn_mse\n",
    "    df['labels'] = wn_labels\n",
    "    wavenet_df_list.append(df)\n",
    "    print('{:.2f} {} ({})'.format(wn_roc_auc_mse, scene, wn_roc_auc_mse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CAE Results:\n",
      "0.69 beach (0.6942539539938926)\n",
      "0.79 bus (0.7929294414196411)\n",
      "0.69 cafe_restaurant (0.6925009523016172)\n",
      "0.79 car (0.7896839815949859)\n",
      "0.75 city_center (0.751457434700052)\n",
      "0.65 forest_path (0.6545196343519264)\n",
      "0.71 grocery_store (0.7050269513862195)\n",
      "0.69 home (0.6898417472535412)\n",
      "0.59 library (0.586443300797961)\n",
      "0.74 metro_station (0.7429961169649938)\n",
      "0.78 office (0.7796394913934745)\n",
      "0.70 park (0.6985115740822218)\n",
      "0.73 residential_area (0.7343311785037272)\n",
      "0.82 train (0.8230979782799646)\n",
      "0.80 tram (0.8045354121379408)\n"
     ]
    }
   ],
   "source": [
    "cae_df_list = []\n",
    "\n",
    "print('CAE Results:')\n",
    "for scene in scenes:\n",
    "    cae_preds, cae_targets, cae_labels = results(scene,cae_preds_dir, cae_targets_dir, cae_labels_dir )\n",
    "    cae_SE = (cae_preds - cae_targets) ** 2\n",
    "    cae_mse = cae_SE.mean(axis=1)\n",
    "    cae_fpr, cae_tpr, cae_thresholds = metrics.roc_curve(cae_labels,cae_mse)\n",
    "    cae_roc_auc_mse = metrics.auc(cae_fpr, cae_tpr)\n",
    "    df_cae = pd.DataFrame()\n",
    "    df_cae['mse'] =  cae_mse\n",
    "    df_cae['labels'] = cae_labels\n",
    "    cae_df_list.append(df_cae)\n",
    "    print('{:.2f} {} ({})'.format(cae_roc_auc_mse, scene, cae_roc_auc_mse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfs = [wavenet_df_list, cae_df_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "R[write to console]: Type 'citation(\"pROC\")' for a citation.\n",
      "\n",
      "R[write to console]: \n",
      "Attaching package: ‘pROC’\n",
      "\n",
      "\n",
      "R[write to console]: The following objects are masked from ‘package:stats’:\n",
      "\n",
      "    cov, smooth, var\n",
      "\n",
      "\n",
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n",
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% CI: 0.7033-0.7465 (DeLong)\n",
      "95% CI: 0.6731-0.7154 (DeLong)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n",
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% CI: 0.8092-0.8482 (DeLong)\n",
      "95% CI: 0.7728-0.8131 (DeLong)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n",
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% CI: 0.7359-0.7764 (DeLong)\n",
      "95% CI: 0.6715-0.7135 (DeLong)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n",
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% CI: 0.7986-0.8449 (DeLong)\n",
      "95% CI: 0.7672-0.8121 (DeLong)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n",
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% CI: 0.8043-0.843 (DeLong)\n",
      "95% CI: 0.7316-0.7713 (DeLong)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n",
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% CI: 0.6961-0.739 (DeLong)\n",
      "95% CI: 0.6322-0.6769 (DeLong)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n",
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% CI: 0.7463-0.797 (DeLong)\n",
      "95% CI: 0.679-0.7311 (DeLong)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n",
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% CI: 0.6717-0.7094 (DeLong)\n",
      "95% CI: 0.6712-0.7084 (DeLong)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n",
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% CI: 0.6449-0.6877 (DeLong)\n",
      "95% CI: 0.5677-0.6052 (DeLong)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n",
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% CI: 0.7712-0.8088 (DeLong)\n",
      "95% CI: 0.7238-0.7622 (DeLong)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n",
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% CI: 0.7668-0.8015 (DeLong)\n",
      "95% CI: 0.7627-0.7966 (DeLong)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n",
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% CI: 0.782-0.8257 (DeLong)\n",
      "95% CI: 0.6783-0.7188 (DeLong)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n",
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% CI: 0.756-0.7994 (DeLong)\n",
      "95% CI: 0.7128-0.7559 (DeLong)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n",
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% CI: 0.8178-0.8586 (DeLong)\n",
      "95% CI: 0.8031-0.8431 (DeLong)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n",
      "R[write to console]: Setting levels: control = 0, case = 1\n",
      "\n",
      "R[write to console]: Setting direction: controls < cases\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95% CI: 0.852-0.887 (DeLong)\n",
      "95% CI: 0.7865-0.8226 (DeLong)\n"
     ]
    }
   ],
   "source": [
    "%%R -i  dfs\n",
    "\n",
    "library(pROC)\n",
    "wavenet_df_list<-dfs[[1]]\n",
    "cae_df_list<-dfs[[2]]\n",
    "for(i in 1:15) {\n",
    "    roc_1<-roc(wavenet_df_list[[i]]$labels, wavenet_df_list[[i]]$mse)\n",
    "    roc_2<-roc(cae_df_list[[i]]$labels, cae_df_list[[i]]$mse)\n",
    "    print(ci.auc(roc_1))\n",
    "    print(ci.auc(roc_2))\n",
    "    #print(roc.test(roc_1, roc_2, method=c(\"delong\")))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WaveNet Results:\n",
      "0.29 beach (0.29379025535415404)\n",
      "0.48 bus (0.47598223339761025)\n",
      "0.24 cafe_restaurant (0.23564432425521645)\n",
      "0.61 car (0.6059104803314989)\n",
      "0.32 city_center (0.3155788168980619)\n",
      "0.09 forest_path (0.08872043440048213)\n",
      "0.24 grocery_store (0.23889167864322527)\n",
      "0.07 home (0.0734346613481804)\n",
      "0.18 library (0.17774899063906918)\n",
      "0.14 metro_station (0.13611963056317913)\n",
      "0.21 office (0.20684201131174249)\n",
      "0.46 park (0.45607483271930704)\n",
      "0.22 residential_area (0.22124409857106883)\n",
      "0.46 train (0.459781283129454)\n",
      "0.56 tram (0.563348983744147)\n"
     ]
    }
   ],
   "source": [
    "wavenet_df_pr_list = []\n",
    "print('WaveNet Results:')\n",
    "for scene in scenes:\n",
    "    wn_preds, wn_targets, wn_labels = results(scene,wavenet_preds_dir, wavenet_targets_dir, wavenet_labels_dir )\n",
    "    wn_SE = (wn_preds - wn_targets) ** 2\n",
    "    wn_mse_col = wn_SE.mean(axis=0)\n",
    "    wn_mse = wn_SE.mean(axis=1)\n",
    "    wn_precision, wn_recall, _ = metrics.precision_recall_curve(wn_labels,wn_mse)\n",
    "    wn_pr_auc_mse = metrics.auc(wn_recall, wn_precision)\n",
    "    df_pr = pd.DataFrame()\n",
    "    df_pr['mse'] =  wn_mse\n",
    "    df_pr['labels'] = wn_labels\n",
    "    wavenet_df_pr_list.append(df_pr)\n",
    "    print('{:.2f} {} ({})'.format(wn_pr_auc_mse, scene, wn_pr_auc_mse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CAE Results:\n",
      "0.12 beach (0.11714791518898612)\n",
      "0.38 bus (0.38061454765119745)\n",
      "0.08 cafe_restaurant (0.07978159613958724)\n",
      "0.47 car (0.46540222073680715)\n",
      "0.13 city_center (0.12912540372694345)\n",
      "0.06 forest_path (0.061924587241173515)\n",
      "0.10 grocery_store (0.09878014991818483)\n",
      "0.05 home (0.04615333908956339)\n",
      "0.03 library (0.025959896021388568)\n",
      "0.06 metro_station (0.0611757922176793)\n",
      "0.15 office (0.14997070729091153)\n",
      "0.06 park (0.05959972071777134)\n",
      "0.05 residential_area (0.05250776384662553)\n",
      "0.37 train (0.36687000095952677)\n",
      "0.39 tram (0.3872904371435507)\n"
     ]
    }
   ],
   "source": [
    "cae_df_pr_list = []\n",
    "print('CAE Results:')\n",
    "for scene in scenes:\n",
    "    cae_preds, cae_targets, cae_labels = results(scene,cae_preds_dir, cae_targets_dir, cae_labels_dir )\n",
    "    cae_SE = (cae_preds - cae_targets) ** 2\n",
    "    cae_mse = cae_SE.mean(axis=1)\n",
    "    cae_precision, cae_recall, _ = metrics.precision_recall_curve(cae_labels,cae_mse)\n",
    "    cae_pr_auc_mse = metrics.auc(cae_recall, cae_precision)\n",
    "    df_pr_cae = pd.DataFrame()\n",
    "    df_pr_cae['mse'] =  cae_mse\n",
    "    df_pr_cae['labels'] = cae_labels\n",
    "    cae_df_pr_list.append(df_pr_cae)\n",
    "    print('{:.2f} {} ({})'.format(cae_pr_auc_mse, scene, cae_pr_auc_mse))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "beach, 0.29379025535415404, confidence interval (0.29317560168127915, 0.2947330323093588)\n",
      "bus, 0.47598223339761025, confidence interval (0.475165777778346, 0.4768180155552121)\n",
      "cafe_restaurant, 0.23564432425521645, confidence interval (0.2356359625636611, 0.23707724421426057)\n",
      "car, 0.6059104803314989, confidence interval (0.6054881630472997, 0.6071591235653107)\n",
      "city_center, 0.3155788168980619, confidence interval (0.31538536740721707, 0.31702628822643825)\n",
      "forest_path, 0.08872043440048213, confidence interval (0.08843804133828642, 0.08935847700009647)\n",
      "grocery_store, 0.23889167864322527, confidence interval (0.23821244738951097, 0.23998854189369356)\n",
      "home, 0.0734346613481804, confidence interval (0.07331548564989593, 0.07400095403810661)\n",
      "library, 0.17774899063906918, confidence interval (0.17762082117296402, 0.17905772023526528)\n",
      "metro_station, 0.13611963056317913, confidence interval (0.13650232428191822, 0.13761500255931736)\n",
      "office, 0.20684201131174249, confidence interval (0.2067461445426902, 0.20788652531082447)\n",
      "park, 0.45607483271930704, confidence interval (0.4544502781076918, 0.456152281368314)\n",
      "residential_area, 0.22124409857106883, confidence interval (0.22150727214451194, 0.2232059705361143)\n",
      "train, 0.459781283129454, confidence interval (0.4594841911486724, 0.461216953290633)\n",
      "tram, 0.563348983744147, confidence interval (0.5621619760870589, 0.5638289100501213)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.utils import resample\n",
    "import scipy.stats as st\n",
    "def ci_calc(df):\n",
    "    sample_auc = []\n",
    "    for i in range(2000):\n",
    "        df_boot = resample(df, replace=True, n_samples=len(df))\n",
    "        precision, recall, _ = metrics.precision_recall_curve(df_boot.labels,df_boot.mse)\n",
    "        pr_auc_mse = metrics.auc(recall, precision)\n",
    "        sample_auc.append(pr_auc_mse)\n",
    "\n",
    "    return st.norm.interval(alpha=0.95, loc=np.mean(sample_auc), scale=st.sem(sample_auc))\n",
    "\n",
    "for i in range(len(scenes)):\n",
    "    df_wn_i = wavenet_df_pr_list[i]\n",
    "    wn_precision, wn_recall, _ = metrics.precision_recall_curve(df_wn_i.labels, df_wn_i.mse)\n",
    "    wn_pr_auc_mse = metrics.auc(wn_recall, wn_precision)\n",
    "    print(\"{}, {}, confidence interval {}\".format(scenes[i], wn_pr_auc_mse, ci_calc(df_wn_i)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "beach, 0.11714791518898612, confidence interval (0.11668453947777523, 0.1177100138291048)\n",
      "bus, 0.38061454765119745, confidence interval (0.3790191222592182, 0.3806519321780527)\n",
      "cafe_restaurant, 0.07978159613958724, confidence interval (0.0796558411181238, 0.08031919634036377)\n",
      "car, 0.46540222073680715, confidence interval (0.46391641805092, 0.46565214546072564)\n",
      "city_center, 0.12912540372694345, confidence interval (0.12851034711065767, 0.1296074515648582)\n",
      "forest_path, 0.061924587241173515, confidence interval (0.06190200470150765, 0.06261800444226524)\n",
      "grocery_store, 0.09878014991818483, confidence interval (0.09913621676530199, 0.10028755070304254)\n",
      "home, 0.04615333908956339, confidence interval (0.04603750882983319, 0.04627930149449154)\n",
      "library, 0.025959896021388568, confidence interval (0.026156017908079258, 0.026325819362201793)\n",
      "metro_station, 0.0611757922176793, confidence interval (0.06110664319310277, 0.06155820429153977)\n",
      "office, 0.14997070729091153, confidence interval (0.14996890728664808, 0.1507591980067397)\n",
      "park, 0.05959972071777134, confidence interval (0.05943174180904865, 0.0599271891890796)\n",
      "residential_area, 0.05250776384662553, confidence interval (0.052456601419379, 0.05281567385385542)\n",
      "train, 0.36687000095952677, confidence interval (0.36674024710770975, 0.3685313750362205)\n",
      "tram, 0.3872904371435507, confidence interval (0.3864293711028078, 0.3881459116436727)\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(scenes)):\n",
    "    df_cae_i = cae_df_pr_list[i]\n",
    "    cae_precision, cae_recall, _ = metrics.precision_recall_curve(df_cae_i.labels, df_cae_i.mse)\n",
    "    cae_pr_auc_mse = metrics.auc(cae_recall, cae_precision)\n",
    "    print(\"{}, {}, confidence interval {}\".format(scenes[i], cae_pr_auc_mse, ci_calc(df_cae_i)))\n"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}