results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 74,
=======
   "execution_count": 1,
>>>>>>> 59cd0aceef02fa4c20a31b60cf4bdd6508ee907e
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle as pickle\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 75,
=======
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-28 10:08:21.890885: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.1/lib64\n",
      "2023-07-28 10:08:21.890909: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\hline & \\multicolumn{4}{c}{Gaussian}  \\\\ \\hline\n",
      "Evidental Gaussian & $3.93 \\pm 0.39$& $2.73 \\pm 0.03$& $2.44 \\pm 0.39$& $4.08 \\pm 0.15$ \\\\\n",
      "Drop & $4.55 \\pm 1.20$& $3.26 \\pm 0.65$& $3.09 \\pm 0.69$& $3.78 \\pm 0.82$ \\\\\n",
      "Ense & $3.62 \\pm 0.71$& $2.44 \\pm 0.50$& $2.28 \\pm 0.33$& $3.06 \\pm 0.64$ \\\\\n",
      "Evi & $3.39 \\pm 0.98$& $2.92 \\pm 0.73$& $2.51 \\pm 0.78$& $3.04 \\pm 1.04$ \\\\\n",
      "\\hline & \\multicolumn{4}{c}{Expo}  \\\\ \\hline\n",
      "Evidental Gaussian & $25.17 \\pm 9.72$& $9.90 \\pm 4.02$& $5.59 \\pm 3.84$& $14.02 \\pm 9.10$ \\\\\n",
      "Drop & $3.90 \\pm 1.52$& $2.82 \\pm 0.70$& $3.18 \\pm 0.88$& $4.83 \\pm 1.21$ \\\\\n",
      "Ense & $3.11 \\pm 0.67$& $2.00 \\pm 0.15$& $2.55 \\pm 0.40$& $4.07 \\pm 0.63$ \\\\\n",
      "Evi & $3.71 \\pm 1.80$& $2.72 \\pm 0.77$& $2.52 \\pm 0.44$& $3.34 \\pm 0.64$ \\\\\n",
      "\\hline & \\multicolumn{4}{c}{Gamma}  \\\\ \\hline\n",
      "Evidental Gaussian & $4.28 \\pm 0.71$& $3.54 \\pm 0.55$& $5.43 \\pm 0.65$& $10.34 \\pm 0.69$ \\\\\n",
      "Drop & $3.75 \\pm 0.61$& $3.48 \\pm 0.54$& $4.84 \\pm 1.15$& $7.76 \\pm 2.00$ \\\\\n",
      "Ense & $3.05 \\pm 0.39$& $2.64 \\pm 0.19$& $3.42 \\pm 0.76$& $5.16 \\pm 1.35$ \\\\\n",
      "Evi & $2.89 \\pm 0.50$& $3.24 \\pm 0.99$& $3.34 \\pm 0.89$& $4.64 \\pm 1.19$ \\\\\n",
      "\\hline & \\multicolumn{4}{c}{Laplace}  \\\\ \\hline\n",
      "Evidental Gaussian & $9.08 \\pm 0.28$& $5.90 \\pm 0.82$& $4.41 \\pm 1.56$& $6.81 \\pm 0.41$ \\\\\n",
      "Drop & $6.32 \\pm 1.92$& $4.44 \\pm 1.37$& $3.72 \\pm 1.11$& $5.28 \\pm 1.84$ \\\\\n",
      "Ense & $4.54 \\pm 0.65$& $3.20 \\pm 0.65$& $3.28 \\pm 0.48$& $4.21 \\pm 0.81$ \\\\\n",
      "Evi & $4.47 \\pm 0.74$& $3.44 \\pm 0.78$& $3.45 \\pm 0.96$& $3.74 \\pm 1.12$ \\\\\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-28 10:08:23.139903: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:975] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-07-28 10:08:23.140742: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:975] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2023-07-28 10:08:23.141555: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.1/lib64\n",
      "2023-07-28 10:08:23.141602: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcublas.so.11'; dlerror: libcublas.so.11: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.1/lib64\n",
      "2023-07-28 10:08:23.141644: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcublasLt.so.11'; dlerror: libcublasLt.so.11: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.1/lib64\n",
      "2023-07-28 10:08:23.142809: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcusolver.so.11'; dlerror: libcusolver.so.11: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.1/lib64\n",
      "2023-07-28 10:08:23.142857: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcusparse.so.11'; dlerror: libcusparse.so.11: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.1/lib64\n",
      "2023-07-28 10:08:23.142899: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudnn.so.8'; dlerror: libcudnn.so.8: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.1/lib64\n",
      "2023-07-28 10:08:23.142906: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1850] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.\n",
      "Skipping registering GPU devices...\n",
      "2023-07-28 10:08:23.143201: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "datasets = ['Gaussian', 'Expo',  'Gamma', 'Laplace']\n",
    "\n",
    "for d in datasets:\n",
    "    evi_erros = []\n",
    "    drop_erros = []\n",
    "    ense_erros = []\n",
    "    gauss_erros = []\n",
    "\n",
    "    evi_calib = []\n",
    "    drop_calib = []\n",
    "    ense_calib = []\n",
    "    gauss_calib = []\n",
    "\n",
    "    evi_calib_plot = []\n",
    "    drop_calib_plot = []\n",
    "    ense_calib_plot = []\n",
    "    gauss_calib_plot = []\n",
    "    for i in range(0,  20):\n",
    "        with open('results/synth/' +d +'_evidental' + str(i) + '.pickle', 'rb') as handle:\n",
    "            evi = pickle.load(handle)\n",
    "\n",
    "        evi_erros.append(np.mean(abs(evi['errors']), axis=0))\n",
    "\n",
    "        with open('results/synth/' +d +'_dropout' + str(i) + '.pickle', 'rb') as handle:\n",
    "            drop = pickle.load(handle)\n",
    "\n",
    "        drop_erros.append(np.mean(abs(drop['errors']), axis=0))\n",
    "        \n",
    "        with open('results/synth/' +d +'_ensemble' + str(i) + '.pickle', 'rb') as handle:\n",
    "            ense = pickle.load(handle)\n",
    "\n",
    "        ense_erros.append(np.mean(abs(ense['errors']), axis=0))\n",
    "        \n",
    "        with open('results/synth/' +d +'_evidental_gauss' + str(i) + '.pickle', 'rb') as handle:\n",
    "            gaussian = pickle.load(handle)\n",
    "\n",
    "        gauss_erros.append(np.mean(abs(gaussian['errors']), axis=0))\n",
    "        \n",
    "    print(\"\\\\hline & \\\\multicolumn{4}{c}{\" + d + \"}  \\\\\\\\ \\\\hline\")\n",
    "    print(\"Evidental Gaussian & ${:,.2f} \\pm {:,.2f}$& ${:,.2f} \\pm {:,.2f}$& ${:,.2f} \\pm {:,.2f}$& ${:,.2f} \\pm {:,.2f}$ \\\\\\\\\".format(np.mean(gauss_erros, axis=0)[0], np.std(gauss_erros, axis=0)[0], np.mean(gauss_erros, axis=0)[1], np.std(gauss_erros, axis=0)[1], np.mean(gauss_erros, axis=0)[2], np.std(gauss_erros, axis=0)[2],np.mean(gauss_erros, axis=0)[3], np.std(gauss_erros, axis=0)[3]))\n",
    "    print(\"Drop & ${:,.2f} \\pm {:,.2f}$& ${:,.2f} \\pm {:,.2f}$& ${:,.2f} \\pm {:,.2f}$& ${:,.2f} \\pm {:,.2f}$ \\\\\\\\\".format(np.mean(drop_erros, axis=0)[0], np.std(drop_erros, axis=0)[0],np.mean(drop_erros, axis=0)[1], np.std(drop_erros, axis=0)[1],np.mean(drop_erros, axis=0)[2], np.std(drop_erros, axis=0)[2],np.mean(drop_erros, axis=0)[3], np.std(drop_erros, axis=0)[3]))\n",
    "    print(\"Ense & ${:,.2f} \\pm {:,.2f}$& ${:,.2f} \\pm {:,.2f}$& ${:,.2f} \\pm {:,.2f}$& ${:,.2f} \\pm {:,.2f}$ \\\\\\\\\".format(np.mean(ense_erros, axis=0)[0], np.std(ense_erros, axis=0)[0],np.mean(ense_erros, axis=0)[1], np.std(ense_erros, axis=0)[1],np.mean(ense_erros, axis=0)[2], np.std(ense_erros, axis=0)[2],np.mean(ense_erros, axis=0)[3], np.std(ense_erros, axis=0)[3]))\n",
    "    print(\"Evi & ${:,.2f} \\pm {:,.2f}$& ${:,.2f} \\pm {:,.2f}$& ${:,.2f} \\pm {:,.2f}$& ${:,.2f} \\pm {:,.2f}$ \\\\\\\\\".format(np.mean(evi_erros, axis=0)[0], np.std(evi_erros, axis=0)[0],np.mean(evi_erros, axis=0)[1], np.std(evi_erros, axis=0)[1],np.mean(evi_erros, axis=0)[2], np.std(evi_erros, axis=0)[2],np.mean(evi_erros, axis=0)[3], np.std(evi_erros, axis=0)[3]))\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
>>>>>>> 59cd0aceef02fa4c20a31b60cf4bdd6508ee907e
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = ['boston', 'concrete', 'energy-efficiency', 'kin8nm', 'naval', 'power-plant', 'protein', 'wine', 'yacht']"
   ]
  },
  {
   "cell_type": "code",
<<<<<<< HEAD
   "execution_count": 76,
=======
   "execution_count": 3,
>>>>>>> 59cd0aceef02fa4c20a31b60cf4bdd6508ee907e
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "boston& $1.09 \\pm 1.12$ &  $0.58 \\pm 0.08$ & $0.53 \\pm 0.03$ & $0.49 \\pm 0.03$ &  $7.18 \\pm 1.12$ &$5.50 \\pm 0.23$ & $5.40 \\pm 0.09$ &  $5.35 \\pm 0.35$ &$1.01$ &$6.36$ & $4.25$ &  $1.00$  \\\\\n",
      "concrete& $1.09 \\pm 0.10$ &  $1.10 \\pm 0.17$ & $1.06 \\pm 0.06$ & $1.08 \\pm 0.10$ &  $7.18 \\pm 0.10$ &$6.78 \\pm 0.31$ & $6.65 \\pm 0.09$ &  $6.77 \\pm 0.27$ &$0.86$ &$5.85$ & $3.89$ &  $1.00$  \\\\\n",
      "energy-efficiency& $0.26 \\pm 0.04$ &  $0.30 \\pm 0.05$ & $0.28 \\pm 0.02$ & $0.24 \\pm 0.02$ &  $6.35 \\pm 0.04$ &$3.91 \\pm 0.29$ & $3.75 \\pm 0.13$ &  $3.41 \\pm 0.16$ &$0.87$ &$6.32$ & $4.05$ &  $1.00$  \\\\\n",
      "kin8nm& $0.01 \\pm 0.00$ &  $0.02 \\pm 0.00$ & $0.02 \\pm 0.00$ & $0.01 \\pm 0.00$ &  $6.11 \\pm 0.00$ &$-1.58 \\pm 0.08$ & $-1.71 \\pm 0.04$ &  $-1.80 \\pm 0.06$ &$0.93$ &$8.22$ & $4.12$ &  $1.00$  \\\\\n",
      "naval& $0.00 \\pm 0.00$ &  $0.00 \\pm 0.00$ & $0.00 \\pm 0.00$ & $0.00 \\pm 0.00$ &  $6.09 \\pm 0.00$ &$-10.52 \\pm 0.41$ & $-10.58 \\pm 0.15$ &  $-10.83 \\pm 0.36$ &$0.94$ &$8.63$ & $4.06$ &  $1.00$  \\\\\n",
      "power-plant& $0.89 \\pm 0.01$ &  $0.89 \\pm 0.02$ & $0.87 \\pm 0.01$ & $0.86 \\pm 0.01$ &  $6.98 \\pm 0.01$ &$6.44 \\pm 0.05$ & $6.38 \\pm 0.02$ &  $6.46 \\pm 0.04$ &$0.93$ &$8.42$ & $4.18$ &  $1.00$  \\\\\n",
      "protein& $nan \\pm nan$ &  $0.74 \\pm 0.01$ & $0.73 \\pm 0.00$ & $0.76 \\pm 0.04$ &  $nan \\pm nan$ &$5.79 \\pm 0.04$ & $5.75 \\pm 0.02$ &  $5.89 \\pm 0.13$ &$nan$ &$9.62$ & $3.32$ &  $1.00$  \\\\\n",
      "wine& $0.13 \\pm 0.03$ &  $0.12 \\pm 0.00$ & $0.12 \\pm 0.00$ & $0.12 \\pm 0.00$ &  $6.23 \\pm 0.03$ &$2.49 \\pm 0.14$ & $2.40 \\pm 0.03$ &  $2.52 \\pm 0.17$ &$0.97$ &$6.83$ & $4.20$ &  $1.00$  \\\\\n",
      "yacht& $0.17 \\pm 0.16$ &  $0.21 \\pm 0.04$ & $0.21 \\pm 0.01$ & $0.17 \\pm 0.02$ &  $6.27 \\pm 0.16$ &$2.71 \\pm 0.62$ & $2.54 \\pm 0.17$ &  $1.87 \\pm 0.41$ &$0.91$ &$5.85$ & $4.11$ &  $1.00$  \\\\\n"
     ]
    }
   ],
   "source": [
    "for d in datasets:\n",
    "    # create a dataframe for each model and their results\n",
    "    evidental_gauss_df = pd.DataFrame()\n",
    "    dropout_df = pd.DataFrame()\n",
    "    ensemble_df = pd.DataFrame()\n",
    "    evidental_df = pd.DataFrame()\n",
    "    bbbp_df = pd.DataFrame()\n",
    "\n",
    "    for i in range(0, 20):\n",
    "        with open('results/'+d+'/evidental_gauss'+str(i)+'.pickle', 'rb') as handle:\n",
    "            evidental_gauss = pickle.load(handle)\n",
    "\n",
    "        evidental_gauss_df = pd.concat([evidental_gauss_df, pd.DataFrame(evidental_gauss)], ignore_index=True)\n",
    "        evidental_gauss_df.replace(np.inf, np.nan, inplace=True)\n",
    "        evidental_gauss_df.dropna(inplace=True)\n",
    "\n",
    "        with open('results/'+d+'/dropout'+str(i)+'.pickle', 'rb') as handle:\n",
    "            dropout = pickle.load(handle)\n",
    "        \n",
    "        # add the dropout results to the dataframe\n",
    "        dropout_df = pd.concat([dropout_df, pd.DataFrame(dropout)], ignore_index=True)\n",
    "\n",
    "\n",
    "        with open('results/'+d+'/ensemble'+str(i)+'.pickle', 'rb') as handle:\n",
    "            ensemble = pickle.load(handle)\n",
    "\n",
    "        # add the ensemble results to the dataframe\n",
    "        ensemble_df = pd.concat([ensemble_df, pd.DataFrame(ensemble)], ignore_index=True)\n",
    "\n",
    "        with open('results/'+d+'/evidental'+str(i)+'.pickle', 'rb') as handle:\n",
    "            evidental = pickle.load(handle)\n",
    "        \n",
    "        # add the evidental results to the dataframe\n",
    "        evidental_df = pd.concat([evidental_df, pd.DataFrame(evidental)], ignore_index=True)\n",
    "\n",
    "    evidental_gauss_df.replace(np.inf, np.nan, inplace=True)\n",
    "    evidental_gauss_df.dropna()\n",
    "    # print the results as a latex table\n",
    "    #print(d + \" & ${:,.2f} \\pm {:,.2f}$  & ${:,.2f} \\pm {:,.2f}$  & ${:,.2f} \\pm {:,.2f}$ & ${:,.2f} \\pm {:,.2f}$ & ${:,.2f} \\pm {:,.2f}$ &  ${:,.2f} \\pm {:,.2f}$ &${:,.2f} \\pm {:,.2f}$ & ${:,.2f} \\pm {:,.2f}$ &  ${:,.2f}$ &${:,.2f}$ & ${:,.2f}$ &  ${:,.2f}$  \\\\\\\\\".format(dropout_df.tls.mean(), 2.0*dropout_df.tls.std(), bbbp_df.tls.mean(), 2.0*bbbp_df.tls.std(), ensemble_df.tls.mean(), 2.0*ensemble_df.tls.std(), evidental_df.tls.mean(), 2.0*evidental_df.tls.std(), dropout_df.nlls.mean(), 2.0*dropout_df.nlls.std(),bbbp_df.nlls.mean(), 2.0*bbbp_df.nlls.std(), ensemble_df.nlls.mean(), 2.0*ensemble_df.nlls.std(), evidental_df.nlls.mean(), 2.0*evidental_df.nlls.std(), dropout_df.times.mean()/evidental_df.times.mean(),bbbp_df.times.mean()/evidental_df.times.mean(), ensemble_df.times.mean()/evidental_df.times.mean(), evidental_df.times.mean()/evidental_df.times.mean()))\n",
<<<<<<< HEAD
    "    #print(d + \" & ${:,.2f} \\pm {:,.2f}$ & ${:,.2f} \\pm {:,.2f}$ &  ${:,.2f} \\pm {:,.2f}$ &${:,.2f} \\pm {:,.2f}$ & ${:,.2f} \\pm {:,.2f}$ &  ${:,.2f} \\pm {:,.2f}$ &${:,.2f}$ & ${:,.2f}$ &  ${:,.2f}$  \\\\\\\\\".format(dropout_df.tls.mean(), 2.0*dropout_df.tls.std(), ensemble_df.tls.mean(), 2.0*ensemble_df.tls.std(), evidental_df.tls.mean(), 2.0*evidental_df.tls.std(), dropout_df.nlls.mean(), 2.0*dropout_df.nlls.std(), ensemble_df.nlls.mean(), 2.0*ensemble_df.nlls.std(), evidental_df.nlls.mean(), 2.0*evidental_df.nlls.std(), dropout_df.times.mean()/evidental_df.times.mean(), ensemble_df.times.mean()/evidental_df.times.mean(), evidental_df.times.mean()/evidental_df.times.mean()))\n",
    "    print(d + \"& ${:,.2f} \\pm {:,.2f}$ &  ${:,.2f} \\pm {:,.2f}$ & ${:,.2f} \\pm {:,.2f}$ & ${:,.2f} \\pm {:,.2f}$ &  ${:,.2f} \\pm {:,.2f}$ &${:,.2f} \\pm {:,.2f}$ & ${:,.2f} \\pm {:,.2f}$ &  ${:,.2f} \\pm {:,.2f}$ &${:,.2f}$ &${:,.2f}$ & ${:,.2f}$ &  ${:,.2f}$  \\\\\\\\\".format(evidental_gauss_df.tls.mean(), 2.0*evidental_gauss_df.tls.std(), dropout_df.tls.mean(), 2.0*dropout_df.tls.std(), ensemble_df.tls.mean(), 2.0*ensemble_df.tls.std(), evidental_df.tls.mean(), 2.0*evidental_df.tls.std(), evidental_gauss_df.nlls.mean(), 2.0*evidental_gauss_df.nlls.std(), dropout_df.nlls.mean(), 2.0*dropout_df.nlls.std(), ensemble_df.nlls.mean(), 2.0*ensemble_df.nlls.std(), evidental_df.nlls.mean(), 2.0*evidental_df.nlls.std(), evidental_gauss_df.times.mean()/evidental_df.times.mean(), dropout_df.times.mean()/evidental_df.times.mean(), ensemble_df.times.mean()/evidental_df.times.mean(), evidental_df.times.mean()/evidental_df.times.mean()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = 'protein'\n",
    "# create a dataframe for each model and their results\n",
    "evidental_gauss_df = pd.DataFrame()\n",
    "dropout_df = pd.DataFrame()\n",
    "ensemble_df = pd.DataFrame()\n",
    "evidental_df = pd.DataFrame()\n",
    "bbbp_df = pd.DataFrame()\n",
    "\n",
    "for i in range(0, 20):\n",
    "    with open('results/'+d+'/evidental_gauss'+str(i)+'.pickle', 'rb') as handle:\n",
    "        evidental_gauss = pickle.load(handle)\n",
    "\n",
    "    evidental_gauss_df = pd.concat([evidental_gauss_df, pd.DataFrame(evidental_gauss)], ignore_index=True)\n",
    "\n",
    "    with open('results/'+d+'/dropout'+str(i)+'.pickle', 'rb') as handle:\n",
    "        dropout = pickle.load(handle)\n",
    "    \n",
    "    # add the dropout results to the dataframe\n",
    "    dropout_df = pd.concat([dropout_df, pd.DataFrame(dropout)], ignore_index=True)\n",
    "\n",
    "\n",
    "    with open('results/'+d+'/ensemble'+str(i)+'.pickle', 'rb') as handle:\n",
    "        ensemble = pickle.load(handle)\n",
    "\n",
    "    # add the ensemble results to the dataframe\n",
    "    ensemble_df = pd.concat([ensemble_df, pd.DataFrame(ensemble)], ignore_index=True)\n",
    "\n",
    "    with open('results/'+d+'/evidental'+str(i)+'.pickle', 'rb') as handle:\n",
    "        evidental = pickle.load(handle)\n",
    "    \n",
    "    # add the evidental results to the dataframe\n",
    "    evidental_df = pd.concat([evidental_df, pd.DataFrame(evidental)], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tls</th>\n",
       "      <th>nlls</th>\n",
       "      <th>times</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.695157</td>\n",
       "      <td>8.789207</td>\n",
       "      <td>0.009144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.434016</td>\n",
       "      <td>8.528068</td>\n",
       "      <td>0.008051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2.311611</td>\n",
       "      <td>8.405663</td>\n",
       "      <td>0.011426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2.553181</td>\n",
       "      <td>8.647232</td>\n",
       "      <td>0.010667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2.532254</td>\n",
       "      <td>8.626307</td>\n",
       "      <td>0.009834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2.473305</td>\n",
       "      <td>8.567357</td>\n",
       "      <td>0.008067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2.440759</td>\n",
       "      <td>8.534811</td>\n",
       "      <td>0.009140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1.831771</td>\n",
       "      <td>7.925822</td>\n",
       "      <td>0.014067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2.561483</td>\n",
       "      <td>8.655535</td>\n",
       "      <td>0.011666</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         tls      nlls     times\n",
       "1   2.695157  8.789207  0.009144\n",
       "3   2.434016  8.528068  0.008051\n",
       "6   2.311611  8.405663  0.011426\n",
       "7   2.553181  8.647232  0.010667\n",
       "9   2.532254  8.626307  0.009834\n",
       "11  2.473305  8.567357  0.008067\n",
       "15  2.440759  8.534811  0.009140\n",
       "17  1.831771  7.925822  0.014067\n",
       "19  2.561483  8.655535  0.011666"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evidental_gauss_df.replace(np.inf, np.nan, inplace=True)\n",
    "evidental_gauss_df.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tls</th>\n",
       "      <th>nlls</th>\n",
       "      <th>times</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.007993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.695157</td>\n",
       "      <td>8.789207</td>\n",
       "      <td>0.009144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.012217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.434016</td>\n",
       "      <td>8.528068</td>\n",
       "      <td>0.008051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.009226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.012836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2.311611</td>\n",
       "      <td>8.405663</td>\n",
       "      <td>0.011426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2.553181</td>\n",
       "      <td>8.647232</td>\n",
       "      <td>0.010667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.010154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2.532254</td>\n",
       "      <td>8.626307</td>\n",
       "      <td>0.009834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.011042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2.473305</td>\n",
       "      <td>8.567357</td>\n",
       "      <td>0.008067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.011336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.011458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.010136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2.440759</td>\n",
       "      <td>8.534811</td>\n",
       "      <td>0.009140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.011304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1.831771</td>\n",
       "      <td>7.925822</td>\n",
       "      <td>0.014067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.010427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2.561483</td>\n",
       "      <td>8.655535</td>\n",
       "      <td>0.011666</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         tls      nlls     times\n",
       "0        NaN       NaN  0.007993\n",
       "1   2.695157  8.789207  0.009144\n",
       "2        NaN       NaN  0.012217\n",
       "3   2.434016  8.528068  0.008051\n",
       "4        NaN       NaN  0.009226\n",
       "5        NaN       NaN  0.012836\n",
       "6   2.311611  8.405663  0.011426\n",
       "7   2.553181  8.647232  0.010667\n",
       "8        NaN       NaN  0.010154\n",
       "9   2.532254  8.626307  0.009834\n",
       "10       NaN       NaN  0.011042\n",
       "11  2.473305  8.567357  0.008067\n",
       "12       NaN       NaN  0.011336\n",
       "13       NaN       NaN  0.011458\n",
       "14       NaN       NaN  0.010136\n",
       "15  2.440759  8.534811  0.009140\n",
       "16       NaN       NaN  0.011304\n",
       "17  1.831771  7.925822  0.014067\n",
       "18       NaN       NaN  0.010427\n",
       "19  2.561483  8.655535  0.011666"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evidental_gauss_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tls      0.743330\n",
      "nlls     5.790858\n",
      "times    0.097403\n",
      "dtype: float64\n",
      "tls      0.731667\n",
      "nlls     5.746283\n",
      "times    0.033587\n",
      "dtype: float64\n",
      "tls      0.755936\n",
      "nlls     5.888663\n",
      "times    0.010120\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print(dropout_df.mean())\n",
    "print(ensemble_df.mean())\n",
    "print(evidental_df.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         tls      nlls     times\n",
      "0   0.784944  5.990781  0.011694\n",
      "1   0.744954  5.847714  0.010884\n",
      "2   0.785459  5.995404  0.008271\n",
      "3   0.747156  5.858718  0.009686\n",
      "4        NaN       NaN  0.010516\n",
      "5   0.743783  5.846037  0.010333\n",
      "6   0.787811  6.009517  0.011070\n",
      "7   0.735230  5.821364  0.010548\n",
      "8   0.748179  5.858616  0.008156\n",
      "9   0.775983  5.957837  0.011255\n",
      "10  0.743635  5.830379  0.007609\n",
      "11  0.741504  5.834481  0.008240\n",
      "12  0.748110  5.854728  0.011185\n",
      "13  0.768675  5.933409  0.010946\n",
      "14  0.766331  5.930435  0.008234\n",
      "15  0.739253  5.817686  0.009998\n",
      "16  0.754034  5.882862  0.012125\n",
      "17  0.753754  5.884056  0.008355\n",
      "18  0.738058  5.841913  0.010259\n",
      "19       NaN       NaN  0.013038\n"
     ]
    }
   ],
   "source": [
    "#print(dropout_df)\n",
    "#print(ensemble_df)\n",
    "print(evidental_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9.624618614869044\n",
      "3.3187629940691097\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "print(dropout_df.times.mean()/evidental_df.times.mean())\n",
    "print(ensemble_df.times.mean()/evidental_df.times.mean())\n",
    "print(evidental_df.times.mean()/evidental_df.times.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
=======
    "    print(d + \" & ${:,.2f} \\pm {:,.2f}$ & ${:,.2f} \\pm {:,.2f}$ &  ${:,.2f} \\pm {:,.2f}$ &${:,.2f} \\pm {:,.2f}$ & ${:,.2f} \\pm {:,.2f}$ &  ${:,.2f} \\pm {:,.2f}$ &${:,.2f}$ & ${:,.2f}$ &  ${:,.2f}$  \\\\\\\\\".format(dropout_df.tls.mean(), 2.0*dropout_df.tls.std(), ensemble_df.tls.mean(), 2.0*ensemble_df.tls.std(), evidental_df.tls.mean(), 2.0*evidental_df.tls.std(), dropout_df.nlls.mean(), 2.0*dropout_df.nlls.std(), ensemble_df.nlls.mean(), 2.0*ensemble_df.nlls.std(), evidental_df.nlls.mean(), 2.0*evidental_df.nlls.std(), dropout_df.times.mean()/evidental_df.times.mean(), ensemble_df.times.mean()/evidental_df.times.mean(), evidental_df.times.mean()/evidental_df.times.mean()))"
   ]
>>>>>>> 59cd0aceef02fa4c20a31b60cf4bdd6508ee907e
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "evi_q_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}