summarize_cp_scorings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "from helpers.tools import *\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Complete summary of all parameter searches that were performed. \n",
    "\n",
    "# Naming conventions changed towards to end of the paper: \n",
    "#SL: base\n",
    "#ML: big_base_mirror\n",
    "#SNL: poly\n",
    "#MNL: big_poly\n",
    "#LNL: nl\n",
    "#XLNL: big_nl\n",
    "\n",
    "\n",
    "# Scale_up: big_base_mirror\n",
    "# Scale_up_2:  Scale_up_2\n",
    "# Scale_up_3: Scale_up\n",
    "# Remaining data sizes are not used (Future work)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_res_folder(mypath = \"experimental_results/synthetic_data_1_grid_results/std/\"):\n",
    "\n",
    "    onlyfiles = [mypath + f for f in listdir(mypath) if isfile(join(mypath, f))]\n",
    "    onlyfiles = sorted([x for x in onlyfiles if \"summary\" in x])\n",
    "    res = pd.concat([pd.read_csv(x, index_col=0) for x in onlyfiles], axis=1).T\n",
    "    return  res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = load_res_folder(\"experimental_results/synthetic_data_1_grid_results/run_1/\")\n",
    "b = load_res_folder(\"experimental_results/synthetic_data_1_grid_results/run_2/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HP</th>\n",
       "      <th>Test1_Auroc</th>\n",
       "      <th>Test2_AUROC</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>data.ds_name</th>\n",
       "      <th>model.model_type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">base</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.999499</td>\n",
       "      <td>0.999961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.999336</td>\n",
       "      <td>0.999894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.999328</td>\n",
       "      <td>0.999933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.999534</td>\n",
       "      <td>0.999968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.999558</td>\n",
       "      <td>0.999948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">big_base_mirror</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.613235</td>\n",
       "      <td>0.626371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.506232</td>\n",
       "      <td>0.513293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.499207</td>\n",
       "      <td>0.498309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.600757</td>\n",
       "      <td>0.621380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.599960</td>\n",
       "      <td>0.621146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">big_nl</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.709532</td>\n",
       "      <td>0.748808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.503338</td>\n",
       "      <td>0.506151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.501747</td>\n",
       "      <td>0.498722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.518067</td>\n",
       "      <td>0.517684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.659285</td>\n",
       "      <td>0.688653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">big_poly</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.547126</td>\n",
       "      <td>0.555322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.518390</td>\n",
       "      <td>0.518896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.520201</td>\n",
       "      <td>0.524884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.537860</td>\n",
       "      <td>0.545076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.784930</td>\n",
       "      <td>0.828722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">nl</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.916400</td>\n",
       "      <td>0.936746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.912467</td>\n",
       "      <td>0.934696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.905332</td>\n",
       "      <td>0.932553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.915942</td>\n",
       "      <td>0.934492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.915071</td>\n",
       "      <td>0.936571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">poly</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.932779</td>\n",
       "      <td>0.948389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.931417</td>\n",
       "      <td>0.948634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.930141</td>\n",
       "      <td>0.950117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.934952</td>\n",
       "      <td>0.955290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.934503</td>\n",
       "      <td>0.954934</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "HP                                Test1_Auroc  Test2_AUROC\n",
       "data.ds_name    model.model_type                          \n",
       "base            bidirectional        0.999499     0.999961\n",
       "                convM                0.999336     0.999894\n",
       "                mlp                  0.999328     0.999933\n",
       "                transformer          0.999534     0.999968\n",
       "                unidirectional       0.999558     0.999948\n",
       "big_base_mirror bidirectional        0.613235     0.626371\n",
       "                convM                0.506232     0.513293\n",
       "                mlp                  0.499207     0.498309\n",
       "                transformer          0.600757     0.621380\n",
       "                unidirectional       0.599960     0.621146\n",
       "big_nl          bidirectional        0.709532     0.748808\n",
       "                convM                0.503338     0.506151\n",
       "                mlp                  0.501747     0.498722\n",
       "                transformer          0.518067     0.517684\n",
       "                unidirectional       0.659285     0.688653\n",
       "big_poly        bidirectional        0.547126     0.555322\n",
       "                convM                0.518390     0.518896\n",
       "                mlp                  0.520201     0.524884\n",
       "                transformer          0.537860     0.545076\n",
       "                unidirectional       0.784930     0.828722\n",
       "nl              bidirectional        0.916400     0.936746\n",
       "                convM                0.912467     0.934696\n",
       "                mlp                  0.905332     0.932553\n",
       "                transformer          0.915942     0.934492\n",
       "                unidirectional       0.915071     0.936571\n",
       "poly            bidirectional        0.932779     0.948389\n",
       "                convM                0.931417     0.948634\n",
       "                mlp                  0.930141     0.950117\n",
       "                transformer          0.934952     0.955290\n",
       "                unidirectional       0.934503     0.954934"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = load_res_folder(\"experimental_results/synthetic_data_1_grid_results/std/\")\n",
    "res.loc[res[\"model\"].isnull(), \"model\"] = \"small.yaml\"\n",
    "# Provide some summary statistics over the runs.\n",
    "res = res[[\"model.model_type\", \"data.ds_name\",'Test1_Auroc', 'Test2_AUROC']]\n",
    "res[\"Test1_Auroc\"] = res[\"Test1_Auroc\"].astype(float)\n",
    "res[\"Test2_AUROC\"] = res[\"Test2_AUROC\"].astype(float)\n",
    "\n",
    "res.groupby([\"data.ds_name\", \"model.model_type\"]).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment 1 - Joint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = load_res_folder(\"experimental_results/synthetic_data_1_joint_grid_results/splitted_run_1/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HP</th>\n",
       "      <th>Test1_Auroc</th>\n",
       "      <th>Test2_AUROC</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th>model.model_type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">big.yaml</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.952198</td>\n",
       "      <td>0.962480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.867127</td>\n",
       "      <td>0.885812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.751937</td>\n",
       "      <td>0.760817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.969769</td>\n",
       "      <td>0.977339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.946323</td>\n",
       "      <td>0.959416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">medium.yaml</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.906445</td>\n",
       "      <td>0.920411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.869493</td>\n",
       "      <td>0.887307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.915602</td>\n",
       "      <td>0.928056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.938883</td>\n",
       "      <td>0.949564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.894770</td>\n",
       "      <td>0.906847</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "HP                            Test1_Auroc  Test2_AUROC\n",
       "model       model.model_type                          \n",
       "big.yaml    bidirectional        0.952198     0.962480\n",
       "            convM                0.867127     0.885812\n",
       "            mlp                  0.751937     0.760817\n",
       "            transformer          0.969769     0.977339\n",
       "            unidirectional       0.946323     0.959416\n",
       "medium.yaml bidirectional        0.906445     0.920411\n",
       "            convM                0.869493     0.887307\n",
       "            mlp                  0.915602     0.928056\n",
       "            transformer          0.938883     0.949564\n",
       "            unidirectional       0.894770     0.906847"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = load_res_folder(\"experimental_results/synthetic_data_1_joint_grid_results/std/\")\n",
    "\n",
    "# Provide some summary statistics over the runs.\n",
    "res = res[[\"model.model_type\", \"model\",'Test1_Auroc', 'Test2_AUROC']]\n",
    "res[\"Test1_Auroc\"] = res[\"Test1_Auroc\"].astype(float)\n",
    "res[\"Test2_AUROC\"] = res[\"Test2_AUROC\"].astype(float)\n",
    "\n",
    "res.groupby([\"model\", \"model.model_type\"]).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = load_res_folder(\"experimental_results/synthetic_data_2_grid_results/splitted_run_2/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>HP</th>\n",
       "      <th>model.optimizer_lr</th>\n",
       "      <th>data.batch_size</th>\n",
       "      <th>model.weight_decay</th>\n",
       "      <th>model.corr_regularization</th>\n",
       "      <th>corr_input</th>\n",
       "      <th>regression_head</th>\n",
       "      <th>hparams_search</th>\n",
       "      <th>max_lags</th>\n",
       "      <th>n_vars</th>\n",
       "      <th>Val_Auroc</th>\n",
       "      <th>Val_MSE</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>data.ds_name</th>\n",
       "      <th>model</th>\n",
       "      <th>model.model_type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">big_base_mirror</th>\n",
       "      <th rowspan=\"4\" valign=\"top\">big.yaml</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>medium.yaml</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">scale_up_3</th>\n",
       "      <th rowspan=\"5\" valign=\"top\">lcm.yaml</th>\n",
       "      <th>bidirectional</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>65 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "HP                                            model.optimizer_lr   \n",
       "data.ds_name    model       model.model_type                       \n",
       "big_base_mirror big.yaml    bidirectional                     24  \\\n",
       "                            convM                             24   \n",
       "                            transformer                       23   \n",
       "                            unidirectional                    24   \n",
       "                medium.yaml bidirectional                     24   \n",
       "...                                                          ...   \n",
       "scale_up_3      lcm.yaml    bidirectional                     24   \n",
       "                            convM                             17   \n",
       "                            mlp                               24   \n",
       "                            transformer                       10   \n",
       "                            unidirectional                    24   \n",
       "\n",
       "HP                                            data.batch_size   \n",
       "data.ds_name    model       model.model_type                    \n",
       "big_base_mirror big.yaml    bidirectional                  24  \\\n",
       "                            convM                          24   \n",
       "                            transformer                    23   \n",
       "                            unidirectional                 24   \n",
       "                medium.yaml bidirectional                  24   \n",
       "...                                                       ...   \n",
       "scale_up_3      lcm.yaml    bidirectional                  24   \n",
       "                            convM                          17   \n",
       "                            mlp                            24   \n",
       "                            transformer                    10   \n",
       "                            unidirectional                 24   \n",
       "\n",
       "HP                                            model.weight_decay   \n",
       "data.ds_name    model       model.model_type                       \n",
       "big_base_mirror big.yaml    bidirectional                     24  \\\n",
       "                            convM                             24   \n",
       "                            transformer                       23   \n",
       "                            unidirectional                    24   \n",
       "                medium.yaml bidirectional                     24   \n",
       "...                                                          ...   \n",
       "scale_up_3      lcm.yaml    bidirectional                     24   \n",
       "                            convM                             17   \n",
       "                            mlp                               24   \n",
       "                            transformer                       10   \n",
       "                            unidirectional                    24   \n",
       "\n",
       "HP                                            model.corr_regularization   \n",
       "data.ds_name    model       model.model_type                              \n",
       "big_base_mirror big.yaml    bidirectional                            24  \\\n",
       "                            convM                                    24   \n",
       "                            transformer                              23   \n",
       "                            unidirectional                           24   \n",
       "                medium.yaml bidirectional                            24   \n",
       "...                                                                 ...   \n",
       "scale_up_3      lcm.yaml    bidirectional                            24   \n",
       "                            convM                                    17   \n",
       "                            mlp                                      24   \n",
       "                            transformer                              10   \n",
       "                            unidirectional                           24   \n",
       "\n",
       "HP                                            corr_input  regression_head   \n",
       "data.ds_name    model       model.model_type                                \n",
       "big_base_mirror big.yaml    bidirectional             24               24  \\\n",
       "                            convM                     24               24   \n",
       "                            transformer               23               23   \n",
       "                            unidirectional            24               24   \n",
       "                medium.yaml bidirectional             24               24   \n",
       "...                                                  ...              ...   \n",
       "scale_up_3      lcm.yaml    bidirectional             24               24   \n",
       "                            convM                     17               17   \n",
       "                            mlp                       24               24   \n",
       "                            transformer               10               10   \n",
       "                            unidirectional            24               24   \n",
       "\n",
       "HP                                            hparams_search  max_lags   \n",
       "data.ds_name    model       model.model_type                             \n",
       "big_base_mirror big.yaml    bidirectional                 24        24  \\\n",
       "                            convM                         24        24   \n",
       "                            transformer                   23        23   \n",
       "                            unidirectional                24        24   \n",
       "                medium.yaml bidirectional                 24        24   \n",
       "...                                                      ...       ...   \n",
       "scale_up_3      lcm.yaml    bidirectional                 24        24   \n",
       "                            convM                         17        17   \n",
       "                            mlp                           24        24   \n",
       "                            transformer                   10        10   \n",
       "                            unidirectional                24        24   \n",
       "\n",
       "HP                                            n_vars  Val_Auroc  Val_MSE  \n",
       "data.ds_name    model       model.model_type                              \n",
       "big_base_mirror big.yaml    bidirectional         24         24       24  \n",
       "                            convM                 24         24       24  \n",
       "                            transformer           23         23       23  \n",
       "                            unidirectional        24         24       24  \n",
       "                medium.yaml bidirectional         24         24       24  \n",
       "...                                              ...        ...      ...  \n",
       "scale_up_3      lcm.yaml    bidirectional         24         24       24  \n",
       "                            convM                 17         17       17  \n",
       "                            mlp                   24         24       24  \n",
       "                            transformer           10         10       10  \n",
       "                            unidirectional        24         24       24  \n",
       "\n",
       "[65 rows x 11 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We dropped quite a bit of runs initially so we might undereport.\n",
    "d.groupby([\"data.ds_name\", \"model\", \"model.model_type\"]).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HP</th>\n",
       "      <th>Test1_Auroc</th>\n",
       "      <th>Test2_AUROC</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model</th>\n",
       "      <th>data.ds_name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">big.yaml</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>0.947499</td>\n",
       "      <td>0.994413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>0.503389</td>\n",
       "      <td>0.506843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>0.847472</td>\n",
       "      <td>0.936330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_3</th>\n",
       "      <td>0.504794</td>\n",
       "      <td>0.505442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">deep.yaml</th>\n",
       "      <th>scale_up</th>\n",
       "      <td>0.538032</td>\n",
       "      <td>0.545041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>0.909038</td>\n",
       "      <td>0.956824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_3</th>\n",
       "      <td>0.503283</td>\n",
       "      <td>0.501950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">lcm.yaml</th>\n",
       "      <th>scale_up</th>\n",
       "      <td>0.668700</td>\n",
       "      <td>0.708848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_3</th>\n",
       "      <td>0.503202</td>\n",
       "      <td>0.505251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">medium.yaml</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>0.878202</td>\n",
       "      <td>0.972686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up</th>\n",
       "      <td>0.504329</td>\n",
       "      <td>0.506382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>0.691704</td>\n",
       "      <td>0.754195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">small.yaml</th>\n",
       "      <th>big_base_mirror</th>\n",
       "      <td>0.670829</td>\n",
       "      <td>0.732150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scale_up_2</th>\n",
       "      <td>0.506243</td>\n",
       "      <td>0.509576</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "HP                           Test1_Auroc  Test2_AUROC\n",
       "model       data.ds_name                             \n",
       "big.yaml    big_base_mirror     0.947499     0.994413\n",
       "            scale_up            0.503389     0.506843\n",
       "            scale_up_2          0.847472     0.936330\n",
       "            scale_up_3          0.504794     0.505442\n",
       "deep.yaml   scale_up            0.538032     0.545041\n",
       "            scale_up_2          0.909038     0.956824\n",
       "            scale_up_3          0.503283     0.501950\n",
       "lcm.yaml    scale_up            0.668700     0.708848\n",
       "            scale_up_3          0.503202     0.505251\n",
       "medium.yaml big_base_mirror     0.878202     0.972686\n",
       "            scale_up            0.504329     0.506382\n",
       "            scale_up_2          0.691704     0.754195\n",
       "small.yaml  big_base_mirror     0.670829     0.732150\n",
       "            scale_up_2          0.506243     0.509576"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = load_res_folder(\"experimental_results/synthetic_data_2_grid_results/std/\")\n",
    "\n",
    "# Provide some summary statistics over the runs.\n",
    "res = res[[\"data.ds_name\",  \"model\",'Test1_Auroc', 'Test2_AUROC']]\n",
    "res[\"Test1_Auroc\"] = res[\"Test1_Auroc\"].astype(float)\n",
    "res[\"Test2_AUROC\"] = res[\"Test2_AUROC\"].astype(float)\n",
    "\n",
    "res.groupby([\"model\",\"data.ds_name\",]).max()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment Kuramoto"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "e = load_res_folder(\"experimental_results/kuramoto_grid_results/splitted_run_1/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>HP</th>\n",
       "      <th>Test1_Auroc</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model.model_type</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>bidirectional</th>\n",
       "      <td>0.852495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>convM</th>\n",
       "      <td>0.954027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mlp</th>\n",
       "      <td>0.824648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>transformer</th>\n",
       "      <td>0.888369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unidirectional</th>\n",
       "      <td>0.905020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "HP                Test1_Auroc\n",
       "model.model_type             \n",
       "bidirectional        0.852495\n",
       "convM                0.954027\n",
       "mlp                  0.824648\n",
       "transformer          0.888369\n",
       "unidirectional       0.905020"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = load_res_folder(\"experimental_results/kuramoto_grid_results/std/\")\n",
    "\n",
    "# Provide some summary statistics over the runs.\n",
    "res = res[[\"model.model_type\",'Test1_Auroc']]\n",
    "res[\"Test1_Auroc\"] = res[\"Test1_Auroc\"].astype(float)\n",
    "\n",
    "res.groupby([ \"model.model_type\"]).mean()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "causal_pretraining",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}