Data Analysis of Group Performances.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd60a9b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d79b25d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_df_into_right_columns(d, samplingmodels = False):\n",
    "    def nested_dict_to_df(d, path=None):\n",
    "        \"\"\" \n",
    "        Convert a nested dictionary with any number of nested dimensions to a pandas DataFrame. \n",
    "        Each row represents a unique path through the nested dictionary.\n",
    "        \"\"\"\n",
    "        if path is None:\n",
    "            path = []\n",
    "\n",
    "        if isinstance(d, dict):\n",
    "            rows = []\n",
    "            for key, value in d.items():\n",
    "                new_path = path + [key]\n",
    "                rows.extend(nested_dict_to_df(value, new_path))\n",
    "            return rows\n",
    "        else:\n",
    "            return [path + [d]]\n",
    "\n",
    "    # Convert the nested dictionary to a list of rows\n",
    "    rows = nested_dict_to_df(d)\n",
    "\n",
    "    # Determine the maximum number of columns\n",
    "    max_cols = max(len(row) for row in rows)\n",
    "\n",
    "    # Create a DataFrame with appropriate column names\n",
    "    df_final = pd.DataFrame(rows, columns=[f'Level_{i+1}' for i in range(max_cols - 1)] + ['Value'])\n",
    "#     display(df_final)\n",
    "    if samplingmodels:\n",
    "        df_final[['data shard', 'nbr of shards']] = df_final['Level_1'].apply(pd.Series)\n",
    "        df_final.drop('Level_1', axis=1, inplace=True)\n",
    "    return df_final\n",
    "\n",
    "def make_df_into_right_columns_secondmode(data, acc_col_name = ''):\n",
    "    df = pd.DataFrame.from_dict(data, orient='index')\n",
    "    df.index.name = 'data shard'\n",
    "    df.reset_index(inplace=True)\n",
    "    \n",
    "    test_df = df[[\"data shard\", \"acc_test\"]].rename(columns={\"acc_test\": acc_col_name})\n",
    "    test_df[\"data split\"] = \"test\"\n",
    "\n",
    "    transformed_df = test_df.copy()\n",
    "    if 'acc_train' in df.columns:\n",
    "        train_df = df[[\"data shard\", \"acc_train\"]].rename(columns={\"acc_train\": acc_col_name})\n",
    "        train_df[\"data split\"] = \"train\"\n",
    "\n",
    "        # Concatenating the train and test dataframes\n",
    "        transformed_df = pd.concat([transformed_df, test_df], ignore_index=True)\n",
    "    \n",
    "    return transformed_df\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cd96192",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('global_accuracies_5beams.pkl', 'rb') as file:\n",
    "    ga_5beams = pickle.load(file)\n",
    "    # Optionally convert back to defaultdict\n",
    "    # my_defaultdict = convert_to_defaultdict(loaded_dict)\n",
    "    \n",
    "with open('inter_group_accs_5beams.pkl', 'rb') as file:\n",
    "    ga_samplingModels_5beams_probs = pickle.load(file)\n",
    "    # Optionally convert back to defaultdict\n",
    "    # my_defaultdict = convert_to_defaultdict(loaded_dict)\n",
    "    \n",
    "\n",
    "# To load and optionally convert back to defaultdict\n",
    "# (You'll need to redefine your defaultdict structure as before)\n",
    "with open('localnglobal_accuracies_allgroups_allpeers.pkl', 'rb') as file:\n",
    "    ga_1beam = pickle.load(file)\n",
    "    # Optionally convert back to defaultdict\n",
    "    # my_defaultdict = convert_to_defaultdict(loaded_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb3368f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "    \n",
    "with open('inter_group_accs_1beams.pkl', 'rb') as file:\n",
    "    ga_samplingModels_1beam_probs = pickle.load(file)\n",
    "    # Optionally convert back to defaultdict\n",
    "    # my_defaultdict = convert_to_defaultdict(loaded_dict)\n",
    "\n",
    "       \n",
    "with open('accuracies_top1.pkl', 'rb') as file:\n",
    "    ga_samplingModels_1beam_probs_intragroup = pickle.load(file)\n",
    "    # Optionally convert back to defaultdict\n",
    "    # my_defaultdict = convert_to_defaultdict(loaded_dict)  \n",
    "    \n",
    "\n",
    "with open('accuracies_top5.pkl', 'rb') as file:\n",
    "    ga_samplingModels_5beams_probs_intragroup = pickle.load(file)\n",
    "    # Optionally convert back to defaultdict\n",
    "    # my_defaultdict = convert_to_defaultdict(loaded_dict)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37eec8e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "print (ga_samplingModels_5beams_probs_intragroup)\n",
    "print (ga_samplingModels_1beam_probs_intragroup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84fae76e",
   "metadata": {},
   "outputs": [],
   "source": [
    "ga_1beam = make_df_into_right_columns(ga_1beam)\n",
    "ga_5beams = make_df_into_right_columns(ga_5beams)\n",
    "\n",
    "\n",
    "ga_samplingModels_5beams_probs = make_df_into_right_columns(ga_samplingModels_5beams_probs, False)\n",
    "ga_samplingModels_1beam_probs = make_df_into_right_columns(ga_samplingModels_1beam_probs, False)\n",
    "\n",
    "\n",
    "ga_samplingModels_1beam_probs_intragroup = make_df_into_right_columns_secondmode(ga_samplingModels_1beam_probs_intragroup, 'probabilistic suggestions')\n",
    "ga_samplingModels_5beams_probs_intragroup = make_df_into_right_columns_secondmode(ga_samplingModels_5beams_probs_intragroup, 'probabilistic suggestions')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a4658b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalizing format to not rewrite code\n",
    "ga_samplingModels_1beam_probs['Level_2'] = 'acc_test'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0040d3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "print (\"ga_1beam\",ga_1beam.columns)\n",
    "print ('ga_5beams',ga_5beams.columns)\n",
    "print ('ga_samplingModels_5beams_probs',ga_samplingModels_5beams_probs.columns)\n",
    "print ('ga_samplingModels_1beam_probs',ga_samplingModels_1beam_probs.columns)\n",
    "print ('ga_samplingModels_1beam_probs_intragroup',ga_samplingModels_1beam_probs_intragroup.columns)\n",
    "print ('ga_samplingModels_5beams_probs_intragroup',ga_samplingModels_5beams_probs_intragroup.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30c9ca2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "ga_1beam = ga_1beam.rename(columns = \n",
    "                             {'Level_1':'data shard', 'Level_2': 'peer', \n",
    "                              'Level_3':'split', 'Level_4':'pers',\n",
    "                             'dataset_group': 'data shard'}\n",
    "                            )\n",
    "ga_5beams = ga_5beams.rename(columns = \n",
    "                             {'Level_1':'data shard', 'Level_2': 'peer', \n",
    "                              'Level_3':'split', 'Level_4':'pers',\n",
    "                             'dataset_group': 'data shard'}\n",
    "                            )\n",
    "\n",
    "ga_samplingModels_5beams_probs = ga_samplingModels_5beams_probs.rename(columns = \n",
    "                             {'Level_1':'data shard', 'Level_2': 'data split', \n",
    "                              'Value': 'probabilistic suggestions',\n",
    "                             'dataset_group': 'data shard'}\n",
    "                            )\n",
    "\n",
    "ga_samplingModels_1beam_probs = ga_samplingModels_1beam_probs.rename(columns = \n",
    "                             {'Level_1':'data shard', 'Level_2': 'data split', \n",
    "                              'Value': 'probabilistic suggestions'}\n",
    "                            )\n",
    "\n",
    "ga_samplingModels_1beam_probs_intragroup['beams'] = 1\n",
    "ga_samplingModels_5beams_probs_intragroup['beams'] = 5\n",
    "\n",
    "ga_samplingModels_1beam_probs_intragroup['beams'] = 1\n",
    "ga_samplingModels_5beams_probs_intragroup['beams'] = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e25c96fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "ga_1beam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c900c6b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "print (\"ga_1beam\",ga_1beam.columns)\n",
    "print ('ga_5beams',ga_5beams.columns)\n",
    "print ('ga_samplingModels_5beams_probs',ga_samplingModels_5beams_probs.columns)\n",
    "print ('ga_samplingModels_1beam_probs',ga_samplingModels_1beam_probs.columns)\n",
    "print ('ga_samplingModels_1beam_probs_intragroup',ga_samplingModels_1beam_probs_intragroup.columns)\n",
    "print ('ga_samplingModels_5beams_probs_intragroup',ga_samplingModels_5beams_probs_intragroup.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7e4112b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "ga_samplingModels_1beam_probs_intragroup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8668fe16",
   "metadata": {},
   "outputs": [],
   "source": [
    "ga_sm_5bms = ga_samplingModels_5beams_probs.drop(columns = 'data split')\n",
    "\n",
    "\n",
    "ga_sm_1bm_ig = ga_samplingModels_1beam_probs_intragroup.drop(columns = 'data split')\n",
    "\n",
    "\n",
    "\n",
    "ga_sm_5bms_ig = ga_samplingModels_5beams_probs_intragroup.drop(columns = 'data split')\n",
    "\n",
    "\n",
    "ga_sm_ig = pd.concat([ga_sm_1bm_ig,ga_sm_5bms_ig])\n",
    "# ga_sm_ig['nbr of shards'] = 3\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57e65ec6",
   "metadata": {},
   "outputs": [],
   "source": [
    "ga_sm_ig = ga_sm_ig[[\n",
    "    'data shard', 'beams', 'probabilistic suggestions']]\n",
    "# ga_sm_5bms[ga_sm_5bms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72b123b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "ga_sm_ig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36af0e1c",
   "metadata": {},
   "outputs": [],
   "source": [
    "ga_sm_1bm = ga_samplingModels_1beam_probs.drop(columns = 'data split')\n",
    "\n",
    "ga_sm_1bm[['probabilistic suggestions']] = ga_sm_1bm[['probabilistic suggestions']].astype(float)\n",
    "\n",
    "ga_sm_1bm['beams'] = 1\n",
    "ga_sm_5bms['beams'] = 5\n",
    "\n",
    "\n",
    "ga_sm_ig = ga_sm_ig.rename(\n",
    "    columns = {'probabilistic suggestions': 'ensemble accuracy', 'beams':'topk'})\n",
    "\n",
    "\n",
    "ga_sm_interg = pd.concat([ga_sm_1bm,ga_sm_5bms])\n",
    "ga_sm_interg = ga_sm_interg.rename(\n",
    "    columns = {'probabilistic suggestions': 'ensemble accuracy', 'beams':'topk'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39938503",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# ga_sm_interg['nbr of shards'] = ga_sm_interg['nbr of shards'].replace({'two': 2, 'three': 3})\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22f62716",
   "metadata": {},
   "source": [
    "Above we can see the intra-group ensemble, where peers were chosen from a pool belonging to the same shard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b459a87",
   "metadata": {},
   "outputs": [],
   "source": [
    "display(ga_sm_interg[['data shard','topk', 'ensemble accuracy']].sort_values(['data shard', 'topk']))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b90fd509",
   "metadata": {},
   "source": [
    "Above we can see inter-group ensemble, where peers were chosen from a pool belonging to different shards, 3 peers sampled from each of the three shards."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "926e0371",
   "metadata": {},
   "outputs": [],
   "source": [
    "ga_sm_ig['model pool'] = 'Only belonging to shard '\n",
    "ga_sm_interg['model pool'] = 'Belonging to all shards'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "131970cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "ga_sm_ig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00f545b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_total = pd.concat([ga_sm_interg, ga_sm_ig])\n",
    "\n",
    "df_total['data shard'] = df_total['data shard'].replace({'0': 'A', '1': 'B', '2': 'C'})\n",
    "\n",
    "ga_sm_ig['model pool'] = 'Only belonging to shard' + ga_sm_ig['data shard'].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "806c5f53",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_total = df_total.pivot_table(index=['data shard', 'model pool'], columns='topk', values='ensemble accuracy', aggfunc='first').reset_index()\n",
    "df_total = df_total.rename(columns = {'data shard': 'Shard', 1: 'top1', 5: 'top5'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4c955f2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_total"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d37db6d0",
   "metadata": {},
   "source": [
    "Thus, df_total represents the accuracies of top1 and top5 for the shards trained. It is equivalent to Table 3 from the paper."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}