Computing intra-group performance.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7262bf67",
   "metadata": {},
   "source": [
    "# The code in this jupyter notebook \n",
    "* reads the models and the datasets\n",
    "* implements the ensemble method and calculates its accuracy when the model pool is solely that of the shard's (i.e. no models can be chosen outside of the shard)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35fb0141",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "from collections import defaultdict\n",
    "from transformers import T5ForConditionalGeneration, T5Tokenizer, AdamW\n",
    "import pickle\n",
    "\n",
    "tokenizer = T5Tokenizer.from_pretrained('t5-small')\n",
    "\n",
    "# \n",
    "groups = [str(i) for i in range(0,3)]\n",
    "peers = [str(i) for i in range(8090, 8100)]\n",
    "epoch_cutoff = 6000\n",
    "number_of_models_sampled = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "786849ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_first_files_with_str(directory, str_contain, epoch_cutoff):\n",
    "    return os.path.join(str_contain + '_' + str(epoch_cutoff))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0865bbe2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# reading individual peer datasets & group datasets, e.g. train_df_group1, train_df_group1_peer1 etc.\n",
    "for group in groups:\n",
    "    # creating train_df's\n",
    "    exec(f'train_df_group{group} = pd.DataFrame()')\n",
    "    for peer in peers:\n",
    "        datasets_folder = os.path.join('aggregated_results',f'group{group}','datasets')\n",
    "        exec_str = f\"train_df_group{group}_peer{int(peer) - 8089} = pd.read_csv(os.path.join(datasets_folder,'{peer}_df.csv'))\"\n",
    "        exec(exec_str)\n",
    "        exec(f'train_df_group{group} = pd.concat([train_df_group{group}, train_df_group{group}_peer{int(peer) - 8089}])')\n",
    "       \n",
    "    exec(f'train_df_group{group} = train_df_group{group}.drop_duplicates()')\n",
    "    \n",
    "    # creating test_df's: test_df_group1, test_df_group1_peer1\n",
    "    \n",
    "    datasets_folder = os.path.join('aggregated_results',f'group{group}','datasets')\n",
    "    exec (f\"test_df_group{group} = pd.read_csv(os.path.join(datasets_folder,'test_df.csv')) \")\n",
    "    exec (f\"test_df_group{group} = test_df_group{group}[test_df_group{group}['doc_id'].isin(train_df_group{group}['doc_id'].unique())]\")\n",
    "    for peer in peers:\n",
    "        exec (f\"test_df_group{group}_peer{int(peer) - 8089} = test_df_group{group}[test_df_group{group}['doc_id'].isin(train_df_group{group}_peer{int(peer) - 8089}['doc_id'].unique())]\")\n",
    "    \n",
    "    \n",
    "    \n",
    "\n",
    "#reading models, e.g. model_group1_peer1\n",
    "for group in groups:\n",
    "    for peer in peers:\n",
    "        model_folder = os.path.join('aggregated_results',f'group{group}', 'models')\n",
    "        model_file = find_first_files_with_str(model_folder, peer, epoch_cutoff) # 10 is the largest number of saved models that all peers have finished training\n",
    "        print (group, peer, model_file)\n",
    "        exec_str = f\"model_group{group}_peer{str(int(peer)-8089)} = T5ForConditionalGeneration.from_pretrained(os.path.join(model_folder, model_file))\"\n",
    "        \n",
    "        \n",
    "        exec(exec_str)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a98d88a2",
   "metadata": {},
   "source": [
    "# Sampling random models and aggregating their suggestions - 5 beams, with probabilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13f25364",
   "metadata": {},
   "outputs": [],
   "source": [
    "models_group1_list = []\n",
    "models_group2_list = []\n",
    "models_group0_list = []\n",
    "\n",
    "for group in groups:\n",
    "    for i, peer in enumerate(peers):\n",
    "        exec(f'models_group{group}_list.append(model_group{group}_peer{int(peer)-8089})')\n",
    "       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5eaaf6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "from collections import defaultdict\n",
    "import threading\n",
    "from torch.nn.functional import softmax\n",
    "\n",
    "class ModelManager:\n",
    "    \"\"\"Class to hold the models, tokenizer and dataframes - including with results after their calculation\"\"\"\n",
    "    def __init__(self, model_list, train_df, test_df, tokenizer):\n",
    "        self.model_list = model_list\n",
    "        self.train_df = train_df.copy()\n",
    "        self.test_df = test_df.copy()\n",
    "        \n",
    "        print ('train set size:', self.train_df.shape[0])\n",
    "        print ('test set size:', self.test_df.shape[0])\n",
    "    \n",
    "        self.tokenizer = tokenizer\n",
    "        self.counter = 0\n",
    "\n",
    "    def generate_text_beams(self, query):\n",
    "        '''Generates the result for each query in the test set'''\n",
    "        self.counter += 1\n",
    "        if self.counter % 300 == 0:\n",
    "            print(f\"Processed {self.counter} queries\")\n",
    "        results = defaultdict(float)\n",
    "        \n",
    "        # sampling 3 models from the ones available\n",
    "        sampled_models = random.sample(self.model_list, number_of_models_sampled)\n",
    "\n",
    "        # Computing the ensemble \n",
    "        for model in sampled_models:\n",
    "            input_ids = self.tokenizer.encode(query, return_tensors='pt')\n",
    "            output = model.generate(input_ids, do_sample=False, return_dict_in_generate=True, output_scores=True,\n",
    "                                    num_beams=5, num_return_sequences=5, max_length = 20)\n",
    "\n",
    "            beam_scores = output.sequences_scores\n",
    "            probabilities = softmax(beam_scores, dim=0).tolist()\n",
    "            model_res = [self.tokenizer.decode(output_id, skip_special_tokens=True) for output_id in output.sequences]\n",
    "\n",
    "        \n",
    "            for res, prob in zip(model_res, probabilities):\n",
    "                results[res] += prob\n",
    "\n",
    "        \n",
    "        return self.top_5_generated_texts(results)\n",
    "\n",
    "    def top_5_generated_texts(self, input_dict):\n",
    "        '''Returns the 5 docids that the model is most confident about'''\n",
    "        sorted_responses = sorted(input_dict.items(), key=lambda x: x[1], reverse=True)\n",
    "        return [response[0] for response in sorted_responses[:5]]\n",
    "\n",
    "    def evaluate_accuracy(self):\n",
    "        self.train_df['generated_doc_id'] = self.train_df['query'].apply(self.generate_text_beams)\n",
    "        self.test_df['generated_doc_id'] = self.test_df['query'].apply(self.generate_text_beams)\n",
    "\n",
    "        acc_train = self.calculate_accuracy(self.train_df)\n",
    "        acc_test = self.calculate_accuracy(self.test_df)\n",
    "\n",
    "        return acc_train, acc_test\n",
    "\n",
    "    def calculate_accuracy(self, df):\n",
    "        return df.apply(lambda row: row['doc_id'] in row['generated_doc_id'], axis=1).sum() / df.shape[0]\n",
    "\n",
    "# Assuming you have predefined dictionaries/lists for models and datasets, such as:\n",
    "# two_groups_list = [...]\n",
    "# three_groups_list = [...]\n",
    "# train_df_group1 = ...\n",
    "# test_df_group1 = ...\n",
    "# ... and so on for other groups\n",
    "# And a tokenizer instance\n",
    "\n",
    "def run_evaluation(group_nbr, tokenizer):\n",
    "    \n",
    "    \n",
    "    model_list = globals()[f'models_group{group_nbr}_list']\n",
    "    train_df = globals()[f'train_df_group{group_nbr}']\n",
    "    test_df = globals()[f'test_df_group{group_nbr}']\n",
    "    \n",
    "    manager = ModelManager(model_list, train_df, test_df, tokenizer)\n",
    "    \n",
    "    key = group_nbr\n",
    "    model_managers[key] = manager\n",
    "    \n",
    "    \n",
    "    acc_train, acc_test = manager.evaluate_accuracy()\n",
    "    global_accuracies[key] = {'acc_train': acc_train, 'acc_test': acc_test}\n",
    "\n",
    "    print(f\"Group: {group_nbr}, Train Acc: {acc_train}, Test Acc: {acc_test}\")\n",
    "\n",
    "    \n",
    "# Global collection to store ModelManager instances\n",
    "model_managers = {}\n",
    "global_accuracies = {}\n",
    "    \n",
    "# Threading\n",
    "threads = []\n",
    "\n",
    "for group_nbr in groups:\n",
    "    thread = threading.Thread(target=run_evaluation, args=(group_nbr, tokenizer))\n",
    "    thread.start()\n",
    "    threads.append(thread)\n",
    "\n",
    "for thread in threads:\n",
    "    thread.join()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "329ce064",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def defaultdict_to_dict(d):\n",
    "    \"\"\" Recursively convert defaultdict to dict. \"\"\"\n",
    "    if isinstance(d, defaultdict):\n",
    "        d = {key: defaultdict_to_dict(value) for key, value in d.items()}\n",
    "    return d\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70580685",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming 'my_defaultdict' is your existing defaultdict\n",
    "# Convert it to a regular dictionary\n",
    "regular_dict_top5 = defaultdict_to_dict(global_accuracies)\n",
    "\n",
    "# Serialize and save to a file\n",
    "with open('accuracies_top5.pkl', 'wb') as file:\n",
    "    pickle.dump(regular_dict_top5, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e36e2d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('accuracies_top5.pkl', 'rb') as file:\n",
    "    loaded_dict = pickle.load(file)\n",
    "    # Optionally convert back to defaultdict\n",
    "    # my_defaultdict = convert_to_defaultdict(loaded_dict)\n",
    "    \n",
    "loaded_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0fa5afd",
   "metadata": {},
   "outputs": [],
   "source": [
    "regular_dict_top1 = {}\n",
    "for group in model_managers:\n",
    "    df_temp = model_managers[group].test_df.copy()\n",
    "    df_temp['generated_doc_id_1beam'] = df_temp['generated_doc_id'].apply(lambda x: [x[0]])\n",
    "    regular_dict_top1[group] = {'acc_test':df_temp.apply(lambda row: row['doc_id'] in row['generated_doc_id_1beam'], \n",
    "                                axis=1).sum() / df_temp.shape[0]}\n",
    "    print (group, df_temp.apply(lambda row: row['doc_id'] in row['generated_doc_id_1beam'], \n",
    "                                axis=1).sum() / df_temp.shape[0])\n",
    "    \n",
    "    \n",
    "# display(df_temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3f8d183",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('accuracies_top1.pkl', 'wb') as file:\n",
    "    pickle.dump(regular_dict_top1, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09770283",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('accuracies_top1.pkl', 'rb') as file:\n",
    "    loaded_dict = pickle.load(file)\n",
    "    # Optionally convert back to defaultdict\n",
    "    # my_defaultdict = convert_to_defaultdict(loaded_dict)\n",
    "    \n",
    "loaded_dict"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}