data_manipulation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import joblib\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datasets import Dataset, load_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_output(df1: pd.DataFrame, df2: pd.DataFrame) -> pd.DataFrame:\n",
    "    columns = [\"is_exact_match\", \"is_accurate\", \"f1_score\", \"answers\", \"new_answers\"]\n",
    "    df1 = df1[columns]\n",
    "    df2 = df2[columns]\n",
    "    concat_df = pd.concat([df1, df2], axis=0)\n",
    "    return concat_df\n",
    "\n",
    "def cal_merted_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   a  b\n",
       "0  1  3\n",
       "1  2  4\n",
       "0  5  7\n",
       "1  6  8"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = pd.DataFrame({\"a\":[1,2],\"b\":[3,4]})\n",
    "b = pd.DataFrame({\"a\":[5,6],\"b\":[7,8]})\n",
    "pd.concat([a,b], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1cbr_2miss\tanswerable\t0.433\t0.627\n",
      "1cbr_2miss\tunans_only\t0.393\t0.393\n",
      "1cbr_2miss\tadv_unans\t0.223\t0.223\n",
      "1cbr_2miss\tadv_only_unans\t0.2\t0.2\n",
      "1cbr_both\tanswerable\t0.44\t0.633\n",
      "1cbr_both\tunans_only\t0.44\t0.44\n",
      "1cbr_both\tadv_unans\t0.269\t0.269\n",
      "1cbr_both\tadv_only_unans\t0.2\t0.2\n",
      "2cbr_1miss\tanswerable\t0.46\t0.627\n",
      "2cbr_1miss\tunans_only\t0.333\t0.333\n",
      "2cbr_1miss\tadv_unans\t0.2\t0.2\n",
      "2cbr_1miss\tadv_only_unans\t0.15\t0.15\n",
      "3cbr\tanswerable\t0.453\t0.66\n",
      "3cbr\tunans_only\t0.12\t0.12\n",
      "3cbr\tadv_unans\t0.062\t0.062\n",
      "3cbr\tadv_only_unans\t0.0\t0.0\n",
      "base\tanswerable\t0.193\t0.48\n",
      "base\tunans_only\t0.207\t0.207\n",
      "base\tadv_unans\t0.115\t0.123\n",
      "base\tadv_only_unans\t0.05\t0.05\n"
     ]
    }
   ],
   "source": [
    "names = [\"1cbr_2miss.csv\",\"1cbr_both.csv\",\"2cbr_1miss.csv\",\"3cbr.csv\",\"base.csv\"]\n",
    "\"\"\"\n",
    "answerable        150\n",
    "unans_only        150\n",
    "adv_unans         130\n",
    "adv_only_unans     20\n",
    "\"\"\"\n",
    "datas = []\n",
    "for name in names:\n",
    "    df = pd.read_csv(name)\n",
    "    data = []\n",
    "    for status in df.status.unique():\n",
    "        df_status = df[df.status == status]\n",
    "        df_status_em = round(df_status.is_exact_match.mean(), 3)\n",
    "        df_status_acc = round(df_status.is_accurate.mean(), 3)\n",
    "        data.extend([df_status_em, df_status_acc])\n",
    "    datas.append(data)\n",
    "status = [\"answerable\", \"unans_only\", \"adv_unans\", \"adv_only_unans\"]\n",
    "metric = [\"em\", \"acc\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['answerable_em',\n",
       " 'answerable_acc',\n",
       " 'unans_only_em',\n",
       " 'unans_only_acc',\n",
       " 'adv_unans_em',\n",
       " 'adv_unans_acc',\n",
       " 'adv_only_unans_em',\n",
       " 'adv_only_unans_acc']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "status = [\"answerable\", \"unans_only\", \"adv_unans\", \"adv_only_unans\"]\n",
    "metric = [\"em\", \"acc\"]\n",
    "columns =[]\n",
    "for s in status:\n",
    "    for m in metric:\n",
    "        columns.append(f'{s}_{m}')\n",
    "columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(datas, columns=columns, index=names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>answerable_em</th>\n",
       "      <th>answerable_acc</th>\n",
       "      <th>unans_only_em</th>\n",
       "      <th>unans_only_acc</th>\n",
       "      <th>adv_unans_em</th>\n",
       "      <th>adv_unans_acc</th>\n",
       "      <th>adv_only_unans_em</th>\n",
       "      <th>adv_only_unans_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1cbr_2miss.csv</th>\n",
       "      <td>0.433</td>\n",
       "      <td>0.627</td>\n",
       "      <td>0.393</td>\n",
       "      <td>0.393</td>\n",
       "      <td>0.223</td>\n",
       "      <td>0.223</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1cbr_both.csv</th>\n",
       "      <td>0.440</td>\n",
       "      <td>0.633</td>\n",
       "      <td>0.440</td>\n",
       "      <td>0.440</td>\n",
       "      <td>0.269</td>\n",
       "      <td>0.269</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2cbr_1miss.csv</th>\n",
       "      <td>0.460</td>\n",
       "      <td>0.627</td>\n",
       "      <td>0.333</td>\n",
       "      <td>0.333</td>\n",
       "      <td>0.200</td>\n",
       "      <td>0.200</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3cbr.csv</th>\n",
       "      <td>0.453</td>\n",
       "      <td>0.660</td>\n",
       "      <td>0.120</td>\n",
       "      <td>0.120</td>\n",
       "      <td>0.062</td>\n",
       "      <td>0.062</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>base.csv</th>\n",
       "      <td>0.193</td>\n",
       "      <td>0.480</td>\n",
       "      <td>0.207</td>\n",
       "      <td>0.207</td>\n",
       "      <td>0.115</td>\n",
       "      <td>0.123</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                answerable_em  answerable_acc  unans_only_em  unans_only_acc  \\\n",
       "1cbr_2miss.csv          0.433           0.627          0.393           0.393   \n",
       "1cbr_both.csv           0.440           0.633          0.440           0.440   \n",
       "2cbr_1miss.csv          0.460           0.627          0.333           0.333   \n",
       "3cbr.csv                0.453           0.660          0.120           0.120   \n",
       "base.csv                0.193           0.480          0.207           0.207   \n",
       "\n",
       "                adv_unans_em  adv_unans_acc  adv_only_unans_em  \\\n",
       "1cbr_2miss.csv         0.223          0.223               0.20   \n",
       "1cbr_both.csv          0.269          0.269               0.20   \n",
       "2cbr_1miss.csv         0.200          0.200               0.15   \n",
       "3cbr.csv               0.062          0.062               0.00   \n",
       "base.csv               0.115          0.123               0.05   \n",
       "\n",
       "                adv_only_unans_acc  \n",
       "1cbr_2miss.csv                0.20  \n",
       "1cbr_both.csv                 0.20  \n",
       "2cbr_1miss.csv                0.15  \n",
       "3cbr.csv                      0.00  \n",
       "base.csv                      0.05  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>answerable_em</th>\n",
       "      <th>answerable_acc</th>\n",
       "      <th>unans_only_em</th>\n",
       "      <th>unans_only_acc</th>\n",
       "      <th>adv_unans_em</th>\n",
       "      <th>adv_unans_acc</th>\n",
       "      <th>adv_only_unans_em</th>\n",
       "      <th>adv_only_unans_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>tqa_1cbr_2miss.csv</th>\n",
       "      <td>0.747</td>\n",
       "      <td>0.880</td>\n",
       "      <td>0.453</td>\n",
       "      <td>0.453</td>\n",
       "      <td>0.200</td>\n",
       "      <td>0.200</td>\n",
       "      <td>0.073</td>\n",
       "      <td>0.073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tqa_1cbr_both.csv</th>\n",
       "      <td>0.760</td>\n",
       "      <td>0.900</td>\n",
       "      <td>0.427</td>\n",
       "      <td>0.427</td>\n",
       "      <td>0.137</td>\n",
       "      <td>0.137</td>\n",
       "      <td>0.091</td>\n",
       "      <td>0.091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tqa_2cbr_1miss.csv</th>\n",
       "      <td>0.733</td>\n",
       "      <td>0.880</td>\n",
       "      <td>0.400</td>\n",
       "      <td>0.400</td>\n",
       "      <td>0.105</td>\n",
       "      <td>0.105</td>\n",
       "      <td>0.073</td>\n",
       "      <td>0.073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tqa_3cbr.csv</th>\n",
       "      <td>0.787</td>\n",
       "      <td>0.947</td>\n",
       "      <td>0.133</td>\n",
       "      <td>0.140</td>\n",
       "      <td>0.042</td>\n",
       "      <td>0.042</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tqa_base.csv</th>\n",
       "      <td>0.733</td>\n",
       "      <td>0.920</td>\n",
       "      <td>0.353</td>\n",
       "      <td>0.353</td>\n",
       "      <td>0.095</td>\n",
       "      <td>0.095</td>\n",
       "      <td>0.018</td>\n",
       "      <td>0.018</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    answerable_em  answerable_acc  unans_only_em  \\\n",
       "tqa_1cbr_2miss.csv          0.747           0.880          0.453   \n",
       "tqa_1cbr_both.csv           0.760           0.900          0.427   \n",
       "tqa_2cbr_1miss.csv          0.733           0.880          0.400   \n",
       "tqa_3cbr.csv                0.787           0.947          0.133   \n",
       "tqa_base.csv                0.733           0.920          0.353   \n",
       "\n",
       "                    unans_only_acc  adv_unans_em  adv_unans_acc  \\\n",
       "tqa_1cbr_2miss.csv           0.453         0.200          0.200   \n",
       "tqa_1cbr_both.csv            0.427         0.137          0.137   \n",
       "tqa_2cbr_1miss.csv           0.400         0.105          0.105   \n",
       "tqa_3cbr.csv                 0.140         0.042          0.042   \n",
       "tqa_base.csv                 0.353         0.095          0.095   \n",
       "\n",
       "                    adv_only_unans_em  adv_only_unans_acc  \n",
       "tqa_1cbr_2miss.csv              0.073               0.073  \n",
       "tqa_1cbr_both.csv               0.091               0.091  \n",
       "tqa_2cbr_1miss.csv              0.073               0.073  \n",
       "tqa_3cbr.csv                    0.000               0.000  \n",
       "tqa_base.csv                    0.018               0.018  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names = [\"tqa_1cbr_2miss.csv\",\"tqa_1cbr_both.csv\",\"tqa_2cbr_1miss.csv\",\"tqa_3cbr.csv\",\"tqa_base.csv\"]\n",
    "\"\"\"\n",
    "answerable        150\n",
    "unans_only        150\n",
    "adv_unans         130\n",
    "adv_only_unans     20\n",
    "\"\"\"\n",
    "datas = []\n",
    "for name in names:\n",
    "    df = pd.read_csv(name)\n",
    "    data = []\n",
    "    for status in df.status.unique():\n",
    "        df_status = df[df.status == status]\n",
    "        df_status_em = round(df_status.is_exact_match.mean(), 3)\n",
    "        df_status_acc = round(df_status.is_accurate.mean(), 3)\n",
    "        data.extend([df_status_em, df_status_acc])\n",
    "    datas.append(data)\n",
    "status = [\"answerable\", \"unans_only\", \"adv_unans\", \"adv_only_unans\"]\n",
    "metric = [\"em\", \"acc\"]\n",
    "status = [\"answerable\", \"unans_only\", \"adv_unans\", \"adv_only_unans\"]\n",
    "metric = [\"em\", \"acc\"]\n",
    "columns =[]\n",
    "for s in status:\n",
    "    for m in metric:\n",
    "        columns.append(f'{s}_{m}')\n",
    "pd.DataFrame(datas, columns=columns, index=[n for n in names])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "exp",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}