quality_consistency.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42df5d32-21dd-434e-a858-1a7755e0c09f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import precision_recall_fscore_support"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38e444c0-cf38-4930-9b2f-e86a60f671ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_decisions_df = pd.read_csv(\"results/all_decisions_df.csv\")\n",
    "all_decisions_df.fillna({\"model\": \"\"}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4007e8ad-55bc-4c56-88a6-141bd0887776",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_df = all_decisions_df.groupby(\n",
    "    [\"model\", \"dataset\", \"task_scope\", \"experiment_run\"]\n",
    ").apply(\n",
    "    lambda group: pd.Series(\n",
    "        precision_recall_fscore_support(\n",
    "            group[\"benchmark\"],\n",
    "            group[\"decision\"] == \"yes\",\n",
    "            average=\"binary\",\n",
    "            pos_label=True,\n",
    "            zero_division=0.0\n",
    "        ),\n",
    "        index=[\"precision\", \"recall\", \"f1-score\", \"support\"]\n",
    "    ),\n",
    "    include_groups=False,\n",
    ").reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a09ecba9-36f7-49db-94d7-4c78d0145ae6",
   "metadata": {},
   "outputs": [],
   "source": [
    "scores_std = scores_df.query(\"task_scope != 'n-gram'\").groupby([\"model\", \"task_scope\", \"dataset\"])[[\"f1-score\", \"precision\", \"recall\"]].std()\n",
    "consistency_table = pd.pivot(\n",
    "    scores_std.groupby([\"model\", \"task_scope\"]).mean().reset_index(),\n",
    "    index=\"task_scope\",\n",
    "    columns=\"model\",\n",
    "    values=[\"f1-score\", \"precision\", \"recall\"],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0ae43d3-1d89-4ab8-a58a-aaf53d5276b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import plotly.graph_objects as go\n",
    "\n",
    "texts = [\n",
    "    [\n",
    "        (\n",
    "            f\"{consistency_table.loc[task_scope, (\"f1-score\", model)]:.3f} \"\n",
    "            f\"({consistency_table.loc[task_scope, ('precision', model)]:.2f}, \"\n",
    "            f\"{consistency_table.loc[task_scope, ('recall', model)]:.2f})\"\n",
    "        )\n",
    "        for model in [\"GPT-3.5\", \"GPT-4\"]\n",
    "    ]\n",
    "    for task_scope in [\"1-to-1\", \"1-to-n\", \"n-to-1\", \"n-to-n\"]\n",
    "]\n",
    "fig = go.Figure(\n",
    "    data=go.Heatmap(\n",
    "        x=[\"GPT-3.5\", \"GPT-4\"],\n",
    "        y=[\"1-to-1\", \"1-to-n\", \"n-to-1\", \"n-to-n\"],\n",
    "        z=consistency_table.loc[:, (\"f1-score\", slice(None))],\n",
    "        text=texts,\n",
    "        texttemplate=\"%{text}\",\n",
    "        textfont={\"size\": 12},\n",
    "        colorscale=\"greens_r\",\n",
    "        zmin=0,\n",
    "        zmax=consistency_table.loc[:, (\"f1-score\", slice(None))].max().max(),\n",
    "        showscale=False,\n",
    "    ),\n",
    "    layout=dict(\n",
    "        title=\"Standard deviation of F1-score, precision and recall.\",\n",
    "        height=600,\n",
    "        width=1000,\n",
    "        yaxis={\"autorange\": \"reversed\"}\n",
    "    ),\n",
    ")\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be8a189b-e49b-45db-957d-7ef186a03006",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}