Evaluations-Full-with-GPT4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bc0c58c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "sns.set_theme()\n",
    "\n",
    "from run_experiment import *"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "467e8a39",
   "metadata": {},
   "source": [
    "!pip3 install --user -U scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e1155ab3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def UAR(gt, pred):\n",
    "    gt = np.array(gt)\n",
    "    pred = np.array(pred)\n",
    "    acc = []\n",
    "    for x in np.unique(gt):\n",
    "        _m = gt == x\n",
    "        acc.append((gt[_m] == pred[_m]).mean())\n",
    "    return np.mean(acc) * 100\n",
    "\n",
    "def ACC(gt, pred):\n",
    "    gt = np.array(gt)\n",
    "    pred = np.array(pred)\n",
    "    return (gt == pred).mean() * 100\n",
    "\n",
    "\n",
    "def significance_test(metric_fn, y_true, y_pred_a, y_pred_b, tries=1<<12, random_state=41, gt2=None):\n",
    "    assert y_true.shape[0] == y_pred_a.shape[0] == y_pred_b.shape[0]\n",
    "\n",
    "    metric_a = metric_fn(y_true, y_pred_a)\n",
    "    metric_b = metric_fn(y_true, y_pred_b)\n",
    "    benchmark = np.abs(metric_a - metric_b)  # observed difference\n",
    "\n",
    "    if random_state is not None:\n",
    "        np.random.seed(random_state)  # fixing seed for reproducability\n",
    "    samples = np.zeros(tries)  # the distribution of differences\n",
    "    for i in range(tries):\n",
    "        msk = np.random.random(size=y_true.shape[0]) < 0.5\n",
    "\n",
    "        y_pred_a_perm = y_pred_a.copy()\n",
    "        y_pred_b_perm = y_pred_b.copy()\n",
    "\n",
    "        y_pred_a_perm[msk] = y_pred_b[msk]\n",
    "        y_pred_b_perm[msk] = y_pred_a[msk]\n",
    "\n",
    "        samples[i] = np.abs(\n",
    "            metric_fn(y_true, y_pred_a_perm) - metric_fn(y_true, y_pred_b_perm)\n",
    "        )\n",
    "    pvalue = 1 - np.mean(benchmark > samples)\n",
    "    sig = sum([int(pvalue < level) for level in [0.05,  0.01]])\n",
    "    if sig == 0:\n",
    "        sig = '     '\n",
    "    elif sig == 1:\n",
    "        sig = \"^*   \"\n",
    "    elif sig == 2:\n",
    "        sig = \"^{**}\"\n",
    "    return metric_a, metric_b, sig\n",
    "    #return \"$ %7.2f^%s $\" % (metric_a, sig), (\"$ %7.2f $\" % metric_b)\n",
    "    #return metric_a, metric_b, pvalue#, benchmark, samples.mean(), samples.std()\n",
    "    \n",
    "def build_row(tab, gt, bow, roberta, gpt3, gpt4):\n",
    "    parts = []\n",
    "    acc_bow, acc_chatgpt3, p_acc_bow = significance_test(ACC, gt, bow, gpt3)\n",
    "    acc_roberta, acc_chatgpt3, p_acc_roberta = significance_test(ACC, gt, roberta, gpt3)\n",
    "    acc_gpt4, acc_chatgpt3, p_acc_gpt4 = significance_test(ACC, gt, gpt4, gpt3)\n",
    "\n",
    "    uar_bow, uar_chatgpt3, p_uar_bow = significance_test(UAR, gt, bow, gpt3)\n",
    "    uar_roberta, uar_chatgpt3, p_uar_roberta = significance_test(UAR, gt, roberta, gpt3)\n",
    "    uar_gpt4, uar_chatgpt3, p_uar_gpt4 = significance_test(UAR, gt, gpt4, gpt3)\n",
    "    \n",
    "    mxacc = np.max([acc_bow, acc_roberta, acc_gpt4, acc_chatgpt3])\n",
    "    mxuar = np.max([uar_bow, uar_roberta, uar_gpt4, uar_chatgpt3])\n",
    "    print(\"%s\\nExamples %d\\n  BoW  :  %.4f   %.4f\\nRoBERTa:  %.4f   %.4f\\n GPT3  :  %.4f   %.4f\\n GPT4  :  %.4f   %.4f\\n\\n\" % (\n",
    "        tab, len(gt), acc_bow, uar_bow, acc_roberta, uar_roberta, acc_chatgpt3, uar_chatgpt3, acc_gpt4, uar_gpt4\n",
    "    ))\n",
    "    accs = [\n",
    "        \"$\\\\textbf{%5.2f}%s $\" % (acc, p) if np.isclose(mxacc, acc) else \"$        %5.2f%s $\" % (acc, p)  \n",
    "        for acc, p in zip([acc_chatgpt3, acc_bow, acc_roberta, acc_gpt4], ['  ', p_acc_bow, p_acc_roberta, p_acc_gpt4 ])\n",
    "    ]\n",
    "    \n",
    "    uars = [\n",
    "        \"$\\\\textbf{%5.2f}%s $\" % (uar, p) if np.isclose(mxuar, uar) else \"$        %5.2f%s $\" % (uar, p)  \n",
    "        for uar, p in zip([uar_chatgpt3, uar_bow, uar_roberta, uar_gpt4], ['  ', p_uar_bow, p_uar_roberta, p_uar_gpt4 ])\n",
    "    ]\n",
    "\n",
    "    return \"&   %s  \\\\\\\\\" % \"  &  \".join([tab.replace(\"_\", \" \").capitalize()] + accs + uars)\n",
    "#         acc_chatgpt3, acc_bow, acc_roberta, acc_gpt4,\n",
    "#         uar_chatgpt3, uar_bow, uar_roberta, uar_gpt4,\n",
    "#     ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d14c9faf",
   "metadata": {},
   "outputs": [],
   "source": [
    "ROOT = \"/outputs/chatgpt_evaluations/iteration_3/results\"\n",
    "\n",
    "def load(path):\n",
    "    with open(path.replace(\"results/\", f\"{ROOT}/\")) as fl:\n",
    "        return json.load(fl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "453e967b",
   "metadata": {},
   "outputs": [],
   "source": [
    "latex_table = []"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c54d1b41",
   "metadata": {},
   "source": [
    "# Engagement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "89fc7473",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkUAAAG5CAYAAACAxkA+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA5QUlEQVR4nO3deXxU1f3/8fcsCWsmhBbhK4sktAlbMEEkBEJUECQRzdcqCi3ggghWkNCfLRSBQqXE+pCGGpAWDKigoIDVtkZEAaEgZVHEWlHRhNUvS1kyIQQyy/39gRlJZyaEZJKZJK/n48Ejzj1nznyGmzu8vefcOybDMAwBAAA0cOZgFwAAABAKCEUAAAAiFAEAAEgiFAEAAEgiFAEAAEgiFAEAAEgiFAEAAEgiFAEAAEgiFAEAAEiSrMEuoK4xDENud+BvAm42m2pkXFQf+yY0sV9CF/smNDXk/WI2m2Qyma7Yj1B0ldxuQ6dPFwd0TKvVrKioZrLbz8vpdAd0bFQP+yY0sV9CF/smNDX0/dKyZTNZLFcORUyfAQAAiFAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgSbIGuwAACDST1aJSh8tnW3iYRYbTdxuAho1QBKDeKXW4lL3yY59tk0f0VJiplgsCUCeE1PTZ5s2bNXLkSPXp00fdu3fXwIEDlZWVpaKionL9Nm7cqDvvvFPx8fG67bbbtHbtWq+xSktL9fvf/179+vVTQkKCHnzwQeXn59fWWwEAAHVMSIWis2fPqkePHpo9e7Zyc3P14IMP6s0339SkSZM8fXbv3q0JEyYoISFBS5YsUVpamp588kmtW7eu3Fhz5szR6tWrNXnyZOXk5Ki0tFQPPPCAV8ACAACQQmz6LCMjo9zjpKQkhYeHa8aMGTp+/Lhat26tRYsWqUePHvrtb38rSerTp48OHz6s5557TkOGDJEkHTt2TGvWrNFvfvMb3XPPPZKk+Ph43XLLLVq1apXGjh1bu28MAACEvJA6U+RLixYtJEkOh0OlpaXasWOHJ/yUSU9P1zfffKMjR45IkrZu3Sq3212uX4sWLdSvXz9t2bKl1moHAAB1R0idKSrjcrnkdDr19ddfa+HChRowYIDatWunr7/+Wg6HQzExMeX6d+rUSZKUn5+vdu3aKT8/Xz/4wQ8UGRnp1W/NmjXVrs9qDWyWtFjM5X4idLBvQtOV9ovTachk8r2a2mQyyWplpXVN4ZgJTeyXygnJUHTLLbfo+PHjkqT+/ftr3rx5kqTCwkJJks1mK9e/7HFZu91uV0REhNe4NpvN06eqzGaToqKaVWsMf2y2JjUyLqqPfROa/O2XE6fPy2q1+GyzWMyKimpak2VBHDOhiv1SsZAMRYsXL1ZJSYm+/vprLVq0SOPHj9eyZcuCXZYkye02ZLefD+iYFotZNlsT2e0lcrncAR0b1cO+CU1X2i8ulyGnn3sRuVxunTlTXNMlNlgcM6Gpoe8Xm61Jpc6ShWQo6ty5syQpMTFR8fHxysjI0Hvvvacf/ehHkuR1BZndbpckz3SZzWbTuXPnvMa12+1eU2pV4XTWzC+Uy+WusbFRPeyb0ORvvxiGZBiGz+cYhiGn03cbAodjJjSxXyoW8pOLcXFxCgsL06FDh9ShQweFhYV53W+o7HHZWqOYmBj95z//8Zoqy8/P91qPBAAAINWBULR37145HA61a9dO4eHhSkpK0rvvvluuT15enjp16qR27dpJklJSUmQ2m7V+/XpPn8LCQm3dulWpqam1Wj8AAKgbQmr6bMKECerevbvi4uLUuHFjffHFF8rNzVVcXJxuvfVWSdKjjz6q0aNHa9asWUpLS9OOHTv097//XdnZ2Z5x2rRpo3vuuUfPPPOMzGazWrdurT//+c+KiIjQ8OHDg/X2AARQUXGpLjgN+Zwl83PlGQBUJKRCUY8ePZSXl6fFixfLMAy1bdtWw4YN05gxYxQeHi5J6tWrl3JycjR//nytWbNG1157rebMmaO0tLRyY02fPl3NmjXTvHnzVFxcrJ49e2rZsmU+r0oDUPeUXHQqe+XHPtcOZY7oGYSKANR1JsPfakT45HK5dfp0YK9csVrNiopqpjNnilkAF2LYN6HJajXLYZj07IrdfkPRfL4QNig4ZkJTQ98vLVs2q9TVZyG/pggAAKA2EIoAAABEKAIAAJBEKAIAAJBEKAIAAJBEKAIAAJBEKAIAAJBEKAIAAJBEKAIAAJBEKAIAAJBEKAIAAJBEKAIAAJAkWYNdAAD4Y7JaVOpweW13Og2JL3UFEGCEIgAhq9ThUraPb7s3mUyaNKJnECoCUJ8xfQYAACBCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCTJGuwCLvfOO+/or3/9q/7973/Lbrfruuuu06hRo3T33XfLZDJJkkaNGqWdO3d6PTcvL0+dOnXyPC4qKlJWVpbef/99ORwO9e/fX9OnT9c111xTa+8HAADUHSEVil588UW1bdtWU6dOVVRUlD788EPNmDFDx44d04QJEzz9evbsqSlTppR7brt27co9zszM1Ndff61Zs2apUaNGmj9/vsaOHau1a9fKag2ptw0AAEJASKWDRYsWqWXLlp7HycnJOnv2rJYtW6af//znMpsvzfbZbDYlJCT4HWfPnj3aunWrcnNzlZKSIkmKjo5Wenq61q9fr/T09Bp9HwAAoO4JqTVFlweiMl26dNG5c+d0/vz5So+zZcsW2Ww29evXz7MtJiZGXbp00ZYtWwJSKwAAqF9CKhT58tFHH6l169Zq3ry5Z9vOnTuVkJCg+Ph4jRw5Urt27Sr3nPz8fEVHR3vWIZWJiYlRfn5+rdQNAADqlpCaPvtvu3fvVl5eXrn1QzfeeKMyMjLUsWNHnThxQrm5uXrwwQe1fPlyJSYmSpLsdrsiIiK8xouMjNRnn31W7bqs1sBmSYvFXO4nQgf7JricTsPrf24kSabvf5rko13y/bzvtlutvttQfRwzoYn9UjkhG4qOHTumyZMnKykpSaNHj/Zsf/zxx8v1u/nmmzV06FA9//zzWrJkSY3XZTabFBXVrEbGttma1Mi4qD72TXCcOH1eVqvFb7vV4qfNZPL7PIvFrKiopoEoDxXgmAlN7JeKhWQostvtGjt2rFq0aKGcnBzPAmtfmjZtqptuuknvvvuuZ5vNZtOxY8e8+hYWFioyMrJatbndhuz2yq9vqgyLxSybrYns9hK5XO6Ajo3qYd8El8tlyOl0eTd8d6LH6XJJho8nGn6eJ8nlcuvMmeLAFYlyOGZCU0PfLzZbk0qdJQu5UHThwgWNGzdORUVFeu2113xOg11JTEyMtm/fLsMof+q9oKBAsbGx1a7R6ayZXyiXy11jY6N62Dc1y2S1qNThK/yYZBjeqcczZWbIZ7tU8Xan03cbAodjJjSxXyoWUqHI6XQqMzNT+fn5euWVV9S6desrPuf8+fP64IMPFB8f79mWmpqq559/Xtu3b1ffvn0lXQpEn3/+uR5++OEaqx9A1ZQ6XMpe+bHX9swRPYNQDYCGKqRC0ezZs7Vp0yZNnTpV586d0yeffOJp69q1qz799FO98MILGjRokNq2basTJ05o2bJlOnnypP74xz96+iYmJiolJUXTpk3TlClT1KhRI2VnZysuLk6DBw8OwjsDAAChLqRC0bZt2yRJTz/9tFfbhg0b1KpVKzkcDmVnZ+vs2bNq0qSJEhMTNXv2bPXo0aNc//nz5ysrK0szZ86U0+lUSkqKpk+fzt2sAQCATyGVEDZu3HjFPrm5uZUaKyIiQnPnztXcuXOrWxYAAGgAuGEBAACAQuxMEQDUtLAwixy+rnSTFB5mkeHnUn4A9R+hCECDUupwab6PK90kafKIngrjZtdAg8X0GQAAgAhFAAAAkghFAAAAkghFAAAAkghFAAAAkghFAAAAkghFAAAAkrhPEQBUislqUamPmz5yw0eg/iAUAUAllDpcyvZx00du+AjUH0yfAQAAiFAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiS+EBQCPsDCLHA4/33hv4ltfgfqOUAQA3yl1uDR/5cc+2zJH9KzlagDUNqbPAAAARCgCAACQRCgCAACQxJoiAAFkslpU6mehcqNwqy6WOv08kUXMAIKPUAQgYEodLmVXsFCZRcwAQhnTZwAAACIUAQAASCIUAQAASCIUAQAASCIUAQAASCIUAQAASCIUAQAASCIUAQAASCIUAQAASCIUAQAASCIUAQAASCIUAQAASCIUAQAASCIUAQAASAqxUPTOO+/o0UcfVWpqqhISEpSRkaE1a9bIMIxy/VavXq3bbrtN8fHxuvPOO7Vp0yavsYqKijRt2jT17t1biYmJevzxx3XixInaeisAAKCOCalQ9OKLL6pJkyaaOnWqFi1apNTUVM2YMUMLFy709Hn77bc1Y8YMpaWlacmSJUpISNCECRP0ySeflBsrMzNT27Zt06xZs/Tss8+qoKBAY8eOldPprOV3BQAA6gJrsAu43KJFi9SyZUvP4+TkZJ09e1bLli3Tz3/+c5nNZj333HO6/fbblZmZKUnq06ePvvrqKy1cuFBLliyRJO3Zs0dbt25Vbm6uUlJSJEnR0dFKT0/X+vXrlZ6eXuvvDagvTFaLSh0uP42m2i0GAAIopELR5YGoTJcuXfT666/r/PnzOnPmjA4cOKBf/vKX5fqkp6frmWeeUWlpqcLDw7VlyxbZbDb169fP0ycmJkZdunTRli1bCEVANZQ6XMpe+bHPtswRPWu5GgAInJAKRb589NFHat26tZo3b66PPvpI0qWzPpfr1KmTHA6HDh8+rE6dOik/P1/R0dEy/df/tcbExCg/P7/aNVmtgZ11tFjM5X4idLBvvDmdhtexdblAt/ncbvr+p0lX8bwaaDOZTLJaOUNWhmMmNLFfKiekQ9Hu3buVl5enKVOmSJIKCwslSTabrVy/ssdl7Xa7XREREV7jRUZG6rPPPqtWTWazSVFRzao1hj82W5MaGRfVx7753onT52W1Wnw3mkyBbavoOZKsllqqo4I2i8WsqKimfmtsqDhmQhP7pWIhG4qOHTumyZMnKykpSaNHjw52OR5utyG7/XxAx7RYzLLZmshuL5HL5Q7o2Kge9o03l8uQ0+lnTZER4DZ/2787MeN0uSTDuzngdVTQ5nK5deZMse/nNEAcM6Gpoe8Xm61Jpc6ShWQostvtGjt2rFq0aKGcnByZzZfeSGRkpKRLl9u3atWqXP/L2202m44dO+Y1bmFhoadPdTidNfML5XK5a2xsVA/75nuGIa/bZJRvD2ybr+2eKbMKaqmtGg3DkNPp/zkNFcdMaGK/VCzkJhcvXLigcePGqaioSC+88EK5abCYmBhJ8loXlJ+fr7CwMLVv397Tr6CgwOsDrKCgwDMGAADA5UIqFDmdTmVmZio/P18vvPCCWrduXa69ffv26tixo9atW1due15enpKTkxUeHi5JSk1NVWFhobZv3+7pU1BQoM8//1ypqak1/0YAAECdE1LTZ7Nnz9amTZs0depUnTt3rtwNGbt27arw8HBNnDhRTzzxhDp06KCkpCTl5eXp008/1YoVKzx9ExMTlZKSomnTpmnKlClq1KiRsrOzFRcXp8GDBwfhnQEAgFAXUqFo27ZtkqSnn37aq23Dhg1q166dhg4dqpKSEi1ZskSLFy9WdHS0FixYoMTExHL958+fr6ysLM2cOVNOp1MpKSmaPn26rNaQessAACBEhFRC2LhxY6X6DRs2TMOGDauwT0REhObOnau5c+cGojQAAFDPhdSaIgAAgGAhFAEAAIhQBAAAIIlQBAAAIIlQBAAAIIlQBAAAIIlQBAAAIIlQBAAAIIlQBAAAIIlQBAAAIIlQBAAAIIlQBAAAIIlQBAAAIIlQBAAAIIlQBAAAIKkaoWj06NHavn273/Z//vOfGj16dFWHBwAAqFVVDkU7d+7Uf/7zH7/tp0+f1q5du6o6PAAAQK2q1vSZyWTy23bw4EE1a9asOsMDAADUGuvVdP7LX/6iv/zlL57HixYt0uuvv+7Vr6ioSF9++aVSU1OrXyEAAEAtuKpQVFJSojNnzngeFxcXy2z2PtnUtGlTDR8+XI899lj1KwQAAKgFVxWKfvrTn+qnP/2pJGnAgAF68sknNXDgwBopDAAAoDZdVSi63MaNGwNZBwAAQFBVORSVOXfunL799lvZ7XYZhuHVfuONN1b3JQAAAGpclUPR6dOnNWfOHK1fv14ul8ur3TAMmUwm7du3r1oFAgAA1IYqh6KZM2dq06ZNGjVqlHr16iWbzRbIugAAAGpVlUPRtm3bdP/99+tXv/pVIOsBAAAIiirfvLFx48Zq27ZtIGsBAAAImiqHojvvvFPvv/9+IGsBAAAImipPn912223atWuXxowZo/vuu09t2rSRxWLx6tetW7dqFQgAAFAbqhyKym7iKEkffvihVztXnwEAgLqkyqEoKysrkHUAAAAEVZVD0V133RXIOgCEEJPVolKH9/3HLjWaarcYAKgl1b6jNYD6p9ThUvbKj322ZY7oWcvVAEDtqHIo+vWvf33FPiaTSXPnzq3qSwAAANSaKoeiHTt2eG1zu906efKkXC6XWrZsqSZNmlSrOAAAgNpS5VC0ceNGn9sdDodee+01vfTSS1q6dGmVCwMAAKhNVb55oz9hYWEaOXKk+vXrp6eeeirQwwNASAkLs8hhyOcfk9X73m0AQleNLbTu3Lmz3nrrrZoaHgBCQqnDpfl+FqVPHtFTYVysB9QZAT9TVObDDz9kTREAAKgzqnymaMGCBT63FxUVadeuXfr888/1yCOPVLkwAACA2hTwUBQZGan27dtr9uzZuvfee6tcGAAAQG2qcij64osvAlkHAABAUNXYmiIAAIC6pNpXn+3cuVMffPCBvv32W0nStddeq5tvvlm9e/eudnEAAAC1pcqhqLS0VP/v//0/vf/++zIMQzabTZJkt9u1bNkyDRo0SPPmzVNYWFilxzx48KByc3O1d+9e7d+/XzExMfr73/9ers+oUaO0c+dOr+fm5eWpU6dOnsdFRUXKysrS+++/L4fDof79+2v69Om65pprqviOAQBAfVblULRw4UK99957euihh/TQQw/phz/8oSTp1KlTWrp0qXJzc7Vw4UJlZmZWesz9+/dr8+bNuv766+V2u2UYhs9+PXv21JQpU8pta9euXbnHmZmZ+vrrrzVr1iw1atRI8+fP19ixY7V27VpZrXwPLgAAKK/K6eBvf/ub7rrrLv3qV78qt/0HP/iBfvnLX+rUqVP661//elWhaMCAAbr11lslSVOnTtVnn33ms5/NZlNCQoLfcfbs2aOtW7cqNzdXKSkpkqTo6Gilp6dr/fr1Sk9Pr3RNAACgYajyQuuTJ0+qR48eftt79OihkydPXl0x5sCs+96yZYtsNpv69evn2RYTE6MuXbpoy5YtAXkNAABQv1Q5hbRp08bn2p4yu3btUps2bao6fIV27typhIQExcfHa+TIkdq1a1e59vz8fEVHR8tkKn9//ZiYGOXn59dITQAAoG6r8vTZ//7v/yonJ0cRERF64IEHdN1118lkMunAgQN66aWXtG7dOk2cODGQtUqSbrzxRmVkZKhjx446ceKEcnNz9eCDD2r58uVKTEyUdGmxd0REhNdzIyMj/U7JXQ2rNbB3MrBYzOV+InQ01H3jdBpe/1Nxudps87nd9P1Pk67iebVZ43fbrdaG9eVnDfWYCXXsl8qpcigaP368Dh8+rNdff12rV6/2TH2VLZC+6667NH78+IAVWubxxx8v9/jmm2/W0KFD9fzzz2vJkiUBf73/ZjabFBXVrEbGttn4rrhQ1dD2zYnT52X19w3vJlPttVX0HElWS2jXaLGYFRXV1Pd49VxDO2bqCvZLxaociiwWi55++mk98MAD2rJli44ePSpJatu2rVJTU9W5c+eAFVmRpk2b6qabbtK7777r2Waz2XTs2DGvvoWFhYqMjKzW67ndhuz289Ua479ZLGbZbE1kt5fI5XIHdGxUT0PdNy6XIafT5bvRqMU2f9u/O/nidLkkXxephkKNklwut86cKfY9Xj3VUI+ZUNfQ94vN1qRSZ8muKhRdvHhRv/vd7/TjH/9Yo0aNkiR17tzZKwC9/PLLWrVqlZ588smruk9RoMTExGj79u0yjPJTAAUFBYqNja32+E5nzfxCuVzuGhsb1dPQ9o1hyO8tMS61116br+2eKbMK6gx2jWXbnU7/49VnDe2YqSvYLxW7qsnF1157TX/5y1908803V9jv5ptv1tq1a7V69erq1FYp58+f1wcffKD4+HjPttTUVBUWFmr79u2ebQUFBfr888+Vmppa4zUBAIC656rOFL3zzjsaPHiw2rdvX2G/Dh06aMiQIXr77bf105/+tNLjl5SUaPPmzZKko0eP6ty5c1q3bp0kqXfv3srPz9cLL7ygQYMGqW3btjpx4oSWLVumkydP6o9//KNnnMTERKWkpGjatGmaMmWKGjVqpOzsbMXFxWnw4MFX85YBAEADcVWh6KuvvtIdd9xRqb6JiYnatGnTVRVz6tQpTZo0qdy2sscvv/yy2rRpI4fDoezsbJ09e1ZNmjRRYmKiZs+e7XXPpPnz5ysrK0szZ86U0+lUSkqKpk+fzt2sAQCAT1eVEBwOR6XXCIWFham0tPSqimnXrp2+/PLLCvvk5uZWaqyIiAjNnTtXc+fOvaoaAABAw3RVa4quueYa7d+/v1J99+/fz5evAgCAOuOqQlHfvn311ltv6dSpUxX2O3XqlN566y317du3WsUBAADUlqsKRWPHjtXFixd1//33a+/evT777N27Vw888IAuXryohx9+OCBFAgAA1LSrWlPUvn17zZ8/X7/4xS80fPhwtW/fXrGxsWrWrJmKi4u1f/9+HTp0SI0bN9Yf/vAHdejQoabqBgAACKirvhTr5ptv1l//+lctWbJEH3zwgd5//31P2zXXXKNhw4Zp7NixV7xsHwAAIJRU6fr0du3aafbs2ZKkc+fOqbi4WM2aNVPz5s0DWhwAAEBtqfZNe5o3b04YAgAAdd5VLbQGAACorwhFAAAAIhQBAABIIhQBAABIIhQBAABICsDVZwiMouJSXXAaMgzvtvAwiwynq/aLAgCgASEUhYiSi05lr/xYho9UNHlET4WZglAUAAANCNNnAAAAIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIkqzBLgBAcJisFpU6XH4aTbVbDACEAEIR0ECVOlzKXvmxz7bMET1ruRoACD6mzwAAAEQoAgAAkEQoAgAAkBRioejgwYOaOXOmMjIy1LVrVw0dOtRnv9WrV+u2225TfHy87rzzTm3atMmrT1FRkaZNm6bevXsrMTFRjz/+uE6cOFHTbwEAANRRIRWK9u/fr82bN+u6665Tp06dfPZ5++23NWPGDKWlpWnJkiVKSEjQhAkT9Mknn5Trl5mZqW3btmnWrFl69tlnVVBQoLFjx8rpdNbCOwEAAHVNSF19NmDAAN16662SpKlTp+qzzz7z6vPcc8/p9ttvV2ZmpiSpT58++uqrr7Rw4UItWbJEkrRnzx5t3bpVubm5SklJkSRFR0crPT1d69evV3p6eu28IQAAUGeE1Jkis7nicg4fPqwDBw4oLS2t3Pb09HRt375dpaWlkqQtW7bIZrOpX79+nj4xMTHq0qWLtmzZEvjCgRBmslrkMOT1h3sRAUB5IXWm6Ery8/MlXTrrc7lOnTrJ4XDo8OHD6tSpk/Lz8xUdHS3Tf33ox8TEeMaoDqs1sFnSYjHL4TQkk2SS9z9UJpNJViv/gAWDxWIu97MuuuB0a/6qPV7bJw1P9DpGLhcqbT63m77/6euYCYka1TCP3fpwzNRH7JfKqVOhqLCwUJJks9nKbS97XNZut9sVERHh9fzIyEifU3JXw2w2KSqqWbXG8OXC6fOyWiw+2ywWs6Kimgb8NVF5NluTYJdQZSdOn5fV6uN3y2TyvT2U2ip6juT3mAmVGhvysVuXj5n6jP1SsToVikKB223Ibj8f0DHLkrvT5ZIM73aXy60zZ4oD+pqoHIvFLJutiez2Erlc7mCXUyUulyGn08fXeRh+todSm7/t35188XfMhESNapjHbn04Zuqjhr5fbLYmlTpLVqdCUWRkpKRLl9u3atXKs91ut5drt9lsOnbsmNfzCwsLPX2qw+msiV8ok2RIhuH9CW8YhpxOX5/8qC0ul7uG9nvNM/z8Xl1q8/97FSptvrZ7psxC5L1VtL2hHrt1+Zipz9gvFatTk4sxMTGS5LUuKD8/X2FhYWrfvr2nX0FBgdcHVUFBgWcMAACAy9WpUNS+fXt17NhR69atK7c9Ly9PycnJCg8PlySlpqaqsLBQ27dv9/QpKCjQ559/rtTU1FqtGUDDFRbm+8o/h3HpqkAAoSWkps9KSkq0efNmSdLRo0d17tw5TwDq3bu3WrZsqYkTJ+qJJ55Qhw4dlJSUpLy8PH366adasWKFZ5zExESlpKRo2rRpmjJliho1aqTs7GzFxcVp8ODBQXlvABqeUodL81d+7LNt8oieCmtYF6YBIS+kQtGpU6c0adKkctvKHr/88stKSkrS0KFDVVJSoiVLlmjx4sWKjo7WggULlJiYWO558+fPV1ZWlmbOnCmn06mUlBRNnz5dVmtIvWUAABAiQiohtGvXTl9++eUV+w0bNkzDhg2rsE9ERITmzp2ruXPnBqo8AABQj9WpNUUAAAA1hVAEAAAgQhEAAICkEFtTBKBqTFaLSh1+7sTMF78CQKUQioB6oNThUrafS78zR/Ss5WoAoG5i+gwAAECEIgAAAEmEIgAAAEmEIgAAAEkstAaAoAgLs8jh54rB8DCLDKefqwkB1BhCEQAEAV8WC4Qeps8AAABEKAIAAJBEKAIAAJBEKAIAAJDEQmugzuD7zQCgZhGKgDqC7zcDgJrF9BkAAIAIRQAAAJIIRQAAAJIIRQAAAJIIRQAAAJIIRQAAAJIIRQAAAJK4TxEAhJywMIscfm7UGR5mkeH0cxNPANVCKAKAEFPqcGm+nxt1Th7RU2HcwByoEUyfAQAAiFAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiVAEAAAgiZs3AiHFZLWo1M+djGXijn0AUJMIRUAIKXW4lO3nTsaZI3rWcjUA0LAwfQYAACBCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgCRCEQAAgKQ6GIreeOMNxcXFef159tlny/VbvXq1brvtNsXHx+vOO+/Upk2bglQxAACoC+rsHa1feOEFRUREeB63bt3a899vv/22ZsyYofHjx6tPnz7Ky8vThAkT9MorryghISEI1QIAgFBXZ0NRt27d1LJlS59tzz33nG6//XZlZmZKkvr06aOvvvpKCxcu1JIlS2qxSgAAUFfUuemzKzl8+LAOHDigtLS0ctvT09O1fft2lZaWBqkyAAAQyupsKBo6dKi6dOmigQMH6s9//rNcrkvfLJ6fny9Jio6OLte/U6dOcjgcOnz4cK3XCgAAQl+dmz5r1aqVJk6cqOuvv14mk0kbN27U/Pnzdfz4cc2cOVOFhYWSJJvNVu55ZY/L2qvDag1slrRYzHI4DckkmWTyajeZTLJavbej5lks5nI/A8EwmXXR4fLdaLq0v/0JdFttvlZV23xuN33/09cxExI11uBrhfLnQU0cM6g+9kvl1LlQ1L9/f/Xv39/zOCUlRY0aNdJLL72k8ePH1/jrm80mRUU1C/i4F06fl9Vi8dlmsZgVFdU04K+JyrPZmgRsrBOnzytn9V6fbY8Nu15Wq+/fA5lMgW0L9Hi1XaPk95gJmRproK2ufB4E8phB4LBfKlbnQpEvaWlpWrp0qfbt26fIyEhJUlFRkVq1auXpY7fbJcnTXlVutyG7/Xy1xvhvZcnd6XJJhne7y+XWmTPFAX1NVI7FYpbN1kR2e4lcLndAxnS5DDmdfs4UGbXYVpuvFegavztR4u+YCYkaa6gt1D8PauKYQfU19P1iszWp1FmyehGKLhcTEyPp0tqisv8uexwWFqb27dtX+zWczpr4hTJJhmQY3p/whmHI6fT1yY/a4nK5A7bfDT/7+fv22msLlToqavO13TNlVsHfZbBrrMnXqgufB4E8ZhA47JeK1YvJxby8PFksFnXt2lXt27dXx44dtW7dOq8+ycnJCg8PD1KVAAAglNW5M0VjxoxRUlKS4uLiJEkbNmzQ66+/rtGjR3umyyZOnKgnnnhCHTp0UFJSkvLy8vTpp59qxYoVwSwdAACEsDoXiqKjo7V27VodO3ZMbrdbHTt21LRp0zRq1ChPn6FDh6qkpERLlizR4sWLFR0drQULFigxMTGIlQMAgFBW50LR9OnTK9Vv2LBhGjZsWA1XAwAA6ot6saYIAACguurcmSIAaMjCwixy+Ln5Z3iYRYa/y/wBXBGhCADqkFKHS/NXfuyzbfKIngoL3ZtdAyGPUAQA9YS/s0icQQIqh1AEAPWEv7NInEECKoeF1gAAACIUAQAASCIUAQAASCIUAQAASCIUAQAASOLqM6BGmKwWlfq5wZ5MXAYEAKGIUARUg9/w43Qr288N9jJH9KzhqgAAVUEoAqqh1OHyGX4IPgBQ97CmCAAAQIQiAAAASYQiAAAASYQiAAAASSy0brD8XTXFt2kDABoqQlED5e+qKb5NGwDQUDF9BgAAIEIRAACAJEIRAACAJEIRAACAJEIRAACAJEIRAACAJEIRAACAJO5TBFxRUXGpLjgNGYaPRhM3dQKA+oJQBFxByUWnsld+LMNHKsoc0TMIFQEAagKhCADqubAwixw+vtZH4qt9gMsRigCgnit1uDTfx9f6SHy1D3A5FloDAACIUAQAACCJ6TMEiMlqUSlrFgAAdRihCAFR6nApmzULAIA6jOkzAAAAcaYIDUhFU3yNwq26WOr02u50GhJnuQCgQSAUocGoaIovc0RPn5csm0wmTeIGjQDQIDB9BgAAIEIRAACAJEIRAACAJNYUAUCDxveiAd8jFAFAA8b3ogHfIxShXqnosnuZ+HQHAPhHKEK9cqXL7gFUXkVTaxXd26uouLSmSwNqBKEIIcvfWR9/H8aXnsTZICBQKppaq+jeXk+M7MW0G+qkeh2KvvnmG82ZM0d79uxRs2bNlJGRoczMTIWHhwe7NFSCv7M+/j6My9oAAKiKehuKCgsLdf/996tjx47KycnR8ePH9fTTT+vChQuaOXNmsMvDd1gDBAAIFfU2FK1atUrFxcVasGCBWrRoIUlyuVyaPXu2xo0bp9atWwe3QEhiDRBQH5nNJl1wuGUY3m3+pr+5/B+hoN6Goi1btig5OdkTiCQpLS1Nv/nNb7Rt2zb95Cc/CV5xAFCPXXS49MeVH8vwkYr8TX9z+T9Cgcnw9VtbDyQnJ+vuu+/WE088UW57//79lZGR4bW9sgzDkNsd2L8yk0kyDOnsuYs+2yObNwr4F7Ubkgp9vF5VX8vfeFca80rP81fj1T6ntttCpQ5qpMZQqqOithbNG8nfJ2t9yErBfm8mk2Q2m+V2+z6DV9tq++/DbDbJVIklGfU2FHXr1k2TJk3SI488Um770KFDlZiYqKeeeipIlQEAgFDEd58BAACoHocim82moqIir+2FhYWKjIwMQkUAACCU1dtQFBMTo/z8/HLbioqKdPLkScXExASpKgAAEKrqbShKTU3Vhx9+KLvd7tm2bt06mc1m9evXL4iVAQCAUFRvF1oXFhbq9ttvV3R0tMaNG+e5eeMdd9zBzRsBAICXehuKpEtf8/HUU0+V+5qPyZMn8zUfAADAS70ORQAAAJVVb9cUAQAAXA1CEQAAgAhFAAAAkghFAAAAkghFAAAAkghFAAAAkghFQfXNN9/owQcfVEJCgvr166dnnnlGpaWlwS4Lkg4ePKiZM2cqIyNDXbt21dChQ4NdEiS98847evTRR5WamqqEhARlZGRozZo14s4iwbV582aNHDlSffr0Uffu3TVw4EBlZWX5/P5JBE9xcbFSU1MVFxenf/3rX8EuJyRZg11AQ1VYWKj7779fHTt2VE5OjueO2xcuXOCO2yFg//792rx5s66//nq53W7+0Q0RL774otq2baupU6cqKipKH374oWbMmKFjx45pwoQJwS6vwTp79qx69OihUaNGqUWLFtq/f79ycnK0f/9+LV26NNjl4TvPP/+8XC5XsMsIaYSiIFm1apWKi4u1YMECtWjRQpLkcrk0e/ZsjRs3Tq1btw5ugQ3cgAEDdOutt0qSpk6dqs8++yzIFUGSFi1apJYtW3oeJycn6+zZs1q2bJl+/vOfy2zm5HcwZGRklHuclJSk8PBwzZgxQ8ePH+fzLAR88803evXVVzVlyhT95je/CXY5IYtPkCDZsmWLkpOTPYFIktLS0uR2u7Vt27bgFQZJ4h/XEHV5ICrTpUsXnTt3TufPnw9CRfCn7LPN4XAEtxBIkubMmaPhw4crOjo62KWEND75gyQ/P18xMTHlttlsNrVq1Ur5+flBqgqoez766CO1bt1azZs3D3YpDZ7L5dLFixf173//WwsXLtSAAQPUrl27YJfV4K1bt05fffWVHnvssWCXEvKYPgsSu90um83mtT0yMlKFhYVBqAioe3bv3q28vDxNmTIl2KVA0i233KLjx49Lkvr376958+YFuSKUlJTo6aef1uTJk/kfh0rgTBGAOunYsWOaPHmykpKSNHr06GCXA0mLFy/WqlWrNGfOHOXn52v8+PEs7A2yRYsW6Qc/+IHuvvvuYJdSJ3CmKEhsNpvPy1ULCwsVGRkZhIqAusNut2vs2LFq0aKFcnJyWAMWIjp37ixJSkxMVHx8vDIyMvTee+9pyJAhQa6sYTp69KiWLl2qhQsXev69KVt7d/78eRUXF6tZs2bBLDHkEIqCJCYmxmvtUFFRkU6ePOm11gjA9y5cuKBx48apqKhIr732miIiIoJdEnyIi4tTWFiYDh06FOxSGqwjR47I4XDokUce8WobPXq0rr/+er3++utBqCx0EYqCJDU1VX/605/KrS1at26dzGaz+vXrF+TqgNDkdDqVmZmp/Px8vfLKK1zqHcL27t0rh8PBQusg6tKli15++eVy2/bt26esrCzNnj1b8fHxQaosdBGKgmT48OFavny5HnvsMY0bN07Hjx/XM888o+HDh/NBHwJKSkq0efNmSZdOQZ87d07r1q2TJPXu3dvnpeGoebNnz9amTZs0depUnTt3Tp988omnrWvXrgoPDw9ecQ3YhAkT1L17d8XFxalx48b64osvlJubq7i4OM/9vlD7bDabkpKSfLZ169ZN3bp1q+WKQp/J4Fa9QfPNN9/oqaee0p49e9SsWTNlZGRo8uTJfLCHgCNHjmjgwIE+215++WW/HzSoWQMGDNDRo0d9tm3YsIGzEkGyePFi5eXl6dChQzIMQ23bttWgQYM0ZswYrngKMTt27NDo0aO1Zs0azhT5QCgCAAAQl+QDAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBAABIIhQBCIA33nhDcXFxOnLkSLBLAYAqIxQBCHlHjhxRXFyc50/nzp3Vu3dvPfzww9qzZ0+VxiwpKVFOTo527NgR4GoD5+OPP1ZOTo7sdnuwSwEaBEIRgDpj6NCheuaZZ5SVlaURI0Zo7969Gj16tL788surHqukpEQLFizQzp07a6DSwNizZ48WLFhAKAJqCV8IC6DO6Nq1qzIyMjyPb7jhBo0dO1YrV67UrFmzglcYgHqBM0UAasQrr7yi22+/Xd27d1dKSopmz57t84zHK6+8ooEDB6pHjx665557tHv3bo0aNUqjRo264mv06tVLknT48OFy2+12u373u9/ppptuUvfu3TVo0CAtXrxYbrdb0qXpuOTkZEnSggULPNNyOTk52rBhg+Li4vTFF194xnv33XcVFxenCRMmlHudtLQ0ZWZmltv21ltv6Sc/+Yl69Oih3r17a/Lkyfq///s/r9r37t2rMWPG6IYbbtD111+vkSNH6qOPPvK05+Tk6JlnnpEkDRw40FNj2bqtbdu2acSIEerVq5cSExN122236Q9/+MMV/84A+MeZIgABl5OTowULFqhv374aMWKECgoKtHLlSv3rX//SypUrFRYWJkl69dVX9dvf/la9evXSAw88oKNHj+qxxx6TzWZTmzZtrvg6R48elSTZbDbPtpKSEo0cOVLHjx/X8OHD9T//8z/as2eP/vCHP+jkyZN68skn1bJlS82aNUuzZs3SoEGDNGjQIElSXFyc2rRpI5PJpN27d6tz586SpN27d8tsNpcLLadPn1Z+fr5Gjhzp2bZo0SL98Y9/VFpamu655x6dPn1aK1as0M9+9jO9+eabnjq3b9+usWPHqnv37powYYJMJpPeeOMN3X///Xr11VfVo0cPDRo0SAcOHNDf//53/frXv1ZUVJQkqWXLltq/f7/GjRunuLg4Pf744woPD9fBgwf18ccfV2e3ATAAoJrWrl1rxMbGGocPHzZOnTpldOvWzXjooYcMl8vl6bNixQojNjbWWLNmjWEYhnHx4kWjd+/ext133204HA5PvzfeeMOIjY01Ro4c6dl2+PBhIzY21sjJyTFOnTplnDx50ti1a5dx9913G7GxscY777zj6btw4UIjISHBKCgoKFfjs88+a3Tp0sX49ttvDcMwjFOnThmxsbHGc8895/V+br/9dmPSpEmex3fddZfx+OOPG7GxscbXX39tGIZhrF+/3oiNjTX27dtnGIZhHDlyxOjSpYuxaNGicmN9+eWXRteuXT3b3W63MXjwYOOhhx4y3G63p19JSYkxYMAA48EHH/Rse+GFFzx/r5dbtmyZERsba5w6dcqrdgBVx/QZgID68MMP5XA4NHr0aJnN33/EDBs2TM2bN9fmzZslSZ999pnOnj2re++9V1br9yet77jjDkVGRvocOycnR8nJyerXr59+9rOf6ZtvvtHUqVM1ZMgQT59169bphhtukM1m0+nTpz1/+vbtK5fLpV27dl3xPdxwww3avXu3JOncuXP64osvdN999ykqKspztmj37t2y2WyKjY2VJL333ntyu91KS0sr97o//OEPdd1113muctu3b58OHDigO+64Q2fOnPH0O3/+vJKTk7Vr1y7PNJ8/ZWecNmzYcMW+ACqP6TMAAfXtt99KkmJiYsptDw8PV/v27T1TXmX9OnToUK6f1WpV27ZtfY593333aciQIbp48aL++c9/avny5XK5XOX6HDx4UF9++aVnzdB/O3369BXfQ69evbRq1SodPHhQhw4dkslkUkJCgnr16qXdu3fr3nvv1e7du9WzZ09P8Dtw4IAMw9DgwYN9jlkW/A4cOCBJmjJlit/XLyoq8hsMJSk9PV2rV6/W9OnTNW/ePCUnJ2vQoEEaMmRIuSAK4OoQigDUGdddd5369u0rSbrllltkNps1b948JSUlKT4+XpLkdrvVr18/Pfzwwz7H6Nix4xVf54YbbpAk7dq1S4cPH1bXrl3VtGlT9erVSy+//LKKi4u1b9++cous3W63TCaTlixZIovF4jVm06ZNJUmGYUiSfvWrX6lLly4+X7+srz+NGzfWK6+8oh07duiDDz7QP/7xD+Xl5em1117T0qVLfb4+gCsjFAEIqGuvvVaSlJ+fr/bt23u2l5aW6siRI55QU9bv0KFD6tOnj6ef0+nU0aNHFRcXd8XXevTRR7V69WrNnz9fubm5ki6deTp//rzndfwxmUwVvodrr71WH330kQ4fPuy5yq1Xr17KysrSunXr5HK5dOONN3qe06FDBxmGoXbt2ik6Otrv2GV/J82bN69WjWazWcnJyUpOTtavf/1r/elPf1J2drZ27NhxxXEB+MZ5VgAB1bdvX4WFhWn58uWesyKStGbNGhUVFemmm26SJHXv3l0tWrTQ66+/LqfT6en3t7/9TYWFhZV6LZvNpvvuu09bt27Vvn37JF26TH7Pnj36xz/+4dXfbrd7XqtJkyaebb7ccMMN+uc//6lPP/3Uc+aoS5cuatasmRYvXqzGjRurW7dunv6DBw+WxWLRggULyr1v6dLZoTNnznjed4cOHbR06VIVFxd7ve7l03tlNRYVFZXrc/bsWa/nlZ11Ki0t9fl+AFwZZ4oABFTLli01btw4LViwQA8//LAGDBiggoICvfrqq4qPj9edd94p6dIao4kTJ+qpp57S/fffr7S0NB09elRvvPGG1zqjiowePVovvfSSFi9erOzsbI0ZM0YbN27U+PHjddddd6lbt24qKSnRV199pXfffVcbNmxQy5Yt1bhxY/3oRz/SO++8o44dO6pFixb68Y9/7Fk43atXL/3tb3+TyWTyhCKLxaLExERt3bpVvXv3Vnh4uKeODh06KDMzU/PmzdPRo0d16623qlmzZjpy5Ijef/993XvvvRozZozMZrPmzJmjsWPHaujQofrJT36i1q1b6/jx49qxY4eaN2+uP/3pT5LkCV3Z2dlKT09XWFiYbrnlFi1cuFC7d+/WTTfdpLZt2+rUqVN69dVX1aZNG0+tAK4eoQhAwE2cOFEtW7bUihUrlJWVpcjISN177736xS9+4blHkSSNHDlShmFo2bJl+v3vf6/OnTtr0aJFmjNnjho1alSp12rdurXuuOMOvfXWWzp06JA6dOig5cuX689//rPWrVunN998U82bN1fHjh01ceJERUREeJ47Z84cPfXUU8rKypLD4dCECRPKhSLp0oLxsnsElW3funWrp/1yjzzyiDp27KgXX3xRCxculCS1adNG/fr104ABAzz9kpKS9Nprr+n555/XihUrdP78ebVq1Uo9evTQfffd5+nXo0cPTZo0SatWrdI//vEPud1ubdiwQQMGDNDRo0e1du1anTlzRlFRUerdu7fX+wNwdUzGf5/nBYAgcrvdnqup5syZE+xyADQgrCkCEDQXL170Wn/z5ptv6uzZs+rdu3eQqgLQUDF9BiBoPvnkE2VlZWnIkCFq0aKFPv/8c61Zs0axsbHlbsgIALWBUAQgaNq2bas2bdpo+fLlKiwsVGRkpDIyMvTEE0+UW8QMALWBNUUAAABiTREAAIAkQhEAAIAkQhEAAIAkQhEAAIAkQhEAAIAkQhEAAIAkQhEAAIAkQhEAAIAkQhEAAIAk6f8DVK/UGWbET+AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def engagement_plot():\n",
    "    df = pd.read_csv(\"datasets/engagement measurement/TEDTalks.csv\")\n",
    "    df = df.sample(frac=1, random_state=41).reset_index()\n",
    "    df['logRetweets'] = np.log10(df['retweetCount'] + 1)\n",
    "    df = df[:4000]\n",
    "    return sns.histplot(df['logRetweets'])\n",
    "\n",
    "engagement_plot();"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e430fef4",
   "metadata": {},
   "source": [
    "### Pair comparisons"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c4870996",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_results(fil, *labels, df=None):\n",
    "    if fil is not None:\n",
    "        df = pd.read_csv(os.path.join(\"chatgpt-responses\", fil), index_col=0)\n",
    "    #print(df.columns)#, np.unique(df.ChatGPT))\n",
    "    shp = df.shape\n",
    "    assert len(labels) == 2\n",
    "    _all = None\n",
    "    for col in [\"GPT3\", \"GPT4\"]:\n",
    "        __mskor = None\n",
    "        for vals in labels:\n",
    "            for a in vals:\n",
    "                msk = (df[col] == a) | (df[col] == f\"{a}.\") | (df[col] == a.lower()) | (df[col] == f\"{a}.\".lower())\n",
    "                if __mskor is None:\n",
    "                    __mskor = msk\n",
    "                else:\n",
    "                    __mskor |= msk\n",
    "        if _all is None:\n",
    "            _all = __mskor\n",
    "        else:\n",
    "            _all &= __mskor\n",
    "    df = df[_all]\n",
    "    \n",
    "    preds = []\n",
    "    for col in [\"GPT3\", \"GPT4\"]:\n",
    "        __mskor = None\n",
    "        for a in labels[0]:\n",
    "            msk = (df[col] == a) | (df[col] == f\"{a}.\") | (df[col] == a.lower()) | (df[col] == f\"{a}.\".lower())\n",
    "            if __mskor is None:\n",
    "                __mskor = msk\n",
    "            else:\n",
    "                __mskor |= msk\n",
    "        preds.append(__mskor.astype(np.int32))\n",
    "\n",
    "    return (df, shp[0]) + tuple(preds)\n",
    "#     return (df[msks_or], shp[0]) + tuple(msks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "aaa13c39",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Engagement\n",
      "Examples 11996\n",
      "  BoW  :  71.0153   71.0163\n",
      "RoBERTa:  79.1847   79.1856\n",
      " GPT3  :  51.9173   51.8458\n",
      " GPT4  :  54.1514   53.7978\n",
      "\n",
      "\n",
      "&   Engagement  &  $        51.92   $  &  $        71.02^{**} $  &  $\\textbf{79.18}^{**} $  &  $        54.15^{**} $  &  $        51.85   $  &  $        71.02^{**} $  &  $\\textbf{79.19}^{**} $  &  $        53.80^{**} $  \\\\\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idx_a</th>\n",
       "      <th>idx_b</th>\n",
       "      <th>content_a</th>\n",
       "      <th>content_b</th>\n",
       "      <th>retweetCount_a</th>\n",
       "      <th>retweetCount_b</th>\n",
       "      <th>GPT3</th>\n",
       "      <th>GPT4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3408</td>\n",
       "      <td>3406</td>\n",
       "      <td>This brain researcher studied her own stroke ....</td>\n",
       "      <td>@D4arkness2016 There are four! https://t.co/pq...</td>\n",
       "      <td>79</td>\n",
       "      <td>2</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1963</td>\n",
       "      <td>1965</td>\n",
       "      <td>3 types of motivation that drive creative work...</td>\n",
       "      <td>Today's #TED: Ben Saunders: Why bother leaving...</td>\n",
       "      <td>321</td>\n",
       "      <td>47</td>\n",
       "      <td>B</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3667</td>\n",
       "      <td>3916</td>\n",
       "      <td>@wickdchiq We hear your concern. She spoke at ...</td>\n",
       "      <td>@Maxim54619878 The totally average human who r...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>A</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1392</td>\n",
       "      <td>1892</td>\n",
       "      <td>12 stupendous talks for word-lovers: http://t....</td>\n",
       "      <td>\"Human beings are works in progress that mista...</td>\n",
       "      <td>212</td>\n",
       "      <td>188</td>\n",
       "      <td>A</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>332</td>\n",
       "      <td>2478</td>\n",
       "      <td>\"The biggest challenges and problems we face w...</td>\n",
       "      <td>\"The brain abhors a vacuum.\" How our memories ...</td>\n",
       "      <td>321</td>\n",
       "      <td>100</td>\n",
       "      <td>A</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11991</th>\n",
       "      <td>2096</td>\n",
       "      <td>2993</td>\n",
       "      <td>\"As a child, I knew that stories were meant to...</td>\n",
       "      <td>@haleyvandyck Your talk was incredible today! ...</td>\n",
       "      <td>53</td>\n",
       "      <td>4</td>\n",
       "      <td>B</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11992</th>\n",
       "      <td>1625</td>\n",
       "      <td>1623</td>\n",
       "      <td>TED believes that Black lives matter and that ...</td>\n",
       "      <td>A gut-wrenchingly honest look at living with d...</td>\n",
       "      <td>374</td>\n",
       "      <td>215</td>\n",
       "      <td>A</td>\n",
       "      <td>A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11993</th>\n",
       "      <td>1680</td>\n",
       "      <td>249</td>\n",
       "      <td>\"Creativity is as important in education as li...</td>\n",
       "      <td>We’re LIVE on #TEDConnects with Dr. Esther Cho...</td>\n",
       "      <td>769</td>\n",
       "      <td>12</td>\n",
       "      <td>B</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11994</th>\n",
       "      <td>1974</td>\n",
       "      <td>166</td>\n",
       "      <td>This talk might leave you questioning the natu...</td>\n",
       "      <td>Why aren't you moving up at work? The career a...</td>\n",
       "      <td>189</td>\n",
       "      <td>48</td>\n",
       "      <td>B</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11995</th>\n",
       "      <td>1323</td>\n",
       "      <td>3012</td>\n",
       "      <td>\"When I was little, I thought my country was t...</td>\n",
       "      <td>7 TED Talks on how we're fighting cancer: http...</td>\n",
       "      <td>51</td>\n",
       "      <td>83</td>\n",
       "      <td>B</td>\n",
       "      <td>B</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11996 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       idx_a  idx_b                                          content_a  \\\n",
       "0       3408   3406  This brain researcher studied her own stroke ....   \n",
       "1       1963   1965  3 types of motivation that drive creative work...   \n",
       "2       3667   3916  @wickdchiq We hear your concern. She spoke at ...   \n",
       "3       1392   1892  12 stupendous talks for word-lovers: http://t....   \n",
       "4        332   2478  \"The biggest challenges and problems we face w...   \n",
       "...      ...    ...                                                ...   \n",
       "11991   2096   2993  \"As a child, I knew that stories were meant to...   \n",
       "11992   1625   1623  TED believes that Black lives matter and that ...   \n",
       "11993   1680    249  \"Creativity is as important in education as li...   \n",
       "11994   1974    166  This talk might leave you questioning the natu...   \n",
       "11995   1323   3012  \"When I was little, I thought my country was t...   \n",
       "\n",
       "                                               content_b  retweetCount_a  \\\n",
       "0      @D4arkness2016 There are four! https://t.co/pq...              79   \n",
       "1      Today's #TED: Ben Saunders: Why bother leaving...             321   \n",
       "2      @Maxim54619878 The totally average human who r...               0   \n",
       "3      \"Human beings are works in progress that mista...             212   \n",
       "4      \"The brain abhors a vacuum.\" How our memories ...             321   \n",
       "...                                                  ...             ...   \n",
       "11991  @haleyvandyck Your talk was incredible today! ...              53   \n",
       "11992  A gut-wrenchingly honest look at living with d...             374   \n",
       "11993  We’re LIVE on #TEDConnects with Dr. Esther Cho...             769   \n",
       "11994  Why aren't you moving up at work? The career a...             189   \n",
       "11995  7 TED Talks on how we're fighting cancer: http...              51   \n",
       "\n",
       "       retweetCount_b GPT3 GPT4  \n",
       "0                   2    A    A  \n",
       "1                  47    B    A  \n",
       "2                   0    A    B  \n",
       "3                 188    A    B  \n",
       "4                 100    A    B  \n",
       "...               ...  ...  ...  \n",
       "11991               4    B    A  \n",
       "11992             215    A    A  \n",
       "11993              12    B    B  \n",
       "11994              48    B    B  \n",
       "11995              83    B    B  \n",
       "\n",
       "[11996 rows x 8 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def engagement_fn():\n",
    "    df = pd.read_csv(\"datasets/engagement measurement/TEDTalks.csv\")\n",
    "    df = df.sample(frac=1, random_state=41).reset_index()\n",
    "    df['logRetweets'] = np.log10(df['retweetCount'] + 1)\n",
    "    df = df[:4000]\n",
    "    \n",
    "    test_df, shp, gpt3_cmp, gpt4_cmp = read_results('engagement_pairs.csv', [\"A\"], [\"B\"])\n",
    "    true = np.array(load(f\"results/outputs_engagement_RoBERTa.json\"))\n",
    "    df['RoBERTa'] = np.array(load(f\"results/predictions_engagement_RoBERTa.json\"))[:, 0]\n",
    "    df['BoW'] = np.array(load(f\"results/predictions_engagement_BoW.json\"))[:, 0]\n",
    "    assert np.mean(np.abs(df[\"logRetweets\"].values[test_df.idx_a.values] - np.log10(test_df['retweetCount_a'].values + 1))) < 1e-8\n",
    "    assert np.mean(np.abs(df[\"logRetweets\"].values[test_df.idx_b.values] - np.log10(test_df['retweetCount_b'].values + 1))) < 1e-8\n",
    "    assert np.abs(true[:, 0, 0] - df['logRetweets'].values).mean() < 1e-8\n",
    "    tab = \"Engagement\"\n",
    "    roberta_cmp = (df[\"RoBERTa\"].values[test_df.idx_a.values] > df[\"RoBERTa\"].values[test_df.idx_b.values]).astype(np.int32)\n",
    "    bow_cmp = (df[\"BoW\"].values[test_df.idx_a.values] > df[\"BoW\"].values[test_df.idx_b.values]).astype(np.int32)\n",
    "    gt_cmp = (df.logRetweets.values[test_df.idx_a.values] > df.logRetweets.values[test_df.idx_b.values]).astype(np.int32)\n",
    "    \n",
    "    gt = (test_df['retweetCount_a'] > test_df['retweetCount_b']).astype(np.int32)\n",
    "    assert (gt == gt_cmp).all()\n",
    "    latex_table.append(build_row(tab, gt, bow_cmp, roberta_cmp, gpt3_cmp, gpt4_cmp))\n",
    "    print(latex_table[-1])\n",
    "    return test_df\n",
    "\n",
    "engagement_fn()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd356361",
   "metadata": {},
   "source": [
    "# Well being"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "eb3b90dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_reddit\n",
      "Examples 960\n",
      "  BoW  :  30.8333   54.4174\n",
      "RoBERTa:  94.5833   84.8777\n",
      " GPT3  :  75.7292   80.4564\n",
      " GPT4  :  90.1042   83.7296\n",
      "\n",
      "\n",
      "&    reddit  &  $        75.73   $  &  $        30.83^{**} $  &  $\\textbf{94.58}^{**} $  &  $        90.10^{**} $  &  $        80.46   $  &  $        54.42^{**} $  &  $\\textbf{84.88}      $  &  $        83.73      $  \\\\\n",
      "---\n",
      "_reddit_body\n",
      "Examples 929\n",
      "  BoW  :  84.4995   68.8208\n",
      "RoBERTa:  89.8816   86.1648\n",
      " GPT3  :  91.9268   84.4086\n",
      " GPT4  :  93.3262   78.6251\n",
      "\n",
      "\n",
      "&    reddit body  &  $        91.93   $  &  $        84.50^{**} $  &  $        89.88      $  &  $\\textbf{93.33}      $  &  $        84.41   $  &  $        68.82^{**} $  &  $\\textbf{86.16}      $  &  $        78.63^{**} $  \\\\\n",
      "---\n",
      "_reddit_titles\n",
      "Examples 985\n",
      "  BoW  :  86.5990   85.6168\n",
      "RoBERTa:  96.7513   96.6451\n",
      " GPT3  :  80.6091   80.0484\n",
      " GPT4  :  89.5431   89.7661\n",
      "\n",
      "\n",
      "&    reddit titles  &  $        80.61   $  &  $        86.60^{**} $  &  $\\textbf{96.75}^{**} $  &  $        89.54^{**} $  &  $        80.05   $  &  $        85.62^{**} $  &  $\\textbf{96.65}^{**} $  &  $        89.77^{**} $  \\\\\n",
      "---\n",
      "_twitter\n",
      "Examples 798\n",
      "  BoW  :  43.3584   45.4465\n",
      "RoBERTa:  93.2331   93.1373\n",
      " GPT3  :  60.5263   65.0455\n",
      " GPT4  :  72.3058   73.4523\n",
      "\n",
      "\n",
      "&    twitter  &  $        60.53   $  &  $        43.36^{**} $  &  $\\textbf{93.23}^{**} $  &  $        72.31^{**} $  &  $        65.05   $  &  $        45.45^{**} $  &  $\\textbf{93.14}^{**} $  &  $        73.45^{**} $  \\\\\n",
      "---\n",
      "_twitter_full\n",
      "Examples 1499\n",
      "  BoW  :  80.3869   80.3900\n",
      "RoBERTa:  84.3896   84.3903\n",
      " GPT3  :  66.2442   66.2587\n",
      " GPT4  :  75.2502   75.2530\n",
      "\n",
      "\n",
      "&    twitter full  &  $        66.24   $  &  $        80.39^{**} $  &  $\\textbf{84.39}^{**} $  &  $        75.25^{**} $  &  $        66.26   $  &  $        80.39^{**} $  &  $\\textbf{84.39}^{**} $  &  $        75.25^{**} $  \\\\\n",
      "---\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>text</th>\n",
       "      <th>hashtags</th>\n",
       "      <th>labels</th>\n",
       "      <th>GPT3</th>\n",
       "      <th>GPT4</th>\n",
       "      <th>RoBERTa</th>\n",
       "      <th>BoW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>393</td>\n",
       "      <td>Free yourself from the stress of anger and res...</td>\n",
       "      <td>[]</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes.</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0.972885</td>\n",
       "      <td>0.984166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2535</td>\n",
       "      <td>Good night world sparkling heart family man wo...</td>\n",
       "      <td>['Alessandro', 'Elena', 'Amori', 'Treasures', ...</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>0.001246</td>\n",
       "      <td>0.030526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7558</td>\n",
       "      <td>Going Live with some more GTA V RP. What will ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>0.006567</td>\n",
       "      <td>0.930452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2610</td>\n",
       "      <td>18 Useful Tips for Healthy Teeth And Gums tagf...</td>\n",
       "      <td>['tagfire', 'lifestyle', 'healthcare', 'health...</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.125966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6743</td>\n",
       "      <td>Here's something different for you. mentalheal...</td>\n",
       "      <td>['mentalhealth', 'stressmanagement', 'ambient'...</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0.004010</td>\n",
       "      <td>0.059737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1495</th>\n",
       "      <td>2127</td>\n",
       "      <td>Healthy habits promote quality sleep which is ...</td>\n",
       "      <td>[\"'health'\", \"'diabetes'\", \"'fit'\", \"'healthy'...</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>0.035582</td>\n",
       "      <td>0.097194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1496</th>\n",
       "      <td>7458</td>\n",
       "      <td>Fields for Foxbridge day are absolutely brilli...</td>\n",
       "      <td>['excited']</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>0.000234</td>\n",
       "      <td>0.060168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1497</th>\n",
       "      <td>443</td>\n",
       "      <td>Paris 2020. I still smile at this photo even t...</td>\n",
       "      <td>[\"'NewProfilePic'\", \"'staysafe'\", \"'stayhome'\"...</td>\n",
       "      <td>1</td>\n",
       "      <td>No.</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0.371444</td>\n",
       "      <td>0.875410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1498</th>\n",
       "      <td>5929</td>\n",
       "      <td>LamechLamarch25 Solidarity amazing SaveAllTheR...</td>\n",
       "      <td>[\"'SaveAllTheRainforests'\", \"'STOPSHELL'\", \"'E...</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>0.960982</td>\n",
       "      <td>0.250854</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1499</th>\n",
       "      <td>1919</td>\n",
       "      <td>Don't let your negative emotions cloud you fro...</td>\n",
       "      <td>['MotivationMonday', 'blessings', 'gratitude',...</td>\n",
       "      <td>1</td>\n",
       "      <td>No.</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0.735286</td>\n",
       "      <td>0.392008</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1499 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index                                               text  \\\n",
       "0       393  Free yourself from the stress of anger and res...   \n",
       "1      2535  Good night world sparkling heart family man wo...   \n",
       "2      7558  Going Live with some more GTA V RP. What will ...   \n",
       "3      2610  18 Useful Tips for Healthy Teeth And Gums tagf...   \n",
       "4      6743  Here's something different for you. mentalheal...   \n",
       "...     ...                                                ...   \n",
       "1495   2127  Healthy habits promote quality sleep which is ...   \n",
       "1496   7458  Fields for Foxbridge day are absolutely brilli...   \n",
       "1497    443  Paris 2020. I still smile at this photo even t...   \n",
       "1498   5929  LamechLamarch25 Solidarity amazing SaveAllTheR...   \n",
       "1499   1919  Don't let your negative emotions cloud you fro...   \n",
       "\n",
       "                                               hashtags  labels  GPT3 GPT4  \\\n",
       "0                                                    []       1  Yes.  Yes   \n",
       "1     ['Alessandro', 'Elena', 'Amori', 'Treasures', ...       0   No.   No   \n",
       "2                                                    []       0   No.   No   \n",
       "3     ['tagfire', 'lifestyle', 'healthcare', 'health...       0   No.   No   \n",
       "4     ['mentalhealth', 'stressmanagement', 'ambient'...       0   No.  Yes   \n",
       "...                                                 ...     ...   ...  ...   \n",
       "1495  [\"'health'\", \"'diabetes'\", \"'fit'\", \"'healthy'...       0   No.   No   \n",
       "1496                                        ['excited']       0   No.   No   \n",
       "1497  [\"'NewProfilePic'\", \"'staysafe'\", \"'stayhome'\"...       1   No.  Yes   \n",
       "1498  [\"'SaveAllTheRainforests'\", \"'STOPSHELL'\", \"'E...       0   No.   No   \n",
       "1499  ['MotivationMonday', 'blessings', 'gratitude',...       1   No.  Yes   \n",
       "\n",
       "       RoBERTa       BoW  \n",
       "0     0.972885  0.984166  \n",
       "1     0.001246  0.030526  \n",
       "2     0.006567  0.930452  \n",
       "3     0.000005  0.125966  \n",
       "4     0.004010  0.059737  \n",
       "...        ...       ...  \n",
       "1495  0.035582  0.097194  \n",
       "1496  0.000234  0.060168  \n",
       "1497  0.371444  0.875410  \n",
       "1498  0.960982  0.250854  \n",
       "1499  0.735286  0.392008  \n",
       "\n",
       "[1499 rows x 8 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def well_being():\n",
    "    for size, tab in zip([1000, 1000, 1000, 800, 1500], ['well_being_reddit', 'well_being_reddit_body', 'well_being_reddit_titles', 'well_being_twitter', 'well_being_twitter_full']):\n",
    "        true = np.array(load(f\"results/outputs_{tab}_RoBERTa.json\"))\n",
    "        df = pd.read_csv(f\"chatgpt-responses/{tab}.csv\", index_col=0)\n",
    "        if 'body' in tab:\n",
    "            col = 'body'\n",
    "        elif 'reddit' in tab:\n",
    "            col = 'title'\n",
    "        else:\n",
    "            col = 'text'\n",
    "        df = filter_long_texts(df[:size], col)\n",
    "        # print(df.shape, true.shape)\n",
    "        assert np.abs(df.rename(columns={\"labels\":\"label\"})['label'].values - true[:, 0, 0]).mean() < 1e-8\n",
    "        df['RoBERTa'] = np.array(load(f\"results/predictions_{tab}_RoBERTa.json\"))[:, 0]\n",
    "        df['BoW'] = np.array(load(f\"results/predictions_{tab}_BoW.json\"))[:, 0]\n",
    "        df, shp, gpt3_pred, gpt4_pred = read_results(None, [\"Yes\"], [\"No\"], df=df)\n",
    "\n",
    "        gt = df.rename(columns={\"labels\":\"label\"})['label'].astype(np.int32)\n",
    "        roberta_pred = (df['RoBERTa'].values > 0.5).astype(np.int32)\n",
    "        bow_pred = (df['BoW'].values > 0.5).astype(np.int32)\n",
    "\n",
    "        latex_table.append(build_row(tab[10:].strip(), gt, bow_pred, roberta_pred, gpt3_pred, gpt4_pred))\n",
    "        print(latex_table[-1])\n",
    "        print('---')\n",
    "    return df\n",
    "\n",
    "well_being()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a800a0d7",
   "metadata": {},
   "source": [
    "# Emotion Intensity\n",
    "\n",
    "\n",
    "The prompt used (once for each emotion, `{emotion}` is replaced each time)\n",
    "```\n",
    "You are an expert at emotion analysis.\n",
    "Given a pair of text A and B from the user,\n",
    "you will output which text expresses higher intensity of the {emotion} emotion.\n",
    "Use the following format:\n",
    "* You are only allowed to answer \"A\" or \"B\".\n",
    "* Don't write an explanation of the answer.\n",
    "* Don't write things like \"My guess is...\", or \"I think ...\". Just write A or B, but nothing else.\n",
    "\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "aa635a50",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "joy\n",
      "Examples 2098\n",
      "  BoW  :  66.4919   66.5100\n",
      "RoBERTa:  75.4051   75.4029\n",
      " GPT3  :  74.0705   74.2889\n",
      " GPT4  :  78.4557   78.6338\n",
      "\n",
      "\n",
      "&   Joy  &  $        74.07   $  &  $        66.49^{**} $  &  $        75.41      $  &  $\\textbf{78.46}^{**} $  &  $        74.29   $  &  $        66.51^{**} $  &  $        75.40      $  &  $\\textbf{78.63}^{**} $  \\\\\n",
      "---\n",
      "fear\n",
      "Examples 2922\n",
      "  BoW  :  68.6516   68.6507\n",
      "RoBERTa:  76.8309   76.8311\n",
      " GPT3  :  72.7584   72.8628\n",
      " GPT4  :  73.9562   74.0945\n",
      "\n",
      "\n",
      "&   Fear  &  $        72.76   $  &  $        68.65^{**} $  &  $\\textbf{76.83}^{**} $  &  $        73.96      $  &  $        72.86   $  &  $        68.65^{**} $  &  $\\textbf{76.83}^{**} $  &  $        74.09      $  \\\\\n",
      "---\n",
      "anger\n",
      "Examples 2224\n",
      "  BoW  :  67.6259   67.6049\n",
      "RoBERTa:  73.4712   73.4608\n",
      " GPT3  :  72.1223   72.0870\n",
      " GPT4  :  75.5845   75.4945\n",
      "\n",
      "\n",
      "&   Anger  &  $        72.12   $  &  $        67.63^{**} $  &  $        73.47      $  &  $\\textbf{75.58}^{**} $  &  $        72.09   $  &  $        67.60^{**} $  &  $        73.46      $  &  $\\textbf{75.49}^{**} $  \\\\\n",
      "---\n",
      "sadness\n",
      "Examples 1972\n",
      "  BoW  :  72.4138   72.4049\n",
      "RoBERTa:  76.0649   76.0519\n",
      " GPT3  :  78.1947   78.2199\n",
      " GPT4  :  78.5497   78.5787\n",
      "\n",
      "\n",
      "&   Sadness  &  $        78.19   $  &  $        72.41^{**} $  &  $        76.06      $  &  $\\textbf{78.55}      $  &  $        78.22   $  &  $        72.40^{**} $  &  $        76.05      $  &  $\\textbf{78.58}      $  \\\\\n",
      "---\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>emotion</th>\n",
       "      <th>score</th>\n",
       "      <th>RoBERTa</th>\n",
       "      <th>BoW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>40860</td>\n",
       "      <td>My 2 teens sons just left in the car to get ha...</td>\n",
       "      <td>sadness</td>\n",
       "      <td>0.667</td>\n",
       "      <td>0.388516</td>\n",
       "      <td>0.585802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>40861</td>\n",
       "      <td>My 2 teens sons just left in the car to get ha...</td>\n",
       "      <td>sadness</td>\n",
       "      <td>0.458</td>\n",
       "      <td>0.300646</td>\n",
       "      <td>0.432004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>40862</td>\n",
       "      <td>HartRamsey'sUPLIFT If you're still discouraged...</td>\n",
       "      <td>sadness</td>\n",
       "      <td>0.396</td>\n",
       "      <td>0.360876</td>\n",
       "      <td>0.397999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>40863</td>\n",
       "      <td>@AmontanaW I nearly dropped my phone into the ...</td>\n",
       "      <td>sadness</td>\n",
       "      <td>0.271</td>\n",
       "      <td>0.415862</td>\n",
       "      <td>0.422215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>40864</td>\n",
       "      <td>Whenever I'm feeling sad I will listen to mons...</td>\n",
       "      <td>sadness</td>\n",
       "      <td>0.604</td>\n",
       "      <td>0.639444</td>\n",
       "      <td>0.716003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>668</th>\n",
       "      <td>41528</td>\n",
       "      <td>Why does Candice constantly pout #GBBO ðŸ’„ðŸ˜’</td>\n",
       "      <td>sadness</td>\n",
       "      <td>0.396</td>\n",
       "      <td>0.407754</td>\n",
       "      <td>0.498224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>669</th>\n",
       "      <td>41529</td>\n",
       "      <td>@redBus_in #unhappy with #redbus CC, when I ta...</td>\n",
       "      <td>sadness</td>\n",
       "      <td>0.604</td>\n",
       "      <td>0.561787</td>\n",
       "      <td>0.523657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>670</th>\n",
       "      <td>41530</td>\n",
       "      <td>@AceOperative789 no pull him afew weeks ago, s...</td>\n",
       "      <td>sadness</td>\n",
       "      <td>0.479</td>\n",
       "      <td>0.310910</td>\n",
       "      <td>0.515773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>671</th>\n",
       "      <td>41531</td>\n",
       "      <td>I'm buying art supplies and I'm debating how s...</td>\n",
       "      <td>sadness</td>\n",
       "      <td>0.375</td>\n",
       "      <td>0.346091</td>\n",
       "      <td>0.313963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>672</th>\n",
       "      <td>41532</td>\n",
       "      <td>@sainsburys Could you ask your Chafford Hundre...</td>\n",
       "      <td>sadness</td>\n",
       "      <td>0.438</td>\n",
       "      <td>0.380024</td>\n",
       "      <td>0.286177</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>673 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                               text  emotion  score  \\\n",
       "0    40860  My 2 teens sons just left in the car to get ha...  sadness  0.667   \n",
       "1    40861  My 2 teens sons just left in the car to get ha...  sadness  0.458   \n",
       "2    40862  HartRamsey'sUPLIFT If you're still discouraged...  sadness  0.396   \n",
       "3    40863  @AmontanaW I nearly dropped my phone into the ...  sadness  0.271   \n",
       "4    40864  Whenever I'm feeling sad I will listen to mons...  sadness  0.604   \n",
       "..     ...                                                ...      ...    ...   \n",
       "668  41528    Why does Candice constantly pout #GBBO ðŸ’„ðŸ˜’  sadness  0.396   \n",
       "669  41529  @redBus_in #unhappy with #redbus CC, when I ta...  sadness  0.604   \n",
       "670  41530  @AceOperative789 no pull him afew weeks ago, s...  sadness  0.479   \n",
       "671  41531  I'm buying art supplies and I'm debating how s...  sadness  0.375   \n",
       "672  41532  @sainsburys Could you ask your Chafford Hundre...  sadness  0.438   \n",
       "\n",
       "      RoBERTa       BoW  \n",
       "0    0.388516  0.585802  \n",
       "1    0.300646  0.432004  \n",
       "2    0.360876  0.397999  \n",
       "3    0.415862  0.422215  \n",
       "4    0.639444  0.716003  \n",
       "..        ...       ...  \n",
       "668  0.407754  0.498224  \n",
       "669  0.561787  0.523657  \n",
       "670  0.310910  0.515773  \n",
       "671  0.346091  0.313963  \n",
       "672  0.380024  0.286177  \n",
       "\n",
       "[673 rows x 6 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def emotions():\n",
    "    for tab in ['joy', 'fear', 'anger', 'sadness']:\n",
    "        df = pd.read_csv(f\"datasets/intensity ranking/emotion intensity/test/{tab}.txt\", delimiter=\"\\t\", names=[\"id\", \"text\", \"emotion\", \"score\"])\n",
    "        outs = np.array(load(f\"results/outputs_{tab}_emotion_intensity_BoW.json\"))\n",
    "        assert np.mean(np.abs(df.score.values - outs[:, 0, 0])) < 1e-8\n",
    "        df[\"RoBERTa\"] = np.array(load(f\"results/predictions_{tab}_emotion_intensity_RoBERTa.json\"))[:, 0]\n",
    "        df[\"BoW\"] = np.array(load(f\"results/predictions_{tab}_emotion_intensity_BoW.json\"))[:, 0]\n",
    "\n",
    "        test_df, shp, gpt3_cmp, gpt4_cmp = read_results(f'{tab}.csv', [\"A\"], [\"B\"])\n",
    "        assert np.all(df.text.values[test_df.idx_a.values] == test_df.text_a)\n",
    "        assert np.all(df.text.values[test_df.idx_b.values] == test_df.text_b)\n",
    "        roberta_cmp = (df[\"RoBERTa\"].values[test_df.idx_a.values] > df[\"RoBERTa\"].values[test_df.idx_b.values]).astype(np.int32)\n",
    "        bow_cmp = (df[\"BoW\"].values[test_df.idx_a.values] > df[\"BoW\"].values[test_df.idx_b.values]).astype(np.int32)\n",
    "        gt_cmp = (df.score.values[test_df.idx_a.values] > df.score.values[test_df.idx_b.values]).astype(np.int32)\n",
    "\n",
    "        gt = (test_df['score_a'] > test_df['score_b']).astype(np.int32)\n",
    "        assert (gt == gt_cmp).all()\n",
    "\n",
    "        latex_table.append(build_row(tab, gt, bow_cmp, roberta_cmp, gpt3_cmp, gpt4_cmp))\n",
    "        print(latex_table[-1])\n",
    "        print('---')\n",
    "    return df\n",
    "\n",
    "emotions()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38aeed0c",
   "metadata": {},
   "source": [
    "# Personality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0f1e2a5c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "openness\n",
      "Examples 5953\n",
      "  BoW  :  58.3571   58.3579\n",
      "RoBERTa:  60.5409   60.5619\n",
      " GPT3  :  50.1092   50.6038\n",
      " GPT4  :  54.7455   54.5876\n",
      "\n",
      "\n",
      "&   Openness  &  $        50.11   $  &  $        58.36^{**} $  &  $\\textbf{60.54}^{**} $  &  $        54.75^{**} $  &  $        50.60   $  &  $        58.36^{**} $  &  $\\textbf{60.56}^{**} $  &  $        54.59^{**} $  \\\\\n",
      "---\n",
      "conscientiousness\n",
      "Examples 5952\n",
      "  BoW  :  56.7876   56.7820\n",
      "RoBERTa:  61.5927   61.5950\n",
      " GPT3  :  55.5444   55.8419\n",
      " GPT4  :  57.4429   57.3327\n",
      "\n",
      "\n",
      "&   Conscientiousness  &  $        55.54   $  &  $        56.79      $  &  $\\textbf{61.59}^{**} $  &  $        57.44^*    $  &  $        55.84   $  &  $        56.78      $  &  $\\textbf{61.59}^{**} $  &  $        57.33      $  \\\\\n",
      "---\n",
      "extraversion\n",
      "Examples 5948\n",
      "  BoW  :  56.5064   56.5077\n",
      "RoBERTa:  59.0282   59.0195\n",
      " GPT3  :  53.5474   53.3846\n",
      " GPT4  :  55.9011   55.8998\n",
      "\n",
      "\n",
      "&   Extraversion  &  $        53.55   $  &  $        56.51^{**} $  &  $\\textbf{59.03}^{**} $  &  $        55.90^{**} $  &  $        53.38   $  &  $        56.51^{**} $  &  $\\textbf{59.02}^{**} $  &  $        55.90^{**} $  \\\\\n",
      "---\n",
      "agreeableness\n",
      "Examples 5951\n",
      "  BoW  :  57.8054   57.8040\n",
      "RoBERTa:  58.1247   58.1416\n",
      " GPT3  :  51.6720   52.0978\n",
      " GPT4  :  54.0413   54.0530\n",
      "\n",
      "\n",
      "&   Agreeableness  &  $        51.67   $  &  $        57.81^{**} $  &  $\\textbf{58.12}^{**} $  &  $        54.04^{**} $  &  $        52.10   $  &  $        57.80^{**} $  &  $\\textbf{58.14}^{**} $  &  $        54.05^*    $  \\\\\n",
      "---\n",
      "neuroticism\n",
      "Examples 5944\n",
      "  BoW  :  58.5969   58.5959\n",
      "RoBERTa:  59.8587   59.8636\n",
      " GPT3  :  48.9401   49.0438\n",
      " GPT4  :  49.6803   49.7275\n",
      "\n",
      "\n",
      "&   Neuroticism  &  $        48.94   $  &  $        58.60^{**} $  &  $\\textbf{59.86}^{**} $  &  $        49.68      $  &  $        49.04   $  &  $        58.60^{**} $  &  $\\textbf{59.86}^{**} $  &  $        49.73      $  \\\\\n",
      "---\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>openness</th>\n",
       "      <th>conscientiousness</th>\n",
       "      <th>extraversion</th>\n",
       "      <th>agreeableness</th>\n",
       "      <th>neuroticism</th>\n",
       "      <th>text</th>\n",
       "      <th>RoBERTa</th>\n",
       "      <th>BoW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.822222</td>\n",
       "      <td>0.669903</td>\n",
       "      <td>0.485981</td>\n",
       "      <td>0.681319</td>\n",
       "      <td>0.645833</td>\n",
       "      <td>... Going nuts from another room, run in there...</td>\n",
       "      <td>0.475278</td>\n",
       "      <td>0.540604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.655556</td>\n",
       "      <td>0.514563</td>\n",
       "      <td>0.616822</td>\n",
       "      <td>0.692308</td>\n",
       "      <td>0.593750</td>\n",
       "      <td>I've got a little bit to go but we need you th...</td>\n",
       "      <td>0.457885</td>\n",
       "      <td>0.574006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.522222</td>\n",
       "      <td>0.524272</td>\n",
       "      <td>0.467290</td>\n",
       "      <td>0.560440</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>A video's quality over quantity, so everyone c...</td>\n",
       "      <td>0.532378</td>\n",
       "      <td>0.552091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.660194</td>\n",
       "      <td>0.411215</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.458333</td>\n",
       "      <td>You really want to. That's made things a lot e...</td>\n",
       "      <td>0.555218</td>\n",
       "      <td>0.647303</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.411111</td>\n",
       "      <td>0.524272</td>\n",
       "      <td>0.317757</td>\n",
       "      <td>0.384615</td>\n",
       "      <td>0.437500</td>\n",
       "      <td>That's perfectly okay. The point isn't actuall...</td>\n",
       "      <td>0.542869</td>\n",
       "      <td>0.616673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1991</th>\n",
       "      <td>0.300000</td>\n",
       "      <td>0.135922</td>\n",
       "      <td>0.289720</td>\n",
       "      <td>0.208791</td>\n",
       "      <td>0.312500</td>\n",
       "      <td>Du du du. No, that's not why I made this video...</td>\n",
       "      <td>0.559876</td>\n",
       "      <td>0.596579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1992</th>\n",
       "      <td>0.722222</td>\n",
       "      <td>0.572816</td>\n",
       "      <td>0.719626</td>\n",
       "      <td>0.670330</td>\n",
       "      <td>0.781250</td>\n",
       "      <td>They do it all the time.</td>\n",
       "      <td>0.414828</td>\n",
       "      <td>0.520885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1993</th>\n",
       "      <td>0.677778</td>\n",
       "      <td>0.446602</td>\n",
       "      <td>0.355140</td>\n",
       "      <td>0.472527</td>\n",
       "      <td>0.395833</td>\n",
       "      <td>Comfortable and I don't want anyone to feel un...</td>\n",
       "      <td>0.542214</td>\n",
       "      <td>0.579495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1994</th>\n",
       "      <td>0.622222</td>\n",
       "      <td>0.669903</td>\n",
       "      <td>0.467290</td>\n",
       "      <td>0.527473</td>\n",
       "      <td>0.645833</td>\n",
       "      <td>You're not giving yourself enough calories to ...</td>\n",
       "      <td>0.556595</td>\n",
       "      <td>0.621109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1995</th>\n",
       "      <td>0.588889</td>\n",
       "      <td>0.728155</td>\n",
       "      <td>0.654206</td>\n",
       "      <td>0.813187</td>\n",
       "      <td>0.635417</td>\n",
       "      <td>Eat enough carbs, eat more fats to get in more...</td>\n",
       "      <td>0.452255</td>\n",
       "      <td>0.587024</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1996 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      openness  conscientiousness  extraversion  agreeableness  neuroticism  \\\n",
       "0     0.822222           0.669903      0.485981       0.681319     0.645833   \n",
       "1     0.655556           0.514563      0.616822       0.692308     0.593750   \n",
       "2     0.522222           0.524272      0.467290       0.560440     0.625000   \n",
       "3     0.400000           0.660194      0.411215       0.714286     0.458333   \n",
       "4     0.411111           0.524272      0.317757       0.384615     0.437500   \n",
       "...        ...                ...           ...            ...          ...   \n",
       "1991  0.300000           0.135922      0.289720       0.208791     0.312500   \n",
       "1992  0.722222           0.572816      0.719626       0.670330     0.781250   \n",
       "1993  0.677778           0.446602      0.355140       0.472527     0.395833   \n",
       "1994  0.622222           0.669903      0.467290       0.527473     0.645833   \n",
       "1995  0.588889           0.728155      0.654206       0.813187     0.635417   \n",
       "\n",
       "                                                   text   RoBERTa       BoW  \n",
       "0     ... Going nuts from another room, run in there...  0.475278  0.540604  \n",
       "1     I've got a little bit to go but we need you th...  0.457885  0.574006  \n",
       "2     A video's quality over quantity, so everyone c...  0.532378  0.552091  \n",
       "3     You really want to. That's made things a lot e...  0.555218  0.647303  \n",
       "4     That's perfectly okay. The point isn't actuall...  0.542869  0.616673  \n",
       "...                                                 ...       ...       ...  \n",
       "1991  Du du du. No, that's not why I made this video...  0.559876  0.596579  \n",
       "1992                           They do it all the time.  0.414828  0.520885  \n",
       "1993  Comfortable and I don't want anyone to feel un...  0.542214  0.579495  \n",
       "1994  You're not giving yourself enough calories to ...  0.556595  0.621109  \n",
       "1995  Eat enough carbs, eat more fats to get in more...  0.452255  0.587024  \n",
       "\n",
       "[1996 rows x 8 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def personalities():\n",
    "    traits = ['openness', 'conscientiousness', 'extraversion', 'agreeableness', 'neuroticism']\n",
    "    data = read_personality('test')\n",
    "    df = {t: np.array(data[1])[:, i] for i, t in enumerate(traits)}\n",
    "    df['text'] = data[0]\n",
    "    df = pd.DataFrame(df)\n",
    "\n",
    "    outs = np.array(load(f\"results/outputs_personality_BoW.json\"))\n",
    "    # assert np.mean(np.abs(df.score.values - outs[:, 0, 0])) < 1e-8\n",
    "    for i, tab in enumerate(traits):\n",
    "        df[f\"RoBERTa\"] = np.array(load(f\"results/predictions_personality_RoBERTa.json\"))[:, i]\n",
    "        df[f\"BoW\"] = np.array(load(f\"results/predictions_personality_BoW.json\"))[:, i]\n",
    "        test_df, shp, gpt3_cmp, gpt4_cmp = read_results(f'personality_{tab}.csv', [\"A\"], [\"B\"])\n",
    "        assert np.all(df.text.values[test_df.idx_a.values] == test_df.text_a)\n",
    "        assert np.all(df.text.values[test_df.idx_b.values] == test_df.text_b)\n",
    "        roberta_cmp = (df[\"RoBERTa\"].values[test_df.idx_a.values] > df[\"RoBERTa\"].values[test_df.idx_b.values]).astype(np.int32)\n",
    "        bow_cmp = (df[\"BoW\"].values[test_df.idx_a.values] > df[\"BoW\"].values[test_df.idx_b.values]).astype(np.int32)\n",
    "        gt_cmp = (df[tab].values[test_df.idx_a.values] > df[tab].values[test_df.idx_b.values]).astype(np.int32)\n",
    "\n",
    "        gt = (test_df[f'{tab}_a'] > test_df[f'{tab}_b']).astype(np.int32)\n",
    "        assert (gt == gt_cmp).all()\n",
    "        latex_table.append(build_row(tab, gt, bow_cmp, roberta_cmp, gpt3_cmp, gpt4_cmp))\n",
    "        print(latex_table[-1])\n",
    "        print('---')\n",
    "    return df\n",
    "\n",
    "personalities()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32586112",
   "metadata": {},
   "source": [
    "# Sentiment intensity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "64745a68",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "364 364\n",
      "sentiment_microblogs\n",
      "Examples 1075\n",
      "  BoW  :  70.8837   70.8266\n",
      "RoBERTa:  72.3721   72.4105\n",
      " GPT3  :  69.3023   68.6942\n",
      " GPT4  :  73.2093   73.0797\n",
      "\n",
      "\n",
      "&   Sentiment microblogs  &  $        69.30   $  &  $        70.88      $  &  $        72.37      $  &  $\\textbf{73.21}^{**} $  &  $        68.69   $  &  $        70.83      $  &  $        72.41^*    $  &  $\\textbf{73.08}^{**} $  \\\\\n",
      "---\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>spans</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>RoBERTa</th>\n",
       "      <th>BoW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>looks primed poised to run too</td>\n",
       "      <td>0.400</td>\n",
       "      <td>-0.226472</td>\n",
       "      <td>0.211981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>new HOD</td>\n",
       "      <td>0.432</td>\n",
       "      <td>0.453825</td>\n",
       "      <td>0.390131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Nice day rally!</td>\n",
       "      <td>0.529</td>\n",
       "      <td>0.537030</td>\n",
       "      <td>0.507975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>new HOD with conviction keeping $570 on watch ...</td>\n",
       "      <td>0.529</td>\n",
       "      <td>0.332714</td>\n",
       "      <td>0.399209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>then shorted it down.</td>\n",
       "      <td>-0.405</td>\n",
       "      <td>-0.439985</td>\n",
       "      <td>0.176595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360</th>\n",
       "      <td>Slips on Downgrade</td>\n",
       "      <td>-0.093</td>\n",
       "      <td>-0.403032</td>\n",
       "      <td>-0.435785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>5-star analyst @brianwieser from Pivotal Resea...</td>\n",
       "      <td>0.561</td>\n",
       "      <td>0.357204</td>\n",
       "      <td>0.171257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>362</th>\n",
       "      <td>see if they maintain 15% yield should be annou...</td>\n",
       "      <td>0.078</td>\n",
       "      <td>0.200502</td>\n",
       "      <td>0.191174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>Announces Record #Esports Viewership</td>\n",
       "      <td>0.208</td>\n",
       "      <td>0.419580</td>\n",
       "      <td>0.290590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>Don't catch falling knives</td>\n",
       "      <td>0.286</td>\n",
       "      <td>-0.323794</td>\n",
       "      <td>-0.242122</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>365 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 spans  sentiment   RoBERTa  \\\n",
       "0                       looks primed poised to run too      0.400 -0.226472   \n",
       "1                                              new HOD      0.432  0.453825   \n",
       "2                                      Nice day rally!      0.529  0.537030   \n",
       "3    new HOD with conviction keeping $570 on watch ...      0.529  0.332714   \n",
       "4                                then shorted it down.     -0.405 -0.439985   \n",
       "..                                                 ...        ...       ...   \n",
       "360                                 Slips on Downgrade     -0.093 -0.403032   \n",
       "361  5-star analyst @brianwieser from Pivotal Resea...      0.561  0.357204   \n",
       "362  see if they maintain 15% yield should be annou...      0.078  0.200502   \n",
       "363               Announces Record #Esports Viewership      0.208  0.419580   \n",
       "364                         Don't catch falling knives      0.286 -0.323794   \n",
       "\n",
       "          BoW  \n",
       "0    0.211981  \n",
       "1    0.390131  \n",
       "2    0.507975  \n",
       "3    0.399209  \n",
       "4    0.176595  \n",
       "..        ...  \n",
       "360 -0.435785  \n",
       "361  0.171257  \n",
       "362  0.191174  \n",
       "363  0.290590  \n",
       "364 -0.242122  \n",
       "\n",
       "[365 rows x 4 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def sentiment_intensities():\n",
    "    data = read_sentiment_ranking_microblogs(\"test\")\n",
    "    df = pd.DataFrame({\"spans\": data[0], \"sentiment\": data[1]})\n",
    "    tab = \"sentiment_microblogs\"\n",
    "\n",
    "    outs = np.array(load(f\"results/outputs_sentiment_ranking_microblogs_RoBERTa.json\"))[:, 0, 0]\n",
    "    assert np.mean(np.abs(df.sentiment.values - outs)) < 1e-5\n",
    "\n",
    "    df[\"RoBERTa\"] = np.array(load(f\"results/predictions_sentiment_ranking_microblogs_RoBERTa.json\"))[:, 0]\n",
    "    df[\"BoW\"] = np.array(load(f\"results/predictions_sentiment_ranking_microblogs_BoW.json\"))[:, 0]\n",
    "    df\n",
    "    test_df, shp, gpt3_cmp, gpt4_cmp = read_results('sentiment_microblogs.csv', [\"A\"], [\"B\"])\n",
    "    print(test_df.idx_a.max(), test_df.idx_b.max())\n",
    "    test_df = test_df[(test_df.idx_a < df.shape[0]) & (test_df.idx_b < df.shape[0])]\n",
    "\n",
    "    assert np.all(df.spans.values[test_df.idx_a.values] == test_df.spans_a)\n",
    "    assert np.all(df.spans.values[test_df.idx_b.values] == test_df.spans_b)\n",
    "    roberta_cmp = (df[\"RoBERTa\"].values[test_df.idx_a.values] > df[\"RoBERTa\"].values[test_df.idx_b.values]).astype(np.int32)\n",
    "    bow_cmp = (df[\"BoW\"].values[test_df.idx_a.values] > df[\"BoW\"].values[test_df.idx_b.values]).astype(np.int32)\n",
    "    gt_cmp = (df.sentiment.values[test_df.idx_a.values] > df.sentiment.values[test_df.idx_b.values]).astype(np.int32)\n",
    "\n",
    "    gt = (test_df['sentiment_a'] > test_df['sentiment_b']).astype(np.int32)\n",
    "    assert (gt == gt_cmp).all()\n",
    "\n",
    "    latex_table.append(build_row(tab, gt, bow_cmp, roberta_cmp, gpt3_cmp, gpt4_cmp))\n",
    "    print(latex_table[-1])\n",
    "    print('---')\n",
    "    return df\n",
    "\n",
    "sentiment_intensities()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13cb066b",
   "metadata": {},
   "source": [
    "# Sarcasm\n",
    "\n",
    "\n",
    "The prompt used:\n",
    "```\n",
    "You are an expert at sarcasm analysis.\n",
    "Given a text by the user, estimate if the given text is sarcastic or not.\n",
    "Use the following format:\n",
    "* You are only allowed to answer \"Yes\" or \"No\".\n",
    "* Don't write an explanation of the answer.\n",
    "* Don't write things like \"My guess is...\", or \"I think ...\". Just write Yes or No, but nothing else.\n",
    "\n",
    "```\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62b7ff70",
   "metadata": {},
   "source": [
    "tab = \"sarcasm.csv\"\n",
    "df, shp, msk_a, msk_b = read_results(tab, [\"Yes\"], [\"No\"])\n",
    "pred = msk_a.astype(np.int32)\n",
    "gt = df['is_sarcastic'].astype(np.int32)\n",
    "print(\"%30s %5d %3d,\\tAcc: %.4f , UAR: %.4f\\n\" % (tab, df.shape[0], shp - df.shape[0], ACC(gt, pred), UAR(gt, pred)))\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ffdbcc40",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sarcasm\n",
      "Examples 3983\n",
      "  BoW  :  63.1434   66.2883\n",
      "RoBERTa:  90.6603   90.6964\n",
      " GPT3  :  59.1263   56.8235\n",
      " GPT4  :  66.6583   69.4528\n",
      "\n",
      "\n",
      "&   Sarcasm  &  $        59.13   $  &  $        63.14^{**} $  &  $\\textbf{90.66}^{**} $  &  $        66.66^{**} $  &  $        56.82   $  &  $        66.29^{**} $  &  $\\textbf{90.70}^{**} $  &  $        69.45^{**} $  \\\\\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>article_link</th>\n",
       "      <th>headline</th>\n",
       "      <th>is_sarcastic</th>\n",
       "      <th>GPT3</th>\n",
       "      <th>GPT4</th>\n",
       "      <th>RoBERTa</th>\n",
       "      <th>BoW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3923</td>\n",
       "      <td>https://www.huffingtonpost.com/entry/lets-wish...</td>\n",
       "      <td>let's wish medicare and medicaid a happy birth...</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1.784492e-04</td>\n",
       "      <td>0.209161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>25404</td>\n",
       "      <td>https://politics.theonion.com/clinton-credits-...</td>\n",
       "      <td>clinton credits nevada victory to inescapable,...</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes.</td>\n",
       "      <td>Yes</td>\n",
       "      <td>9.999926e-01</td>\n",
       "      <td>0.969197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3321</td>\n",
       "      <td>https://www.huffingtonpost.com/entry/cat-celeb...</td>\n",
       "      <td>human throws her cat a snazzy birthday bash, f...</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes.</td>\n",
       "      <td>Yes</td>\n",
       "      <td>3.397014e-06</td>\n",
       "      <td>0.933317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23931</td>\n",
       "      <td>https://www.huffingtonpost.com/entry/laurie-he...</td>\n",
       "      <td>gymnast laurie hernandez and val chmerkovskiy ...</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes.</td>\n",
       "      <td>No</td>\n",
       "      <td>4.644791e-06</td>\n",
       "      <td>0.327732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5237</td>\n",
       "      <td>https://www.huffingtonpost.com/entry/the-best-...</td>\n",
       "      <td>the best italian winter salads have a surprisi...</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>4.989724e-07</td>\n",
       "      <td>0.201781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3995</th>\n",
       "      <td>7998</td>\n",
       "      <td>https://www.huffingtonpost.com/entry/sepp-blat...</td>\n",
       "      <td>sepp blatter faces 90-day suspension from fifa</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes.</td>\n",
       "      <td>No</td>\n",
       "      <td>1.530639e-03</td>\n",
       "      <td>0.619448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3996</th>\n",
       "      <td>7104</td>\n",
       "      <td>https://www.huffingtonpost.com/entry/warren-lo...</td>\n",
       "      <td>missouri gop lawmaker urges lynching for vanda...</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes.</td>\n",
       "      <td>Yes</td>\n",
       "      <td>4.895643e-05</td>\n",
       "      <td>0.979511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3997</th>\n",
       "      <td>13790</td>\n",
       "      <td>https://www.theonion.com/overpopulation-of-the...</td>\n",
       "      <td>overpopulation of the earth: will it create va...</td>\n",
       "      <td>1</td>\n",
       "      <td>No.</td>\n",
       "      <td>Yes</td>\n",
       "      <td>9.561596e-04</td>\n",
       "      <td>0.866452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3998</th>\n",
       "      <td>3619</td>\n",
       "      <td>https://www.theonion.com/the-cyberspace-revolu...</td>\n",
       "      <td>the cyberspace revolution: why are the media i...</td>\n",
       "      <td>1</td>\n",
       "      <td>No.</td>\n",
       "      <td>Yes</td>\n",
       "      <td>7.982037e-07</td>\n",
       "      <td>0.848396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3999</th>\n",
       "      <td>19734</td>\n",
       "      <td>https://www.huffingtonpost.com/entry/from-bicy...</td>\n",
       "      <td>from bicycles to spaceships (video)</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>1.095196e-05</td>\n",
       "      <td>0.626530</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3983 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index                                       article_link  \\\n",
       "0      3923  https://www.huffingtonpost.com/entry/lets-wish...   \n",
       "1     25404  https://politics.theonion.com/clinton-credits-...   \n",
       "2      3321  https://www.huffingtonpost.com/entry/cat-celeb...   \n",
       "3     23931  https://www.huffingtonpost.com/entry/laurie-he...   \n",
       "4      5237  https://www.huffingtonpost.com/entry/the-best-...   \n",
       "...     ...                                                ...   \n",
       "3995   7998  https://www.huffingtonpost.com/entry/sepp-blat...   \n",
       "3996   7104  https://www.huffingtonpost.com/entry/warren-lo...   \n",
       "3997  13790  https://www.theonion.com/overpopulation-of-the...   \n",
       "3998   3619  https://www.theonion.com/the-cyberspace-revolu...   \n",
       "3999  19734  https://www.huffingtonpost.com/entry/from-bicy...   \n",
       "\n",
       "                                               headline  is_sarcastic  GPT3  \\\n",
       "0     let's wish medicare and medicaid a happy birth...             0   No.   \n",
       "1     clinton credits nevada victory to inescapable,...             1  Yes.   \n",
       "2     human throws her cat a snazzy birthday bash, f...             0  Yes.   \n",
       "3     gymnast laurie hernandez and val chmerkovskiy ...             0  Yes.   \n",
       "4     the best italian winter salads have a surprisi...             0   No.   \n",
       "...                                                 ...           ...   ...   \n",
       "3995     sepp blatter faces 90-day suspension from fifa             0  Yes.   \n",
       "3996  missouri gop lawmaker urges lynching for vanda...             0  Yes.   \n",
       "3997  overpopulation of the earth: will it create va...             1   No.   \n",
       "3998  the cyberspace revolution: why are the media i...             1   No.   \n",
       "3999                from bicycles to spaceships (video)             0   No.   \n",
       "\n",
       "     GPT4       RoBERTa       BoW  \n",
       "0     Yes  1.784492e-04  0.209161  \n",
       "1     Yes  9.999926e-01  0.969197  \n",
       "2     Yes  3.397014e-06  0.933317  \n",
       "3      No  4.644791e-06  0.327732  \n",
       "4      No  4.989724e-07  0.201781  \n",
       "...   ...           ...       ...  \n",
       "3995   No  1.530639e-03  0.619448  \n",
       "3996  Yes  4.895643e-05  0.979511  \n",
       "3997  Yes  9.561596e-04  0.866452  \n",
       "3998  Yes  7.982037e-07  0.848396  \n",
       "3999   No  1.095196e-05  0.626530  \n",
       "\n",
       "[3983 rows x 8 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def run_sarcasm():\n",
    "    tab = \"sarcasm.csv\"\n",
    "    df = pd.read_csv(\"chatgpt-responses/sarcasm.csv\", index_col=0)\n",
    "    true = np.array(load(\"results/outputs_sarcasm_RoBERTa.json\"))\n",
    "    assert np.mean(np.abs(df[\"is_sarcastic\"].values - true[:, 0, 0])) < 1e-8\n",
    "    df['RoBERTa'] = np.array(load(\"results/predictions_sarcasm_RoBERTa.json\"))[:, 0]\n",
    "    df['BoW'] = np.array(load(\"results/predictions_sarcasm_BoW.json\"))[:, 0]\n",
    "    df, shp, gpt3_pred, gpt4_pred = read_results(None, [\"Yes\"], [\"No\"], df=df)\n",
    "    gt = df['is_sarcastic'].astype(np.int32)\n",
    "    roberta_pred = (df[\"RoBERTa\"] > 0.5).astype(np.int32)\n",
    "    bow_pred = (df[\"BoW\"] > 0.5).astype(np.int32)\n",
    "    latex_table.append(build_row(tab[:-4].strip(), gt, bow_pred, roberta_pred, gpt3_pred, gpt4_pred))\n",
    "    print(latex_table[-1])\n",
    "    return df\n",
    "\n",
    "run_sarcasm()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "00a5baac",
   "metadata": {},
   "source": [
    "```python\n",
    "tab = \"sarcasm.csv\"\n",
    "df = pd.read_csv(\"chatgpt-responses/sarcasm.csv\", index_col=0)\n",
    "true = np.array(load(\"results/outputs_sarcasm_RoBERTa.json\"))\n",
    "assert np.mean(np.abs(df[\"is_sarcastic\"].values - true[:, 0, 0])) < 1e-8\n",
    "df['RoBERTa'] = np.array(load(\"results/predictions_sarcasm_RoBERTa.json\"))[:, 0]\n",
    "df['BoW'] = np.array(load(\"results/predictions_sarcasm_BoW.json\"))[:, 0]\n",
    "df, shp, msk_a, msk_b = read_results(None, [\"Yes\"], [\"No\"], df=df)\n",
    "chatgpt_pred = msk_a.astype(np.int32)\n",
    "gt = df['is_sarcastic'].astype(np.int32)\n",
    "roberta_pred = (df[\"RoBERTa\"] > 0.5).astype(np.int32)\n",
    "bow_pred = (df[\"BoW\"] > 0.5).astype(np.int32)\n",
    "print(\"ChatGPT %30s %5d %3d,\\tAcc: %.4f , UAR: %.4f\\n\" % (tab, df.shape[0], shp - df.shape[0], ACC(gt, chatgpt_pred), UAR(gt, chatgpt_pred)))\n",
    "print(\"RoBERTa %30s %5d %3d,\\tAcc: %.4f , UAR: %.4f\\n\" % (tab, df.shape[0], shp - df.shape[0], ACC(gt, roberta_pred), UAR(gt, roberta_pred)))\n",
    "print(\"BoW     %30s %5d %3d,\\tAcc: %.4f , UAR: %.4f\\n\" % (tab, df.shape[0], shp - df.shape[0], ACC(gt, bow_pred), UAR(gt, bow_pred)))\n",
    "# print(\"& %15s  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  \\\\\\\\\" % (\n",
    "#     tab[:-4].replace(\"_\", \" \").capitalize(),\n",
    "#     ACC(gt, bow_pred), ACC(gt, roberta_pred), ACC(gt, chatgpt_pred),\n",
    "#     UAR(gt, bow_pred), UAR(gt, roberta_pred), UAR(gt, chatgpt_pred),\n",
    "# ))\n",
    "# latex_table.append(\"& %15s  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  \\\\\\\\\" % (\n",
    "#     tab[:-4].replace(\"_\", \" \").capitalize(),\n",
    "#     ACC(gt, bow_pred), ACC(gt, roberta_pred), ACC(gt, chatgpt_pred),\n",
    "#     UAR(gt, bow_pred), UAR(gt, roberta_pred), UAR(gt, chatgpt_pred),\n",
    "# ))\n",
    "\n",
    "latex_table.append(build_row(tab[:-4].strip(), gt, bow_pred, roberta_pred, chatgpt_pred))\n",
    "print(latex_table[-1])\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19ad59ff",
   "metadata": {},
   "source": [
    "# Sentiment analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7d52187e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sentiment\n",
      "Examples 1439\n",
      "  BoW  :  78.8742   78.8359\n",
      "RoBERTa:  88.7422   88.7471\n",
      " GPT3  :  80.5420   79.9343\n",
      " GPT4  :  84.0862   83.6862\n",
      "\n",
      "\n",
      "&   Sentiment  &  $        80.54   $  &  $        78.87      $  &  $\\textbf{88.74}^{**} $  &  $        84.09^{**} $  &  $        79.93   $  &  $        78.84      $  &  $\\textbf{88.75}^{**} $  &  $        83.69^{**} $  \\\\\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>index</th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "      <th>GPT3</th>\n",
       "      <th>GPT4</th>\n",
       "      <th>RoBERTa</th>\n",
       "      <th>BoW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>22330</td>\n",
       "      <td>121213</td>\n",
       "      <td>1833453441</td>\n",
       "      <td>0.0</td>\n",
       "      <td>is up, wishing i was sleep!</td>\n",
       "      <td>negative</td>\n",
       "      <td>negative</td>\n",
       "      <td>0.084942</td>\n",
       "      <td>0.167909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>72684</td>\n",
       "      <td>474389</td>\n",
       "      <td>2177321136</td>\n",
       "      <td>0.0</td>\n",
       "      <td>hates doing assignments.</td>\n",
       "      <td>negative</td>\n",
       "      <td>negative</td>\n",
       "      <td>0.014642</td>\n",
       "      <td>0.052236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>73235</td>\n",
       "      <td>419687</td>\n",
       "      <td>2062043792</td>\n",
       "      <td>0.0</td>\n",
       "      <td>@VAsNum1BaD yep you gotta hit me with sum song...</td>\n",
       "      <td>positive</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.192441</td>\n",
       "      <td>0.181997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>10661</td>\n",
       "      <td>452693</td>\n",
       "      <td>2070110306</td>\n",
       "      <td>0.0</td>\n",
       "      <td>@CrazyMikesapps good thanks, Funy Video, bring...</td>\n",
       "      <td>positive</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.990039</td>\n",
       "      <td>0.790992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>30699</td>\n",
       "      <td>986875</td>\n",
       "      <td>1834607616</td>\n",
       "      <td>1.0</td>\n",
       "      <td>@Stephanieee55 I listened to bullet for by val...</td>\n",
       "      <td>positive</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.980828</td>\n",
       "      <td>0.831771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2491</th>\n",
       "      <td>2491</td>\n",
       "      <td>13669</td>\n",
       "      <td>353360</td>\n",
       "      <td>2032438213</td>\n",
       "      <td>0.0</td>\n",
       "      <td>I don't even get a full 30 min break today</td>\n",
       "      <td>negative</td>\n",
       "      <td>negative</td>\n",
       "      <td>0.013156</td>\n",
       "      <td>0.159326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2493</th>\n",
       "      <td>2493</td>\n",
       "      <td>39851</td>\n",
       "      <td>842784</td>\n",
       "      <td>1563740039</td>\n",
       "      <td>1.0</td>\n",
       "      <td>@wolfbane Jealous, say hi to him for me! And t...</td>\n",
       "      <td>positive</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.980491</td>\n",
       "      <td>0.658321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2494</th>\n",
       "      <td>2494</td>\n",
       "      <td>34155</td>\n",
       "      <td>1329144</td>\n",
       "      <td>2015623877</td>\n",
       "      <td>1.0</td>\n",
       "      <td>@gtomita Great idea! Thanks v much for that - ...</td>\n",
       "      <td>positive</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.985527</td>\n",
       "      <td>0.840226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2495</th>\n",
       "      <td>2495</td>\n",
       "      <td>36679</td>\n",
       "      <td>518136</td>\n",
       "      <td>2191466407</td>\n",
       "      <td>0.0</td>\n",
       "      <td>haiz.. homeworks..</td>\n",
       "      <td>negative</td>\n",
       "      <td>negative</td>\n",
       "      <td>0.339497</td>\n",
       "      <td>0.533420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2496</th>\n",
       "      <td>2496</td>\n",
       "      <td>518</td>\n",
       "      <td>1325128</td>\n",
       "      <td>2015098380</td>\n",
       "      <td>1.0</td>\n",
       "      <td>@oakkoa Shi-booyah is best  Like Mr Pwnyaaaah</td>\n",
       "      <td>positive</td>\n",
       "      <td>positive</td>\n",
       "      <td>0.978248</td>\n",
       "      <td>0.701824</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1439 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0  Unnamed: 0.1    index    tweet_id  sentiment  \\\n",
       "0              0         22330   121213  1833453441        0.0   \n",
       "1              1         72684   474389  2177321136        0.0   \n",
       "2              2         73235   419687  2062043792        0.0   \n",
       "3              3         10661   452693  2070110306        0.0   \n",
       "7              7         30699   986875  1834607616        1.0   \n",
       "...          ...           ...      ...         ...        ...   \n",
       "2491        2491         13669   353360  2032438213        0.0   \n",
       "2493        2493         39851   842784  1563740039        1.0   \n",
       "2494        2494         34155  1329144  2015623877        1.0   \n",
       "2495        2495         36679   518136  2191466407        0.0   \n",
       "2496        2496           518  1325128  2015098380        1.0   \n",
       "\n",
       "                                                   text      GPT3      GPT4  \\\n",
       "0                         is up, wishing i was sleep!    negative  negative   \n",
       "1                             hates doing assignments.   negative  negative   \n",
       "2     @VAsNum1BaD yep you gotta hit me with sum song...  positive  positive   \n",
       "3     @CrazyMikesapps good thanks, Funy Video, bring...  positive  positive   \n",
       "7     @Stephanieee55 I listened to bullet for by val...  positive  positive   \n",
       "...                                                 ...       ...       ...   \n",
       "2491        I don't even get a full 30 min break today   negative  negative   \n",
       "2493  @wolfbane Jealous, say hi to him for me! And t...  positive  positive   \n",
       "2494  @gtomita Great idea! Thanks v much for that - ...  positive  positive   \n",
       "2495                                haiz.. homeworks..   negative  negative   \n",
       "2496      @oakkoa Shi-booyah is best  Like Mr Pwnyaaaah  positive  positive   \n",
       "\n",
       "       RoBERTa       BoW  \n",
       "0     0.084942  0.167909  \n",
       "1     0.014642  0.052236  \n",
       "2     0.192441  0.181997  \n",
       "3     0.990039  0.790992  \n",
       "7     0.980828  0.831771  \n",
       "...        ...       ...  \n",
       "2491  0.013156  0.159326  \n",
       "2493  0.980491  0.658321  \n",
       "2494  0.985527  0.840226  \n",
       "2495  0.339497  0.533420  \n",
       "2496  0.978248  0.701824  \n",
       "\n",
       "[1439 rows x 10 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def run_sentiment():\n",
    "    tab = \"sentiment.csv\"\n",
    "    df = pd.read_csv(\"chatgpt-responses/sentiment.csv\")\n",
    "\n",
    "    true = np.array(load(\"results/outputs_sentiment_RoBERTa.json\"))\n",
    "    assert np.mean(np.abs(df[\"sentiment\"].values - true[:, 0, 0])) < 1e-8\n",
    "    df['RoBERTa'] = np.array(load(\"results/predictions_sentiment_RoBERTa.json\"))[:, 0]\n",
    "    df['BoW'] = np.array(load(\"results/predictions_sentiment_BoW.json\"))[:, 0]\n",
    "    df, shp, gpt3_pred, gpt4_pred = read_results(None, [\"positive\"], [\"negative\"], df=df)\n",
    "    gt = df['sentiment'].astype(np.int32)\n",
    "    roberta_pred = (df[\"RoBERTa\"] > 0.5).astype(np.int32)\n",
    "    bow_pred = (df[\"BoW\"] > 0.5).astype(np.int32)\n",
    "\n",
    "    latex_table.append(build_row(tab[:-4].strip(), gt, bow_pred, roberta_pred, gpt3_pred, gpt4_pred))\n",
    "    print(latex_table[-1])\n",
    "    return df\n",
    "\n",
    "run_sentiment()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62bc2a74",
   "metadata": {},
   "source": [
    "```python\n",
    "tab = \"sentiment.csv\"\n",
    "df = pd.read_csv(\"chatgpt-responses/sentiment.csv\")\n",
    "\n",
    "true = np.array(load(\"results/outputs_sentiment_RoBERTa.json\"))\n",
    "assert np.mean(np.abs(df[\"sentiment\"].values - true[:, 0, 0])) < 1e-8\n",
    "df['RoBERTa'] = np.array(load(\"results/predictions_sentiment_RoBERTa.json\"))[:, 0]\n",
    "df['BoW'] = np.array(load(\"results/predictions_sentiment_BoW.json\"))[:, 0]\n",
    "df, shp, msk_a, msk_b = read_results(None, [\"Positive\"], [\"Negative\"], df=df)\n",
    "chatgpt_pred = msk_a.astype(np.int32)\n",
    "gt = df['sentiment'].astype(np.int32)\n",
    "roberta_pred = (df[\"RoBERTa\"] > 0.5).astype(np.int32)\n",
    "bow_pred = (df[\"BoW\"] > 0.5).astype(np.int32)\n",
    "print(\"ChatGPT %30s %5d %3d,\\tAcc: %.4f , UAR: %.4f\\n\" % (tab, df.shape[0], shp - df.shape[0], ACC(gt, chatgpt_pred), UAR(gt, chatgpt_pred)))\n",
    "print(\"RoBERTa %30s %5d %3d,\\tAcc: %.4f , UAR: %.4f\\n\" % (tab, df.shape[0], shp - df.shape[0], ACC(gt, roberta_pred), UAR(gt, roberta_pred)))\n",
    "print(\"BoW     %30s %5d %3d,\\tAcc: %.4f , UAR: %.4f\\n\" % (tab, df.shape[0], shp - df.shape[0], ACC(gt, bow_pred), UAR(gt, bow_pred)))\n",
    "# print(\"& %15s  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  \\\\\\\\\" % (\n",
    "#     tab[:-4].replace(\"_\", \" \").capitalize(),\n",
    "#     ACC(gt, bow_pred), ACC(gt, roberta_pred), ACC(gt, chatgpt_pred),\n",
    "#     UAR(gt, bow_pred), UAR(gt, roberta_pred), UAR(gt, chatgpt_pred),\n",
    "# ))\n",
    "# latex_table.append(\"& %15s  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  \\\\\\\\\" % (\n",
    "#     tab[:-4].replace(\"_\", \" \").capitalize(),\n",
    "#     ACC(gt, bow_pred), ACC(gt, roberta_pred), ACC(gt, chatgpt_pred),\n",
    "#     UAR(gt, bow_pred), UAR(gt, roberta_pred), UAR(gt, chatgpt_pred),\n",
    "# ))\n",
    "\n",
    "\n",
    "latex_table.append(build_row(tab[:-4].strip(), gt, bow_pred, roberta_pred, chatgpt_pred))\n",
    "print(latex_table[-1])\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3fb5e3f7",
   "metadata": {},
   "source": [
    "# Suicide detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4076117a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "suicide\n",
      "Examples 2295\n",
      "  BoW  :  84.7495   85.3232\n",
      "RoBERTa:  98.4314   98.4620\n",
      " GPT3  :  89.4553   89.4426\n",
      " GPT4  :  93.4641   93.3191\n",
      "\n",
      "\n",
      "&   Suicide  &  $        89.46   $  &  $        84.75^{**} $  &  $\\textbf{98.43}^{**} $  &  $        93.46^{**} $  &  $        89.44   $  &  $        85.32^{**} $  &  $\\textbf{98.46}^{**} $  &  $        93.32^{**} $  \\\\\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>index</th>\n",
       "      <th>text</th>\n",
       "      <th>class</th>\n",
       "      <th>GPT3</th>\n",
       "      <th>GPT4</th>\n",
       "      <th>RoBERTa</th>\n",
       "      <th>BoW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3822</td>\n",
       "      <td>47484</td>\n",
       "      <td>Found this girl, can somebody check this?I don...</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes.</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.940242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2711</td>\n",
       "      <td>130026</td>\n",
       "      <td>Doing it tonight. I'm not looking at it as the...</td>\n",
       "      <td>1</td>\n",
       "      <td>No.</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.962123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>604</td>\n",
       "      <td>231288</td>\n",
       "      <td>time to tell the truth if you diss someone on ...</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>1.687421e-07</td>\n",
       "      <td>0.314545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3354</td>\n",
       "      <td>68799</td>\n",
       "      <td>I feel like unless I am genuinely about to kil...</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes.</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.989850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>3707</td>\n",
       "      <td>46936</td>\n",
       "      <td>I feel like i can eat 3 pufferfish without bei...</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>9.337607e-09</td>\n",
       "      <td>0.458263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2340</th>\n",
       "      <td>2340</td>\n",
       "      <td>1444</td>\n",
       "      <td>36509</td>\n",
       "      <td>I have been search for video that change my vi...</td>\n",
       "      <td>1</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.977159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2341</th>\n",
       "      <td>2341</td>\n",
       "      <td>3230</td>\n",
       "      <td>312771</td>\n",
       "      <td>J'aime publier des choses en français Si je me...</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>9.794822e-09</td>\n",
       "      <td>0.551134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2342</th>\n",
       "      <td>2342</td>\n",
       "      <td>447</td>\n",
       "      <td>324659</td>\n",
       "      <td>i have question What's the fucking problem to ...</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>2.000301e-09</td>\n",
       "      <td>0.188090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2343</th>\n",
       "      <td>2343</td>\n",
       "      <td>3213</td>\n",
       "      <td>271471</td>\n",
       "      <td>Things are bad and I don't have the energy to ...</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes.</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.988960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2344</th>\n",
       "      <td>2344</td>\n",
       "      <td>980</td>\n",
       "      <td>229895</td>\n",
       "      <td>Everyday I swear problems just pop up everyday...</td>\n",
       "      <td>1</td>\n",
       "      <td>Yes.</td>\n",
       "      <td>Yes</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.986540</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2295 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0  Unnamed: 0.1   index  \\\n",
       "0              0          3822   47484   \n",
       "1              1          2711  130026   \n",
       "2              2           604  231288   \n",
       "3              3          3354   68799   \n",
       "4              4          3707   46936   \n",
       "...          ...           ...     ...   \n",
       "2340        2340          1444   36509   \n",
       "2341        2341          3230  312771   \n",
       "2342        2342           447  324659   \n",
       "2343        2343          3213  271471   \n",
       "2344        2344           980  229895   \n",
       "\n",
       "                                                   text  class  GPT3 GPT4  \\\n",
       "0     Found this girl, can somebody check this?I don...      1  Yes.  Yes   \n",
       "1     Doing it tonight. I'm not looking at it as the...      1   No.  Yes   \n",
       "2     time to tell the truth if you diss someone on ...      0   No.   No   \n",
       "3     I feel like unless I am genuinely about to kil...      1  Yes.  Yes   \n",
       "4     I feel like i can eat 3 pufferfish without bei...      0   No.   No   \n",
       "...                                                 ...    ...   ...  ...   \n",
       "2340  I have been search for video that change my vi...      1   No.   No   \n",
       "2341  J'aime publier des choses en français Si je me...      0   No.   No   \n",
       "2342  i have question What's the fucking problem to ...      0   No.   No   \n",
       "2343  Things are bad and I don't have the energy to ...      1  Yes.  Yes   \n",
       "2344  Everyday I swear problems just pop up everyday...      1  Yes.  Yes   \n",
       "\n",
       "           RoBERTa       BoW  \n",
       "0     1.000000e+00  0.940242  \n",
       "1     1.000000e+00  0.962123  \n",
       "2     1.687421e-07  0.314545  \n",
       "3     1.000000e+00  0.989850  \n",
       "4     9.337607e-09  0.458263  \n",
       "...            ...       ...  \n",
       "2340  1.000000e+00  0.977159  \n",
       "2341  9.794822e-09  0.551134  \n",
       "2342  2.000301e-09  0.188090  \n",
       "2343  1.000000e+00  0.988960  \n",
       "2344  1.000000e+00  0.986540  \n",
       "\n",
       "[2295 rows x 9 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def run_suicides():\n",
    "    tab = \"suicide.csv\"\n",
    "    df = pd.read_csv(\"chatgpt-responses/suicide.csv\")\n",
    "\n",
    "    true = np.array(load(\"results/outputs_suicide_BoW.json\"))\n",
    "    assert np.mean(np.abs(df[\"class\"].values - true[:, 0, 0])) < 1e-8\n",
    "    df['RoBERTa'] = np.array(load(\"results/predictions_suicide_RoBERTa.json\"))[:, 0]\n",
    "    df['BoW'] = np.array(load(\"results/predictions_suicide_BoW.json\"))[:, 0]\n",
    "    df, shp, gpt3_pred, gpt4_pred = read_results(None, [\"Yes\"], [\"No\"], df=df)\n",
    "    gt = df['class'].astype(np.int32)\n",
    "    roberta_pred = (df[\"RoBERTa\"] > 0.5).astype(np.int32)\n",
    "    bow_pred = (df[\"BoW\"] > 0.5).astype(np.int32)\n",
    "\n",
    "    latex_table.append(build_row(tab[:-4].strip(), gt, bow_pred, roberta_pred, gpt3_pred, gpt4_pred))\n",
    "    print(latex_table[-1])\n",
    "    return df\n",
    "\n",
    "run_suicides()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbcb913c",
   "metadata": {},
   "source": [
    "# Subjectivity detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "85e6310b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "subjectivity\n",
      "Examples 1894\n",
      "  BoW  :  87.2756   87.1944\n",
      "RoBERTa:  95.5649   95.5101\n",
      " GPT3  :  59.5565   58.5925\n",
      " GPT4  :  88.3844   88.1881\n",
      "\n",
      "\n",
      "&   Subjectivity  &  $        59.56   $  &  $        87.28^{**} $  &  $\\textbf{95.56}^{**} $  &  $        88.38^{**} $  &  $        58.59   $  &  $        87.19^{**} $  &  $\\textbf{95.51}^{**} $  &  $        88.19^{**} $  \\\\\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>GPT3</th>\n",
       "      <th>GPT4</th>\n",
       "      <th>RoBERTa</th>\n",
       "      <th>BoW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1060</td>\n",
       "      <td>\" the best disney movie since the lion king \"</td>\n",
       "      <td>1</td>\n",
       "      <td>subjective</td>\n",
       "      <td>subjective</td>\n",
       "      <td>0.999268</td>\n",
       "      <td>0.988961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4915</td>\n",
       "      <td>he also shies away from the shallow young wome...</td>\n",
       "      <td>0</td>\n",
       "      <td>objective</td>\n",
       "      <td>objective</td>\n",
       "      <td>0.000063</td>\n",
       "      <td>0.023651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>755</td>\n",
       "      <td>tommy , who is now a well known activist , has...</td>\n",
       "      <td>0</td>\n",
       "      <td>subjective</td>\n",
       "      <td>objective</td>\n",
       "      <td>0.000050</td>\n",
       "      <td>0.149007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>973</td>\n",
       "      <td>first , for a movie that tries to be smart , i...</td>\n",
       "      <td>1</td>\n",
       "      <td>subjective</td>\n",
       "      <td>subjective</td>\n",
       "      <td>0.999426</td>\n",
       "      <td>0.973268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2167</td>\n",
       "      <td>these four will collide in a titanic battle to...</td>\n",
       "      <td>0</td>\n",
       "      <td>subjective</td>\n",
       "      <td>objective</td>\n",
       "      <td>0.000072</td>\n",
       "      <td>0.053076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1995</th>\n",
       "      <td>3532</td>\n",
       "      <td>he is a cop , and chances are he's going to ma...</td>\n",
       "      <td>0</td>\n",
       "      <td>subjective</td>\n",
       "      <td>subjective</td>\n",
       "      <td>0.000082</td>\n",
       "      <td>0.111219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1996</th>\n",
       "      <td>2963</td>\n",
       "      <td>both got stuck in a broken elevator , 17 level...</td>\n",
       "      <td>0</td>\n",
       "      <td>objective</td>\n",
       "      <td>objective</td>\n",
       "      <td>0.000358</td>\n",
       "      <td>0.522912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1997</th>\n",
       "      <td>443</td>\n",
       "      <td>the plot mechanics of read my lips eventually ...</td>\n",
       "      <td>1</td>\n",
       "      <td>subjective</td>\n",
       "      <td>subjective</td>\n",
       "      <td>0.999350</td>\n",
       "      <td>0.994147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1998</th>\n",
       "      <td>2681</td>\n",
       "      <td>one of the pleasures in walter's documentary ....</td>\n",
       "      <td>1</td>\n",
       "      <td>subjective</td>\n",
       "      <td>subjective</td>\n",
       "      <td>0.999419</td>\n",
       "      <td>0.446346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1999</th>\n",
       "      <td>1102</td>\n",
       "      <td>birot has succeeded in making a movie that has...</td>\n",
       "      <td>1</td>\n",
       "      <td>subjective</td>\n",
       "      <td>subjective</td>\n",
       "      <td>0.997805</td>\n",
       "      <td>0.986587</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1894 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index                                               text  label  \\\n",
       "0      1060     \" the best disney movie since the lion king \"       1   \n",
       "1      4915  he also shies away from the shallow young wome...      0   \n",
       "2       755  tommy , who is now a well known activist , has...      0   \n",
       "3       973  first , for a movie that tries to be smart , i...      1   \n",
       "4      2167  these four will collide in a titanic battle to...      0   \n",
       "...     ...                                                ...    ...   \n",
       "1995   3532  he is a cop , and chances are he's going to ma...      0   \n",
       "1996   2963  both got stuck in a broken elevator , 17 level...      0   \n",
       "1997    443  the plot mechanics of read my lips eventually ...      1   \n",
       "1998   2681  one of the pleasures in walter's documentary ....      1   \n",
       "1999   1102  birot has succeeded in making a movie that has...      1   \n",
       "\n",
       "            GPT3        GPT4   RoBERTa       BoW  \n",
       "0     subjective  subjective  0.999268  0.988961  \n",
       "1      objective   objective  0.000063  0.023651  \n",
       "2     subjective   objective  0.000050  0.149007  \n",
       "3     subjective  subjective  0.999426  0.973268  \n",
       "4     subjective   objective  0.000072  0.053076  \n",
       "...          ...         ...       ...       ...  \n",
       "1995  subjective  subjective  0.000082  0.111219  \n",
       "1996   objective   objective  0.000358  0.522912  \n",
       "1997  subjective  subjective  0.999350  0.994147  \n",
       "1998  subjective  subjective  0.999419  0.446346  \n",
       "1999  subjective  subjective  0.997805  0.986587  \n",
       "\n",
       "[1894 rows x 7 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def run_subjectivity():\n",
    "    tab = \"subjectivity.csv\"\n",
    "    df = pd.read_csv(\"chatgpt-responses/subjectivity.csv\", index_col=0)\n",
    "    true = np.array(load(\"results/outputs_subjectivity_BoW.json\"))\n",
    "    assert np.mean(np.abs(true[:, 0, 0] - df.label.values)) < 1e-9\n",
    "    df['RoBERTa'] = np.array(load(\"results/predictions_subjectivity_RoBERTa.json\"))[:, 0]\n",
    "    df['BoW'] = np.array(load(\"results/predictions_subjectivity_BoW.json\"))[:, 0]\n",
    "    df, shp, gpt3_pred, gpt4_pred = read_results(None, [\"subjective\"], [\"objective\"], df=df)\n",
    "    gt = df['label'].astype(np.int32)\n",
    "    roberta_pred = (df['RoBERTa'].values > 0.5).astype(np.int32)\n",
    "    bow_pred = (df['BoW'].values > 0.5).astype(np.int32)\n",
    "\n",
    "    latex_table.append(build_row(tab[:-4].strip(), gt, bow_pred, roberta_pred, gpt3_pred, gpt4_pred))\n",
    "    print(latex_table[-1])\n",
    "    return df\n",
    "\n",
    "run_subjectivity()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df4afd7e",
   "metadata": {},
   "source": [
    "# Toxicity\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "429fd0f9",
   "metadata": {},
   "source": [
    "```python\n",
    "toxicity_df = pd.read_csv(\"datasets/toxicity detection/data/test.csv\").set_index(\"id\")\n",
    "toxicity_df = toxicity_df.join(pd.read_csv(\"datasets/toxicity detection/data/test_labels.csv\").set_index(\"id\"))\n",
    "toxicity_df = toxicity_df.reset_index()\n",
    "toxicity_df = toxicity_df[~np.any(toxicity_df[toxicity_df.columns[2:]] == -1, axis=1)]\n",
    "# msk = np.any(toxicity_df[toxicity_df.columns[1:]] == 1, axis=1)\n",
    "msk = toxicity_df[toxicity_df.columns[2:]].astype(np.float32)\n",
    "weights = msk * np.power((1 - msk).sum(axis=0) / msk.sum(axis=0) + 1, 1.1)\n",
    "weights = (weights * 0.5).mean(axis=1) + 1\n",
    "# ##weights = (weights * 2).sum(axis=1) + 1\n",
    "# ##weights = msk.astype(np.float32) * 4 + 1\n",
    "dq = pd.concat([toxicity_df[c].value_counts() for c in toxicity_df.columns[2:]], axis=1)\n",
    "print(dq.sort_index())\n",
    "df = toxicity_df.sample(frac=1, random_state=41, weights=weights).reset_index()[:1000]\n",
    "dq = pd.concat([df[c].value_counts() for c in df.columns[3:]], axis=1)\n",
    "print(dq.sort_index())\n",
    "print(df.shape, df.columns[3:])\n",
    "df = filter_long_texts(df, \"comment_text\")\n",
    "dq = pd.concat([df[c].value_counts() for c in df.columns[3:]], axis=1)\n",
    "print(dq.sort_index())\n",
    "print(df.shape, df.columns[3:])\n",
    "df\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "15b72405",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "toxic\n",
      "Examples 887\n",
      "  BoW  :  81.8489   82.7507\n",
      "RoBERTa:  85.2311   86.0097\n",
      " GPT3  :  87.3732   87.1927\n",
      " GPT4  :  89.2897   89.6502\n",
      "\n",
      "\n",
      "&   Toxic  &  $        87.37   $  &  $        81.85^{**} $  &  $        85.23      $  &  $\\textbf{89.29}      $  &  $        87.19   $  &  $        82.75^{**} $  &  $        86.01      $  &  $\\textbf{89.65}^*    $  \\\\\n",
      "---\n",
      "severe_toxic\n",
      "Examples 858\n",
      "  BoW  :  87.6457   82.4769\n",
      "RoBERTa:  80.0699   84.7826\n",
      " GPT3  :  66.5501   80.6544\n",
      " GPT4  :  75.5245   85.2877\n",
      "\n",
      "\n",
      "&   Severe toxic  &  $        66.55   $  &  $\\textbf{87.65}^{**} $  &  $        80.07^{**} $  &  $        75.52^{**} $  &  $        80.65   $  &  $        82.48      $  &  $        84.78^*    $  &  $\\textbf{85.29}^{**} $  \\\\\n",
      "---\n",
      "obscene\n",
      "Examples 870\n",
      "  BoW  :  85.4023   86.6163\n",
      "RoBERTa:  84.8276   86.6030\n",
      " GPT3  :  83.4483   83.4776\n",
      " GPT4  :  88.1609   86.7815\n",
      "\n",
      "\n",
      "&   Obscene  &  $        83.45   $  &  $        85.40      $  &  $        84.83      $  &  $\\textbf{88.16}^{**} $  &  $        83.48   $  &  $        86.62^*    $  &  $        86.60^*    $  &  $\\textbf{86.78}^*    $  \\\\\n",
      "---\n",
      "threat\n",
      "Examples 874\n",
      "  BoW  :  94.0503   73.9865\n",
      "RoBERTa:  95.5378   87.4277\n",
      " GPT3  :  70.5950   80.1194\n",
      " GPT4  :  91.9908   91.5105\n",
      "\n",
      "\n",
      "&   Threat  &  $        70.59   $  &  $        94.05^{**} $  &  $\\textbf{95.54}^{**} $  &  $        91.99^{**} $  &  $        80.12   $  &  $        73.99      $  &  $        87.43^{**} $  &  $\\textbf{91.51}^{**} $  \\\\\n",
      "---\n",
      "insult\n",
      "Examples 886\n",
      "  BoW  :  84.6501   84.6422\n",
      "RoBERTa:  87.2460   89.2790\n",
      " GPT3  :  80.1354   83.2060\n",
      " GPT4  :  80.6998   84.8851\n",
      "\n",
      "\n",
      "&   Insult  &  $        80.14   $  &  $        84.65^{**} $  &  $\\textbf{87.25}^{**} $  &  $        80.70      $  &  $        83.21   $  &  $        84.64      $  &  $\\textbf{89.28}^{**} $  &  $        84.89      $  \\\\\n",
      "---\n",
      "identity_hate\n",
      "Examples 865\n",
      "  BoW  :  90.5202   81.6145\n",
      "RoBERTa:  90.9827   86.1643\n",
      " GPT3  :  66.8208   78.6069\n",
      " GPT4  :  82.6590   87.8762\n",
      "\n",
      "\n",
      "&   Identity hate  &  $        66.82   $  &  $        90.52^{**} $  &  $\\textbf{90.98}^{**} $  &  $        82.66^{**} $  &  $        78.61   $  &  $        81.61      $  &  $        86.16^{**} $  &  $\\textbf{87.88}^{**} $  \\\\\n",
      "---\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "      <th>GPT3</th>\n",
       "      <th>GPT4</th>\n",
       "      <th>RoBERTa</th>\n",
       "      <th>BoW</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7366</td>\n",
       "      <td>0c65981f8c1e26fd</td>\n",
       "      <td>== Disney Kids == \\n\\n The Disney Kids rebrand...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes.</td>\n",
       "      <td>No</td>\n",
       "      <td>0.581859</td>\n",
       "      <td>0.659417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>103978</td>\n",
       "      <td>ad89eccb5456659c</td>\n",
       "      <td>I rather disagree with some of the actions by ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>0.000229</td>\n",
       "      <td>0.000362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6858</td>\n",
       "      <td>0b8abd149f7da6b1</td>\n",
       "      <td>== OBL == \\n\\n That's right, its Osama bin Lat...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes.</td>\n",
       "      <td>Yes</td>\n",
       "      <td>0.395430</td>\n",
       "      <td>0.002460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>93019</td>\n",
       "      <td>9b2506a652ec76cd</td>\n",
       "      <td>\" \\n :I have not initially protected the templ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>0.000371</td>\n",
       "      <td>0.000582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>28920</td>\n",
       "      <td>301dcc7eb453cc2c</td>\n",
       "      <td>==BUK is not rocket shell () shell be heard fo...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>0.000250</td>\n",
       "      <td>0.000482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>127658</td>\n",
       "      <td>d551c0c0492daf90</td>\n",
       "      <td>== Asako Tatu　Kazumi == \\n\\n 　　　　　　　　 \\n たすけてい...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>0.000200</td>\n",
       "      <td>0.121299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>90747</td>\n",
       "      <td>9753a1825493b26a</td>\n",
       "      <td>\" \\n *******Pray tell, what would be the \"\"nex...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>0.002253</td>\n",
       "      <td>0.000545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>23741</td>\n",
       "      <td>27969c379ce89f67</td>\n",
       "      <td>:If you believe that Image:Michael&amp;fan.jpg; be...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>0.000278</td>\n",
       "      <td>0.000646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>117412</td>\n",
       "      <td>c3fb146fe589bd0b</td>\n",
       "      <td>\" \\n\\n == Why Safe Water Drinking Act is relev...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>No.</td>\n",
       "      <td>No</td>\n",
       "      <td>0.000279</td>\n",
       "      <td>0.000425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>17362</td>\n",
       "      <td>1d25a41d38c55e09</td>\n",
       "      <td>For fuck sake, the article should not be consi...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Yes.</td>\n",
       "      <td>No</td>\n",
       "      <td>0.008995</td>\n",
       "      <td>0.118039</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>865 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index                id  \\\n",
       "1      7366  0c65981f8c1e26fd   \n",
       "2    103978  ad89eccb5456659c   \n",
       "3      6858  0b8abd149f7da6b1   \n",
       "5     93019  9b2506a652ec76cd   \n",
       "6     28920  301dcc7eb453cc2c   \n",
       "..      ...               ...   \n",
       "995  127658  d551c0c0492daf90   \n",
       "996   90747  9753a1825493b26a   \n",
       "997   23741  27969c379ce89f67   \n",
       "998  117412  c3fb146fe589bd0b   \n",
       "999   17362  1d25a41d38c55e09   \n",
       "\n",
       "                                          comment_text  toxic  severe_toxic  \\\n",
       "1    == Disney Kids == \\n\\n The Disney Kids rebrand...      1             0   \n",
       "2    I rather disagree with some of the actions by ...      0             0   \n",
       "3    == OBL == \\n\\n That's right, its Osama bin Lat...      0             0   \n",
       "5    \" \\n :I have not initially protected the templ...      0             0   \n",
       "6    ==BUK is not rocket shell () shell be heard fo...      0             0   \n",
       "..                                                 ...    ...           ...   \n",
       "995  == Asako Tatu　Kazumi == \\n\\n 　　　　　　　　 \\n たすけてい...      0             0   \n",
       "996  \" \\n *******Pray tell, what would be the \"\"nex...      0             0   \n",
       "997  :If you believe that Image:Michael&fan.jpg; be...      0             0   \n",
       "998  \" \\n\\n == Why Safe Water Drinking Act is relev...      0             0   \n",
       "999  For fuck sake, the article should not be consi...      1             0   \n",
       "\n",
       "     obscene  threat  insult  identity_hate  GPT3 GPT4   RoBERTa       BoW  \n",
       "1          0       0       0              0  Yes.   No  0.581859  0.659417  \n",
       "2          0       0       0              0   No.   No  0.000229  0.000362  \n",
       "3          0       0       0              0  Yes.  Yes  0.395430  0.002460  \n",
       "5          0       0       0              0   No.   No  0.000371  0.000582  \n",
       "6          0       0       0              0   No.   No  0.000250  0.000482  \n",
       "..       ...     ...     ...            ...   ...  ...       ...       ...  \n",
       "995        0       0       0              0   No.   No  0.000200  0.121299  \n",
       "996        0       0       0              0   No.   No  0.002253  0.000545  \n",
       "997        0       0       0              0   No.   No  0.000278  0.000646  \n",
       "998        0       0       0              0   No.   No  0.000279  0.000425  \n",
       "999        1       0       1              0  Yes.   No  0.008995  0.118039  \n",
       "\n",
       "[865 rows x 13 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def run_toxic():\n",
    "    cols = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']\n",
    "    for idx, tab in enumerate(cols):\n",
    "        df = pd.read_csv(f\"chatgpt-responses/toxicity_{tab}.csv\", index_col=0)\n",
    "        df = filter_long_texts(df, \"comment_text\")\n",
    "        true = np.array(load(f\"results/outputs_toxicity_RoBERTa.json\"))\n",
    "        assert np.all(np.size(x) == 6 for x in true)\n",
    "        assert np.mean(np.abs(df[cols].values.astype(np.int32) - true[:, 0, :])) < 1e-8\n",
    "        df['RoBERTa'] = np.array(load(f\"results/predictions_toxicity_RoBERTa.json\"))[:, idx]\n",
    "        df['BoW'] = np.array(load(f\"results/predictions_toxicity_BoW.json\"))[:,  idx]\n",
    "        df, shp, gpt3_pred, gpt4_pred = read_results(None, [\"Yes\"], [\"No\"], df=df)\n",
    "        roberta_pred = (df['RoBERTa'].values > 0.5).astype(np.int32)\n",
    "        bow_pred = (df['BoW'].values > 0.5).astype(np.int32)\n",
    "        gt = df[tab].astype(np.int32)\n",
    "        latex_table.append(build_row(tab.strip(), gt, bow_pred, roberta_pred, gpt3_pred, gpt4_pred))\n",
    "        print(latex_table[-1])\n",
    "        print('---')\n",
    "    return df\n",
    "\n",
    "run_toxic()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "290fe50d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "&   Engagement  &  $        51.92   $  &  $        71.02^{**} $  &  $\\textbf{79.18}^{**} $  &  $        54.15^{**} $  &  $        51.85   $  &  $        71.02^{**} $  &  $\\textbf{79.19}^{**} $  &  $        53.80^{**} $  \\\\\n",
      "&    reddit  &  $        75.73   $  &  $        30.83^{**} $  &  $\\textbf{94.58}^{**} $  &  $        90.10^{**} $  &  $        80.46   $  &  $        54.42^{**} $  &  $\\textbf{84.88}      $  &  $        83.73      $  \\\\\n",
      "&    reddit body  &  $        91.93   $  &  $        84.50^{**} $  &  $        89.88      $  &  $\\textbf{93.33}      $  &  $        84.41   $  &  $        68.82^{**} $  &  $\\textbf{86.16}      $  &  $        78.63^{**} $  \\\\\n",
      "&    reddit titles  &  $        80.61   $  &  $        86.60^{**} $  &  $\\textbf{96.75}^{**} $  &  $        89.54^{**} $  &  $        80.05   $  &  $        85.62^{**} $  &  $\\textbf{96.65}^{**} $  &  $        89.77^{**} $  \\\\\n",
      "&    twitter  &  $        60.53   $  &  $        43.36^{**} $  &  $\\textbf{93.23}^{**} $  &  $        72.31^{**} $  &  $        65.05   $  &  $        45.45^{**} $  &  $\\textbf{93.14}^{**} $  &  $        73.45^{**} $  \\\\\n",
      "&    twitter full  &  $        66.24   $  &  $        80.39^{**} $  &  $\\textbf{84.39}^{**} $  &  $        75.25^{**} $  &  $        66.26   $  &  $        80.39^{**} $  &  $\\textbf{84.39}^{**} $  &  $        75.25^{**} $  \\\\\n",
      "&   Joy  &  $        74.07   $  &  $        66.49^{**} $  &  $        75.41      $  &  $\\textbf{78.46}^{**} $  &  $        74.29   $  &  $        66.51^{**} $  &  $        75.40      $  &  $\\textbf{78.63}^{**} $  \\\\\n",
      "&   Fear  &  $        72.76   $  &  $        68.65^{**} $  &  $\\textbf{76.83}^{**} $  &  $        73.96      $  &  $        72.86   $  &  $        68.65^{**} $  &  $\\textbf{76.83}^{**} $  &  $        74.09      $  \\\\\n",
      "&   Anger  &  $        72.12   $  &  $        67.63^{**} $  &  $        73.47      $  &  $\\textbf{75.58}^{**} $  &  $        72.09   $  &  $        67.60^{**} $  &  $        73.46      $  &  $\\textbf{75.49}^{**} $  \\\\\n",
      "&   Sadness  &  $        78.19   $  &  $        72.41^{**} $  &  $        76.06      $  &  $\\textbf{78.55}      $  &  $        78.22   $  &  $        72.40^{**} $  &  $        76.05      $  &  $\\textbf{78.58}      $  \\\\\n",
      "&   Openness  &  $        50.11   $  &  $        58.36^{**} $  &  $\\textbf{60.54}^{**} $  &  $        54.75^{**} $  &  $        50.60   $  &  $        58.36^{**} $  &  $\\textbf{60.56}^{**} $  &  $        54.59^{**} $  \\\\\n",
      "&   Conscientiousness  &  $        55.54   $  &  $        56.79      $  &  $\\textbf{61.59}^{**} $  &  $        57.44^*    $  &  $        55.84   $  &  $        56.78      $  &  $\\textbf{61.59}^{**} $  &  $        57.33      $  \\\\\n",
      "&   Extraversion  &  $        53.55   $  &  $        56.51^{**} $  &  $\\textbf{59.03}^{**} $  &  $        55.90^{**} $  &  $        53.38   $  &  $        56.51^{**} $  &  $\\textbf{59.02}^{**} $  &  $        55.90^{**} $  \\\\\n",
      "&   Agreeableness  &  $        51.67   $  &  $        57.81^{**} $  &  $\\textbf{58.12}^{**} $  &  $        54.04^{**} $  &  $        52.10   $  &  $        57.80^{**} $  &  $\\textbf{58.14}^{**} $  &  $        54.05^*    $  \\\\\n",
      "&   Neuroticism  &  $        48.94   $  &  $        58.60^{**} $  &  $\\textbf{59.86}^{**} $  &  $        49.68      $  &  $        49.04   $  &  $        58.60^{**} $  &  $\\textbf{59.86}^{**} $  &  $        49.73      $  \\\\\n",
      "&   Sentiment microblogs  &  $        69.30   $  &  $        70.88      $  &  $        72.37      $  &  $\\textbf{73.21}^{**} $  &  $        68.69   $  &  $        70.83      $  &  $        72.41^*    $  &  $\\textbf{73.08}^{**} $  \\\\\n",
      "&   Sarcasm  &  $        59.13   $  &  $        63.14^{**} $  &  $\\textbf{90.66}^{**} $  &  $        66.66^{**} $  &  $        56.82   $  &  $        66.29^{**} $  &  $\\textbf{90.70}^{**} $  &  $        69.45^{**} $  \\\\\n",
      "&   Sentiment  &  $        80.54   $  &  $        78.87      $  &  $\\textbf{88.74}^{**} $  &  $        84.09^{**} $  &  $        79.93   $  &  $        78.84      $  &  $\\textbf{88.75}^{**} $  &  $        83.69^{**} $  \\\\\n",
      "&   Suicide  &  $        89.46   $  &  $        84.75^{**} $  &  $\\textbf{98.43}^{**} $  &  $        93.46^{**} $  &  $        89.44   $  &  $        85.32^{**} $  &  $\\textbf{98.46}^{**} $  &  $        93.32^{**} $  \\\\\n",
      "&   Subjectivity  &  $        59.56   $  &  $        87.28^{**} $  &  $\\textbf{95.56}^{**} $  &  $        88.38^{**} $  &  $        58.59   $  &  $        87.19^{**} $  &  $\\textbf{95.51}^{**} $  &  $        88.19^{**} $  \\\\\n",
      "&   Toxic  &  $        87.37   $  &  $        81.85^{**} $  &  $        85.23      $  &  $\\textbf{89.29}      $  &  $        87.19   $  &  $        82.75^{**} $  &  $        86.01      $  &  $\\textbf{89.65}^*    $  \\\\\n",
      "&   Severe toxic  &  $        66.55   $  &  $\\textbf{87.65}^{**} $  &  $        80.07^{**} $  &  $        75.52^{**} $  &  $        80.65   $  &  $        82.48      $  &  $        84.78^*    $  &  $\\textbf{85.29}^{**} $  \\\\\n",
      "&   Obscene  &  $        83.45   $  &  $        85.40      $  &  $        84.83      $  &  $\\textbf{88.16}^{**} $  &  $        83.48   $  &  $        86.62^*    $  &  $        86.60^*    $  &  $\\textbf{86.78}^*    $  \\\\\n",
      "&   Threat  &  $        70.59   $  &  $        94.05^{**} $  &  $\\textbf{95.54}^{**} $  &  $        91.99^{**} $  &  $        80.12   $  &  $        73.99      $  &  $        87.43^{**} $  &  $\\textbf{91.51}^{**} $  \\\\\n",
      "&   Insult  &  $        80.14   $  &  $        84.65^{**} $  &  $\\textbf{87.25}^{**} $  &  $        80.70      $  &  $        83.21   $  &  $        84.64      $  &  $\\textbf{89.28}^{**} $  &  $        84.89      $  \\\\\n",
      "&   Identity hate  &  $        66.82   $  &  $        90.52^{**} $  &  $\\textbf{90.98}^{**} $  &  $        82.66^{**} $  &  $        78.61   $  &  $        81.61      $  &  $        86.16^{**} $  &  $\\textbf{87.88}^{**} $  \\\\\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n\".join(latex_table))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1b03212",
   "metadata": {},
   "source": [
    "manual = [\n",
    "    (\"frustration\", \"may feel frustrated\", \"frustrated\"),\n",
    "    ('weight', 's heavy for mac book', 'heavy'),\n",
    "    ('built-in virus control', 'quick and has built in virus control', 'built in virus control'),\n",
    "    ('iphone', 'as the iphone/ipod touch', 'iphone/ipod'),\n",
    "    (\"ipod touch\", \"as the iphone/ipod touch\", 'iphone/ipod touch'),\n",
    "    ('multi-component', 'i was told that it seems to be a multi - component failure', 'multi - component'),\n",
    "    ('tech-savvy', \"being a tech savvy , apple - product loving person , i 'm glad i finally got the macbook pro !\", 'tech savvy'),\n",
    "    (\"apple-product loving\", 'apple - product loving person , i ', 'apple - product loving'),\n",
    "    ('self-ejecting slot',\n",
    "  'the sd card reader is slightly recessed and upside down ( the nail slot on the card can not be accessed ) , if this was a self ejecting slot this would not be an issue , but its not', 'self ejecting slot'),\n",
    "    ('- packing', 'nice packing', 'packing'),\n",
    "    ('built-in screen size', \"at home and the office it gets plugged into an external 24 '' lcd screen , so built in screen size is not terribly important\", 'built in screen size'),\n",
    "    ('home', \"the only thing i miss are the home/end '' type keys and other things that i grew accustomed to after so long\", 'home/end'),\n",
    "    ('ultrabooks', 'i found it toughest to decide between dell ultra books and apple', 'ultra books'),\n",
    "    ('laptop', 'great deal on an amazing lap top !', 'lap top'),\n",
    "    ('darkness', 'the atmosphere was nice but it was a little too dark', 'dark'),\n",
    "    ('lunch', \"went for mom 's birthday brunch/lunch\", 'brunch/lunch'),\n",
    "    ('overpriced', \"ca n't argue about that , but they are clearly over priced\", \"over priced\"),\n",
    "    ('portobello', 'had a great experience at trio staff was pleasant food was tasty and large in portion size - i would highly recommend the portobello/gorgonzola/sausage appetizer and the lobster risotto', 'portobello/gorgonzola/sausage'),\n",
    "    ('gorgonzola', 'had a great experience at trio staff was pleasant food was tasty and large in portion size - i would highly recommend the portobello/gorgonzola/sausage appetizer and the lobster risotto', 'portobello/gorgonzola/sausage'),\n",
    "\n",
    "    ('chinese-style indian food', 'not a very fancy place but very good chinese style indian food', 'chinese style indian food'),\n",
    "    ('restaurant', 'i stumbled upon this resteraunt on my way home from the subway', 'resteraunt'),\n",
    "    ('overpriced', 'over priced , overrated away !', 'over priced'),\n",
    "    ('mouth-watering', \"el nidos one of the best restaurants in new york which i 've ever been to , has a great variety of tasty , mouth watering pizza 's\", 'mouth watering'),\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "535e6af1",
   "metadata": {},
   "outputs": [],
   "source": [
    "manual = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "3b16bc3a",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import functools\n",
    "bad = []\n",
    "def is_similar(word1, word2, tolerance=2):\n",
    "    import Levenshtein\n",
    "\n",
    "    distance = Levenshtein.distance(word1, word2)\n",
    "    return distance <= tolerance\n",
    "\n",
    "def extract(point):\n",
    "    separators = [\" is \", \" are \", \": \"]\n",
    "    if all(sep not in point for sep in separators):\n",
    "        #print(\"No separators\", point, '-----------')\n",
    "        return point, None\n",
    "    if point.startswith('* '):\n",
    "        point = point[2:]\n",
    "\n",
    "    for sep in separators:\n",
    "        if sep in point:\n",
    "            parts = point.split(sep)\n",
    "            expr = sep.join(parts[:-1]).strip()\n",
    "            break\n",
    "    if expr.startswith('\"') or expr.startswith(\"'\"):\n",
    "        expr = expr[1:]\n",
    "    if expr.endswith('\"') or expr.endswith(\"'\"):\n",
    "        expr = expr[:-1]\n",
    "    \n",
    "    label = parts[-1]\n",
    "    if label.endswith('.'):\n",
    "        label = label[:-1]\n",
    "    return expr, label.strip()\n",
    "\n",
    "def get_index(expr, words):\n",
    "    subwords = expr.lower().split()\n",
    "\n",
    "    k = len(subwords)\n",
    "    for i in range(len(words)):\n",
    "        if words[i: i + k] == subwords:\n",
    "            return list(range(i, i + k))\n",
    "    for i in range(len(words)):\n",
    "        if is_similar(words[i], subwords[0]):\n",
    "            return [i]\n",
    "\n",
    "    for a, b, c in manual:\n",
    "        if a == expr and b in \" \".join(words):\n",
    "            return get_index(c, words)\n",
    "    bad.append((expr, \" \".join(words)))\n",
    "#     print(f'\"{expr}\" not found, {words}  <<==============\\n\\n')\n",
    "    return []\n",
    "\n",
    "\n",
    "def fn(row, typ, col):\n",
    "\n",
    "    chatgpt = row[col]\n",
    "    words = row[\"text\"].lower().split()\n",
    "    # print(chatgpt.replace(\"\\n\", \" \") , '----')\n",
    "    labels = [0 for _ in words]\n",
    "    if chatgpt.strip() == \"BACKGROUND\":\n",
    "        # print(row[\"text\"], \"All background -----------\")\n",
    "        return \" \".join(map(str, labels))\n",
    "    outs = list(map(extract, chatgpt.split(\"\\n\")))\n",
    "    for expr, label in outs:\n",
    "        if expr == \"BACKGROUND\" or label is None:\n",
    "            continue\n",
    "        idxs = get_index(expr, words)\n",
    "        if not idxs:\n",
    "            return None\n",
    "        output = {\"positive\": 1, \"negative\": 2, \"neutral\": 3, \"conflict\": 4}.get(label.lower(), -1)\n",
    "\n",
    "        labels[idxs[0]] = output if typ == \"polarity\" else 1\n",
    "        \n",
    "        for i in idxs[1:]:\n",
    "            labels[i] = output if typ == \"polarity\" else 2\n",
    "    return \" \".join(map(str, labels))\n",
    "\n",
    "\n",
    "def read_chatgpt_aspects(tab):\n",
    "    df = pd.read_csv(f'chatgpt-responses/{tab}.csv', index_col=0)[:-1]\n",
    "    df['GPT3-target'] = df.apply(functools.partial(fn, typ=\"target\", col=\"GPT3\"), axis=1)\n",
    "    df['GPT3-polarity'] = df.apply(functools.partial(fn, typ=\"polarity\", col=\"GPT3\"), axis=1)\n",
    "    df['GPT4-target'] = df.apply(functools.partial(fn, typ=\"target\", col=\"GPT4\"), axis=1)\n",
    "    df['GPT4-polarity'] = df.apply(functools.partial(fn, typ=\"polarity\", col=\"GPT4\"), axis=1)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "26cedaac",
   "metadata": {},
   "outputs": [],
   "source": [
    "badop = []\n",
    "def extractop(expr):\n",
    "    if expr.startswith('* ') or expr.startswith(\"- \"):\n",
    "        expr = expr[2:]\n",
    "\n",
    "    if expr.startswith('\"') or expr.startswith(\"'\"):\n",
    "        expr = expr[1:]\n",
    "    if expr.endswith('\"') or expr.endswith(\"'\"):\n",
    "        expr = expr[:-1]\n",
    "    \n",
    "    return expr.strip()\n",
    "\n",
    "def get_index_op(expr, words):\n",
    "    subwords = expr.lower().split()\n",
    "\n",
    "    k = len(subwords)\n",
    "    for i in range(len(words)):\n",
    "        if words[i: i + k] == subwords:\n",
    "            return list(range(i, i + k))\n",
    "    for i in range(len(words)):\n",
    "        if is_similar(words[i], subwords[0]):\n",
    "            return [i]\n",
    "\n",
    "#     for a, b, c in manual:\n",
    "#         if a == expr and b in \" \".join(words):\n",
    "#             return get_index(c, words)\n",
    "    badop.append((expr, \" \".join(words)))\n",
    "#     print(f'\"{expr}\" not found, {words}  <<==============\\n\\n')\n",
    "    return []\n",
    "\n",
    "\n",
    "def fnop(row, col):\n",
    "\n",
    "    chatgpt = row[col]\n",
    "    words = row[\"text\"].lower().split()\n",
    "    labels = [0 for _ in words]\n",
    "    \n",
    "    outs = list(map(extractop, chatgpt.split(\"\\n\")))\n",
    "    for expr in outs:\n",
    "        if expr.startswith(\"BACKGROUND\") or expr.startswith(\"None\"):\n",
    "            return \" \".join(map(str, labels))\n",
    "        idxs = get_index_op(expr, words)\n",
    "        if not idxs:\n",
    "            return None\n",
    "        labels[idxs[0]] = 1\n",
    "        \n",
    "        for i in idxs[1:]:\n",
    "            labels[i] = 2\n",
    "    return \" \".join(map(str, labels))\n",
    "\n",
    "\n",
    "def read_chatgpt_aspects_opinion(tab):\n",
    "    df = pd.read_csv(f'chatgpt-responses/aspect_{tab}_opinion.csv', index_col=0)[:-1]\n",
    "    df['GPT3-opinion'] = df.apply(functools.partial(fnop, col=\"GPT3\"), axis=1)\n",
    "    df['GPT4-opinion'] = df.apply(functools.partial(fnop, col=\"GPT4\"), axis=1)\n",
    "    # df['ChatGPT-polarity'] = df.apply(functools.partial(fn, typ=\"polarity\"), axis=1)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ac228cae",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def flatten(array2d, mask=None):\n",
    "    if mask is None:\n",
    "        return np.array([item for array1d in array2d for item in array1d])\n",
    "    else:\n",
    "        assert len(mask) == len(array2d)\n",
    "        return np.array([item for take, array1d in zip(mask, array2d) if take for item in array1d])\n",
    "    \n",
    "invert = lambda arr, inv: [arr[idx] for idx in inv]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e30b44b7",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "800 filtered examples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at roberta-base were not used when initializing TFRobertaModel: ['lm_head']\n",
      "- This IS expected if you are initializing TFRobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFRobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFRobertaModel were initialized from the model checkpoint at roberta-base.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaModel for predictions without further training.\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 13/13 [00:21<00:00,  1.65s/it]\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 1/1 [00:00<00:00, 169.62it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "800 filtered examples...\n",
      "/outputs/chatgpt_evaluations/iteration_3/encoders/aspect_res14_target_BoW.json already exists...\n",
      "target res14\n",
      "Examples 11641\n",
      "  BoW  :  81.7284   81.1773\n",
      "RoBERTa:  92.0024   91.5420\n",
      " GPT3  :  86.9513   76.1823\n",
      " GPT4  :  71.4973   73.2296\n",
      "\n",
      "\n",
      "&   Target res14  &  $        86.95   $  &  $        81.73^{**} $  &  $\\textbf{92.00}^{**} $  &  $        71.50^{**} $  &  $        76.18   $  &  $        81.18^{**} $  &  $\\textbf{91.54}^{**} $  &  $        73.23^{**} $  \\\\\n",
      "800 filtered examples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at roberta-base were not used when initializing TFRobertaModel: ['lm_head']\n",
      "- This IS expected if you are initializing TFRobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFRobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFRobertaModel were initialized from the model checkpoint at roberta-base.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaModel for predictions without further training.\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 13/13 [00:22<00:00,  1.75s/it]\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 1/1 [00:00<00:00, 184.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "800 filtered examples...\n",
      "/outputs/chatgpt_evaluations/iteration_3/encoders/aspect_lap14_target_BoW.json already exists...\n",
      "target lap14\n",
      "Examples 10687\n",
      "  BoW  :  78.2165   82.2146\n",
      "RoBERTa:  87.1900   90.6439\n",
      " GPT3  :  84.5981   77.6244\n",
      " GPT4  :  70.3191   75.9428\n",
      "\n",
      "\n",
      "&   Target lap14  &  $        84.60   $  &  $        78.22^{**} $  &  $\\textbf{87.19}^{**} $  &  $        70.32^{**} $  &  $        77.62   $  &  $        82.21^{**} $  &  $\\textbf{90.64}^{**} $  &  $        75.94      $  \\\\\n",
      "685 filtered examples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at roberta-base were not used when initializing TFRobertaModel: ['lm_head']\n",
      "- This IS expected if you are initializing TFRobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFRobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFRobertaModel were initialized from the model checkpoint at roberta-base.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaModel for predictions without further training.\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 11/11 [00:22<00:00,  2.05s/it]\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 1/1 [00:00<00:00, 209.56it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "685 filtered examples...\n",
      "/outputs/chatgpt_evaluations/iteration_3/encoders/aspect_res15_target_BoW.json already exists...\n",
      "target res15\n",
      "Examples 9682\n",
      "  BoW  :  81.2849   72.2815\n",
      "RoBERTa:  73.0221   85.1277\n",
      " GPT3  :  84.5693   78.5623\n",
      " GPT4  :  70.0475   73.8118\n",
      "\n",
      "\n",
      "&   Target res15  &  $\\textbf{84.57}   $  &  $        81.28^{**} $  &  $        73.02^{**} $  &  $        70.05^{**} $  &  $        78.56   $  &  $        72.28^{**} $  &  $\\textbf{85.13}^{**} $  &  $        73.81^{**} $  \\\\\n",
      "800 filtered examples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at roberta-base were not used when initializing TFRobertaModel: ['lm_head']\n",
      "- This IS expected if you are initializing TFRobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFRobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFRobertaModel were initialized from the model checkpoint at roberta-base.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaModel for predictions without further training.\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 13/13 [00:41<00:00,  3.17s/it]\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 1/1 [00:00<00:00, 178.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "800 filtered examples...\n",
      "/outputs/chatgpt_evaluations/iteration_3/encoders/aspect_res14_polarity_BoW.json already exists...\n",
      "polarity res14\n",
      "Examples 11641\n",
      "  BoW  :  86.1009   49.7182\n",
      "RoBERTa:  71.8495   58.0942\n",
      " GPT3  :  85.1301   49.9649\n",
      " GPT4  :  69.2982   48.2570\n",
      "\n",
      "\n",
      "&   Polarity res14  &  $        85.13   $  &  $\\textbf{86.10}^*    $  &  $        71.85^{**} $  &  $        69.30^{**} $  &  $        49.96   $  &  $        49.72      $  &  $\\textbf{58.09}^*    $  &  $        48.26      $  \\\\\n",
      "800 filtered examples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at roberta-base were not used when initializing TFRobertaModel: ['lm_head']\n",
      "- This IS expected if you are initializing TFRobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFRobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFRobertaModel were initialized from the model checkpoint at roberta-base.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaModel for predictions without further training.\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 13/13 [00:23<00:00,  1.79s/it]\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 1/1 [00:00<00:00, 174.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "800 filtered examples...\n",
      "/outputs/chatgpt_evaluations/iteration_3/encoders/aspect_lap14_polarity_BoW.json already exists...\n",
      "polarity lap14\n",
      "Examples 10687\n",
      "  BoW  :  72.5742   45.1024\n",
      "RoBERTa:  90.2218   58.2251\n",
      " GPT3  :  82.2307   47.3737\n",
      " GPT4  :  67.6336   44.5410\n",
      "\n",
      "\n",
      "&   Polarity lap14  &  $        82.23   $  &  $        72.57^{**} $  &  $\\textbf{90.22}^{**} $  &  $        67.63^{**} $  &  $        47.37   $  &  $        45.10      $  &  $\\textbf{58.23}^{**} $  &  $        44.54^*    $  \\\\\n",
      "685 filtered examples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at roberta-base were not used when initializing TFRobertaModel: ['lm_head']\n",
      "- This IS expected if you are initializing TFRobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFRobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFRobertaModel were initialized from the model checkpoint at roberta-base.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaModel for predictions without further training.\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 11/11 [00:22<00:00,  2.06s/it]\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 1/1 [00:00<00:00, 216.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "685 filtered examples...\n",
      "/outputs/chatgpt_evaluations/iteration_3/encoders/aspect_res15_polarity_BoW.json already exists...\n",
      "polarity res15\n",
      "Examples 9682\n",
      "  BoW  :  79.0849   36.8353\n",
      "RoBERTa:  84.3111   48.9593\n",
      " GPT3  :  82.3797   47.7933\n",
      " GPT4  :  67.5067   44.2664\n",
      "\n",
      "\n",
      "&   Polarity res15  &  $        82.38   $  &  $        79.08^{**} $  &  $\\textbf{84.31}^{**} $  &  $        67.51^{**} $  &  $        47.79   $  &  $        36.84^{**} $  &  $\\textbf{48.96}      $  &  $        44.27      $  \\\\\n",
      "800 filtered examples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at roberta-base were not used when initializing TFRobertaModel: ['lm_head']\n",
      "- This IS expected if you are initializing TFRobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFRobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFRobertaModel were initialized from the model checkpoint at roberta-base.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaModel for predictions without further training.\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 13/13 [00:41<00:00,  3.21s/it]\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 1/1 [00:00<00:00, 183.25it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "800 filtered examples...\n",
      "/outputs/chatgpt_evaluations/iteration_3/encoders/aspect_res14_opinion_BoW.json already exists...\n",
      "opinion res14\n",
      "Examples 10932\n",
      "  BoW  :  81.6136   80.8696\n",
      "RoBERTa:  93.2583   80.0640\n",
      " GPT3  :  91.0446   77.2827\n",
      " GPT4  :  80.9276   83.0222\n",
      "\n",
      "\n",
      "&   Opinion res14  &  $        91.04   $  &  $        81.61^{**} $  &  $\\textbf{93.26}^{**} $  &  $        80.93^{**} $  &  $        77.28   $  &  $        80.87      $  &  $        80.06      $  &  $\\textbf{83.02}^*    $  \\\\\n",
      "800 filtered examples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at roberta-base were not used when initializing TFRobertaModel: ['lm_head']\n",
      "- This IS expected if you are initializing TFRobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFRobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFRobertaModel were initialized from the model checkpoint at roberta-base.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaModel for predictions without further training.\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 13/13 [00:23<00:00,  1.81s/it]\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 1/1 [00:00<00:00, 175.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "800 filtered examples...\n",
      "/outputs/chatgpt_evaluations/iteration_3/encoders/aspect_lap14_opinion_BoW.json already exists...\n",
      "opinion lap14\n",
      "Examples 10411\n",
      "  BoW  :  74.3348   66.4323\n",
      "RoBERTa:  73.8066   76.7258\n",
      " GPT3  :  89.4343   73.5071\n",
      " GPT4  :  76.8994   77.4245\n",
      "\n",
      "\n",
      "&   Opinion lap14  &  $\\textbf{89.43}   $  &  $        74.33^{**} $  &  $        73.81^{**} $  &  $        76.90^{**} $  &  $        73.51   $  &  $        66.43^*    $  &  $        76.73      $  &  $\\textbf{77.42}      $  \\\\\n",
      "685 filtered examples...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some layers from the model checkpoint at roberta-base were not used when initializing TFRobertaModel: ['lm_head']\n",
      "- This IS expected if you are initializing TFRobertaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFRobertaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the layers of TFRobertaModel were initialized from the model checkpoint at roberta-base.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaModel for predictions without further training.\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 11/11 [00:21<00:00,  1.93s/it]\n",
      "100%|▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅| 1/1 [00:00<00:00, 215.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "685 filtered examples...\n",
      "/outputs/chatgpt_evaluations/iteration_3/encoders/aspect_res15_opinion_BoW.json already exists...\n",
      "opinion res15\n",
      "Examples 9129\n",
      "  BoW  :  79.4172   68.5929\n",
      "RoBERTa:  89.1554   77.3132\n",
      " GPT3  :  89.3198   76.8998\n",
      " GPT4  :  78.1027   77.1872\n",
      "\n",
      "\n",
      "&   Opinion res15  &  $\\textbf{89.32}   $  &  $        79.42^{**} $  &  $        89.16      $  &  $        78.10^{**} $  &  $        76.90   $  &  $        68.59      $  &  $\\textbf{77.31}      $  &  $        77.19      $  \\\\\n"
     ]
    }
   ],
   "source": [
    "for target_type in [\"target\", \"polarity\", \"opinion\"]:\n",
    "    for lap in [\"res14\", \"lap14\", \"res15\"]:\n",
    "        tab = f\"aspect_{lap}_{target_type}\"\n",
    "        ## RoBERTa\n",
    "        texts, subword_labels, inverse_index, word_labels = list(zip(*list(\n",
    "            generate_examples(tab, \"test\", \"RoBERTa\")\n",
    "        )))\n",
    "        true = load(f\"results/outputs_{tab}_RoBERTa.json\")[:-1]  # TODO: fix this if reran\n",
    "        assert list(subword_labels) == [x[0] for x in true], (subword_labels[:5], [x[0] for x in true[:5]])\n",
    "        _roberta_pred = load(f\"results/predictions_{tab}_RoBERTa.json\")\n",
    "        \n",
    "        roberta_pred = []\n",
    "        for x, inv in zip(_roberta_pred, inverse_index):\n",
    "            r = np.argmax(x, axis=-1).tolist()\n",
    "            roberta_pred.append(invert(r, inv))\n",
    "        # gt_flat = flatten(word_labels)\n",
    "        #roberta_pred_flat = flatten(roberta_pred)\n",
    "        \n",
    "        ## BoW\n",
    "        _texts, _word_labels = list(zip(*list(generate_examples(tab, \"test\", \"BoW\"))))\n",
    "        assert word_labels == _word_labels\n",
    "        true = load(f\"results/outputs_{tab}_BoW.json\")\n",
    "        true = [batch[0] for batch in true]\n",
    "        assert list(word_labels) == true\n",
    "        bow_pred = load(f\"results/predictions_{tab}_BoW.json\")\n",
    "        bow_pred = [np.argmax(x, axis=-1) for x in bow_pred]\n",
    "        #bow_pred_flat = flatten(bow_pred)\n",
    "        \n",
    "\n",
    "        if target_type == \"opinion\":\n",
    "            ## ChatGPT\n",
    "            df = read_chatgpt_aspects_opinion(f\"{lap}\")\n",
    "            _true = np.array([int(x) for labels in df['opinion'] for x in labels.split()])\n",
    "            assert len(true) == df.shape[0]\n",
    "            assert [\" \".join(map(str, x)) for x in true] == df[target_type].tolist()\n",
    "#             assert (_true == gt_flat).all()\n",
    "            msk = [(x is not None and y is not None) for x, y in zip(df[f\"GPT3-opinion\"].tolist(), df[f\"GPT4-opinion\"].tolist())]\n",
    "            gpt3_flat = np.array([int(x) for labels in df[msk][f\"GPT3-opinion\"] for x in labels.split()])\n",
    "            gpt4_flat = np.array([int(x) for labels in df[msk][f\"GPT4-opinion\"] for x in labels.split()])\n",
    "            true_flat = np.array([int(x) for labels in df[msk]['opinion'] for x in labels.split()])\n",
    "            roberta_pred_flat = flatten(roberta_pred, msk)\n",
    "            bow_pred_flat = flatten(bow_pred, msk)\n",
    "            _true = flatten(true, msk)\n",
    "            assert np.all(_true == true_flat)\n",
    "            gt_flat = true_flat\n",
    "            \n",
    "#             print(\"RoBERTa %30s %5d %3d,\\tAcc: %.4f , UAR: %.4f\\n\" % (tab, len(roberta_pred), len(gt_flat), ACC(gt_flat, roberta_pred_flat), UAR(gt_flat, roberta_pred_flat)))\n",
    "#             print(\"BoW     %30s %5d %3d,\\tAcc: %.4f , UAR: %.4f\\n\" % (tab, len(bow_pred), len(gt_flat), ACC(gt_flat, bow_pred_flat), UAR(gt_flat, bow_pred_flat)))\n",
    "#             print(\"ChatGPT %30s %5d %3d,\\tAcc: %.4f , UAR: %.4f\\n\" % (tab, len(df), len(true_flat), ACC(true_flat, chatgpt_flat), UAR(true_flat, chatgpt_flat)))\n",
    "#             latex_table.append(\"%8s  &  %8s  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  \\\\\\\\\" % (\n",
    "#                 target_type.capitalize(), lap,\n",
    "#                 ACC(gt_flat, bow_pred_flat),\n",
    "#                 ACC(gt_flat, roberta_pred_flat),\n",
    "#                 ACC(true_flat, chatgpt_flat),\n",
    "#                 UAR(gt_flat, bow_pred_flat),\n",
    "#                 UAR(gt_flat, roberta_pred_flat),\n",
    "#                 UAR(true_flat, chatgpt_flat),\n",
    "#             ))\n",
    "            # opinion\n",
    "            latex_table.append(build_row(f\"{target_type} {lap}\", gt_flat, bow_pred_flat, roberta_pred_flat, gpt3_flat, gpt4_flat))\n",
    "            print(latex_table[-1])\n",
    "\n",
    "        else:\n",
    "\n",
    "            ## ChatGPT\n",
    "            df = read_chatgpt_aspects(f\"aspect_{lap}\")\n",
    "            assert len(true) == df.shape[0]\n",
    "            assert [\" \".join(map(str, x)) for x in true] == df[{\"polarity\": \"target_polarity\", \"target\": \"target\"}[target_type]].tolist()\n",
    "            _true = np.array([int(x) for labels in df[{\"polarity\": \"target_polarity\", \"target\": \"target\"}[target_type]] for x in labels.split()])\n",
    "#             assert (_true == gt_flat).all()\n",
    "#             msk = [x is not None for x in df[f\"GPT3-{target_type}\"].tolist()]\n",
    "            msk = [(x is not None and y is not None) for x, y in zip(df[f\"GPT3-{target_type}\"].tolist(), df[f\"GPT4-{target_type}\"].tolist())]\n",
    "            gpt3_flat = np.array([int(x) for labels in df[msk][f\"GPT3-{target_type}\"] for x in labels.split()])\n",
    "            gpt4_flat = np.array([int(x) for labels in df[msk][f\"GPT4-{target_type}\"] for x in labels.split()])\n",
    "            true_flat = np.array([int(x) for labels in df[msk][{\"polarity\": \"target_polarity\", \"target\": \"target\"}[target_type]] for x in labels.split()])\n",
    "            roberta_pred_flat = flatten(roberta_pred, msk)\n",
    "            bow_pred_flat = flatten(bow_pred, msk)\n",
    "            _true = flatten(true, msk)\n",
    "            assert np.all(_true == true_flat)\n",
    "            gt_flat = true_flat\n",
    "#             print(\"RoBERTa %30s %5d %3d,\\tAcc: %.4f , UAR: %.4f\\n\" % (tab, len(roberta_pred), len(gt_flat), ACC(gt_flat, roberta_pred_flat), UAR(gt_flat, roberta_pred_flat)))\n",
    "#             print(\"BoW     %30s %5d %3d,\\tAcc: %.4f , UAR: %.4f\\n\" % (tab, len(bow_pred), len(gt_flat), ACC(gt_flat, bow_pred_flat), UAR(gt_flat, bow_pred_flat)))\n",
    "#             print(\"ChatGPT %30s %5d %3d,\\tAcc: %.4f , UAR: %.4f\\n\" % (tab, len(df), len(true_flat), ACC(true_flat, chatgpt_flat), UAR(true_flat, chatgpt_flat)))\n",
    "#             latex_table.append(\"%8s  &  %8s  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  &  %7.2f  \\\\\\\\\" % (\n",
    "#                 target_type.capitalize(), lap,\n",
    "#                 ACC(gt_flat, bow_pred_flat),\n",
    "#                 ACC(gt_flat, roberta_pred_flat),\n",
    "#                 ACC(true_flat, chatgpt_flat),\n",
    "#                 UAR(gt_flat, bow_pred_flat),\n",
    "#                 UAR(gt_flat, roberta_pred_flat),\n",
    "#                 UAR(true_flat, chatgpt_flat),\n",
    "#             ))\n",
    "            # not\n",
    "            latex_table.append(build_row(f\"{target_type} {lap}\", gt_flat, bow_pred_flat, roberta_pred_flat, gpt3_flat, gpt4_flat))\n",
    "            print(latex_table[-1])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "31ff0274",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>target_polarity</th>\n",
       "      <th>target</th>\n",
       "      <th>opinion</th>\n",
       "      <th>GPT3</th>\n",
       "      <th>GPT4</th>\n",
       "      <th>GPT3-opinion</th>\n",
       "      <th>GPT4-opinion</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>love al di la</td>\n",
       "      <td>0 1 1 1</td>\n",
       "      <td>0 1 2 2</td>\n",
       "      <td>1 0 0 0</td>\n",
       "      <td>* love</td>\n",
       "      <td>* love</td>\n",
       "      <td>1 0 0 0</td>\n",
       "      <td>1 0 0 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>i recommend this place to everyone</td>\n",
       "      <td>0 0 0 1 0 0</td>\n",
       "      <td>0 0 0 1 0 0</td>\n",
       "      <td>0 1 0 0 0 0</td>\n",
       "      <td>* recommend</td>\n",
       "      <td>* recommend</td>\n",
       "      <td>0 1 0 0 0 0</td>\n",
       "      <td>0 1 0 0 0 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>great food</td>\n",
       "      <td>0 1</td>\n",
       "      <td>0 1</td>\n",
       "      <td>1 0</td>\n",
       "      <td>* great</td>\n",
       "      <td>* great</td>\n",
       "      <td>1 0</td>\n",
       "      <td>1 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>one of my favorite places in brooklyn</td>\n",
       "      <td>0 0 0 0 0 0 1</td>\n",
       "      <td>0 0 0 0 0 0 0</td>\n",
       "      <td>0 0 0 0 0 0 0</td>\n",
       "      <td>* favorite</td>\n",
       "      <td>* favorite</td>\n",
       "      <td>0 0 0 1 0 0 0</td>\n",
       "      <td>0 0 0 1 0 0 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>the pastas are incredible , the risottos ( par...</td>\n",
       "      <td>0 1 0 0 0 0 1 0 0 0 1 0 0 0 0 0 1 1 0 0</td>\n",
       "      <td>0 1 0 0 0 0 1 0 0 0 1 0 0 0 0 0 1 2 0 0</td>\n",
       "      <td>0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1</td>\n",
       "      <td>* incredible\\n* fantastic\\n* amazing</td>\n",
       "      <td>* incredible\\n* fantastic\\n* amazing</td>\n",
       "      <td>0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1</td>\n",
       "      <td>0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>680</th>\n",
       "      <td>i ca n't believe that it was , but please put ...</td>\n",
       "      <td>0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2</td>\n",
       "      <td>0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0</td>\n",
       "      <td>0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0</td>\n",
       "      <td>* can't believe (negative sentiment)\\n* please...</td>\n",
       "      <td>* ca n't believe\\n* please\\n* put the bag down</td>\n",
       "      <td>0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0</td>\n",
       "      <td>0 1 2 2 0 0 0 0 0 1 1 2 2 2 0 0 0 0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>681</th>\n",
       "      <td>the waitress came to check in on us every few ...</td>\n",
       "      <td>0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...</td>\n",
       "      <td>0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...</td>\n",
       "      <td>0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...</td>\n",
       "      <td>* clear\\n* pet peeve\\n* ignore</td>\n",
       "      <td>* check in\\n* began to clear\\n* pet peeve</td>\n",
       "      <td>0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 ...</td>\n",
       "      <td>0 0 0 0 1 2 0 0 0 0 0 0 0 1 2 2 0 0 0 0 0 0 0 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>682</th>\n",
       "      <td>i could n't ignore the fact that she reach ove...</td>\n",
       "      <td>0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...</td>\n",
       "      <td>0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...</td>\n",
       "      <td>0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...</td>\n",
       "      <td>* couldn't ignore (negative sentiment)\\n* reac...</td>\n",
       "      <td>* ignore\\n* reach over\\n* clear the table</td>\n",
       "      <td>None</td>\n",
       "      <td>0 0 0 1 0 0 0 0 1 2 0 0 0 0 0 0 0 0 0 0 0 0 0 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>683</th>\n",
       "      <td>she then put the check down without asking if ...</td>\n",
       "      <td>0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...</td>\n",
       "      <td>0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...</td>\n",
       "      <td>0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...</td>\n",
       "      <td>* put down (negative sentiment)\\n* without ask...</td>\n",
       "      <td>* without asking\\n* check on the bill every tw...</td>\n",
       "      <td>0 0 1 0 1 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...</td>\n",
       "      <td>0 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0 1 2 2 2 2 2 2 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>684</th>\n",
       "      <td>i wish i could like this place more , and i wi...</td>\n",
       "      <td>0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 2</td>\n",
       "      <td>0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1</td>\n",
       "      <td>0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0</td>\n",
       "      <td>* like\\n* wish\\n* retrain</td>\n",
       "      <td>* like\\n* retrain</td>\n",
       "      <td>0 1 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0</td>\n",
       "      <td>0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>685 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  text  \\\n",
       "0                                        love al di la   \n",
       "1                   i recommend this place to everyone   \n",
       "2                                           great food   \n",
       "3                one of my favorite places in brooklyn   \n",
       "4    the pastas are incredible , the risottos ( par...   \n",
       "..                                                 ...   \n",
       "680  i ca n't believe that it was , but please put ...   \n",
       "681  the waitress came to check in on us every few ...   \n",
       "682  i could n't ignore the fact that she reach ove...   \n",
       "683  she then put the check down without asking if ...   \n",
       "684  i wish i could like this place more , and i wi...   \n",
       "\n",
       "                                       target_polarity  \\\n",
       "0                                              0 1 1 1   \n",
       "1                                          0 0 0 1 0 0   \n",
       "2                                                  0 1   \n",
       "3                                        0 0 0 0 0 0 1   \n",
       "4              0 1 0 0 0 0 1 0 0 0 1 0 0 0 0 0 1 1 0 0   \n",
       "..                                                 ...   \n",
       "680                0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 2   \n",
       "681  0 2 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...   \n",
       "682  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...   \n",
       "683  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...   \n",
       "684                  0 0 0 0 0 0 2 0 0 0 0 0 0 0 0 0 2   \n",
       "\n",
       "                                                target  \\\n",
       "0                                              0 1 2 2   \n",
       "1                                          0 0 0 1 0 0   \n",
       "2                                                  0 1   \n",
       "3                                        0 0 0 0 0 0 0   \n",
       "4              0 1 0 0 0 0 1 0 0 0 1 0 0 0 0 0 1 2 0 0   \n",
       "..                                                 ...   \n",
       "680                0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0   \n",
       "681  0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...   \n",
       "682  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...   \n",
       "683  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...   \n",
       "684                  0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1   \n",
       "\n",
       "                                               opinion  \\\n",
       "0                                              1 0 0 0   \n",
       "1                                          0 1 0 0 0 0   \n",
       "2                                                  1 0   \n",
       "3                                        0 0 0 0 0 0 0   \n",
       "4              0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1   \n",
       "..                                                 ...   \n",
       "680                0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0   \n",
       "681  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...   \n",
       "682  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...   \n",
       "683  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...   \n",
       "684                  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0   \n",
       "\n",
       "                                                  GPT3  \\\n",
       "0                                               * love   \n",
       "1                                          * recommend   \n",
       "2                                              * great   \n",
       "3                                           * favorite   \n",
       "4                 * incredible\\n* fantastic\\n* amazing   \n",
       "..                                                 ...   \n",
       "680  * can't believe (negative sentiment)\\n* please...   \n",
       "681                     * clear\\n* pet peeve\\n* ignore   \n",
       "682  * couldn't ignore (negative sentiment)\\n* reac...   \n",
       "683  * put down (negative sentiment)\\n* without ask...   \n",
       "684                          * like\\n* wish\\n* retrain   \n",
       "\n",
       "                                                  GPT4  \\\n",
       "0                                               * love   \n",
       "1                                          * recommend   \n",
       "2                                              * great   \n",
       "3                                           * favorite   \n",
       "4                 * incredible\\n* fantastic\\n* amazing   \n",
       "..                                                 ...   \n",
       "680     * ca n't believe\\n* please\\n* put the bag down   \n",
       "681          * check in\\n* began to clear\\n* pet peeve   \n",
       "682          * ignore\\n* reach over\\n* clear the table   \n",
       "683  * without asking\\n* check on the bill every tw...   \n",
       "684                                  * like\\n* retrain   \n",
       "\n",
       "                                          GPT3-opinion  \\\n",
       "0                                              1 0 0 0   \n",
       "1                                          0 1 0 0 0 0   \n",
       "2                                                  1 0   \n",
       "3                                        0 0 0 1 0 0 0   \n",
       "4              0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1   \n",
       "..                                                 ...   \n",
       "680                0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0   \n",
       "681  0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 ...   \n",
       "682                                               None   \n",
       "683  0 0 1 0 1 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ...   \n",
       "684                  0 1 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0   \n",
       "\n",
       "                                          GPT4-opinion  \n",
       "0                                              1 0 0 0  \n",
       "1                                          0 1 0 0 0 0  \n",
       "2                                                  1 0  \n",
       "3                                        0 0 0 1 0 0 0  \n",
       "4              0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1  \n",
       "..                                                 ...  \n",
       "680                0 1 2 2 0 0 0 0 0 1 1 2 2 2 0 0 0 0  \n",
       "681  0 0 0 0 1 2 0 0 0 0 0 0 0 1 2 2 0 0 0 0 0 0 0 ...  \n",
       "682  0 0 0 1 0 0 0 0 1 2 0 0 0 0 0 0 0 0 0 0 0 0 0 ...  \n",
       "683  0 0 0 0 0 0 1 2 0 0 0 0 0 0 0 0 1 2 2 2 2 2 2 ...  \n",
       "684                  0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0  \n",
       "\n",
       "[685 rows x 8 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "7312c157",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "&   Engagement  &  $        51.92   $  &  $        71.02^{**} $  &  $\\textbf{79.18}^{**} $  &  $        54.15^{**} $  &  $        51.85   $  &  $        71.02^{**} $  &  $\\textbf{79.19}^{**} $  &  $        53.80^{**} $  \\\\\n",
      "&    reddit  &  $        75.73   $  &  $        30.83^{**} $  &  $\\textbf{94.58}^{**} $  &  $        90.10^{**} $  &  $        80.46   $  &  $        54.42^{**} $  &  $\\textbf{84.88}      $  &  $        83.73      $  \\\\\n",
      "&    reddit body  &  $        91.93   $  &  $        84.50^{**} $  &  $        89.88      $  &  $\\textbf{93.33}      $  &  $        84.41   $  &  $        68.82^{**} $  &  $\\textbf{86.16}      $  &  $        78.63^{**} $  \\\\\n",
      "&    reddit titles  &  $        80.61   $  &  $        86.60^{**} $  &  $\\textbf{96.75}^{**} $  &  $        89.54^{**} $  &  $        80.05   $  &  $        85.62^{**} $  &  $\\textbf{96.65}^{**} $  &  $        89.77^{**} $  \\\\\n",
      "&    twitter  &  $        60.53   $  &  $        43.36^{**} $  &  $\\textbf{93.23}^{**} $  &  $        72.31^{**} $  &  $        65.05   $  &  $        45.45^{**} $  &  $\\textbf{93.14}^{**} $  &  $        73.45^{**} $  \\\\\n",
      "&    twitter full  &  $        66.24   $  &  $        80.39^{**} $  &  $\\textbf{84.39}^{**} $  &  $        75.25^{**} $  &  $        66.26   $  &  $        80.39^{**} $  &  $\\textbf{84.39}^{**} $  &  $        75.25^{**} $  \\\\\n",
      "&   Joy  &  $        74.07   $  &  $        66.49^{**} $  &  $        75.41      $  &  $\\textbf{78.46}^{**} $  &  $        74.29   $  &  $        66.51^{**} $  &  $        75.40      $  &  $\\textbf{78.63}^{**} $  \\\\\n",
      "&   Fear  &  $        72.76   $  &  $        68.65^{**} $  &  $\\textbf{76.83}^{**} $  &  $        73.96      $  &  $        72.86   $  &  $        68.65^{**} $  &  $\\textbf{76.83}^{**} $  &  $        74.09      $  \\\\\n",
      "&   Anger  &  $        72.12   $  &  $        67.63^{**} $  &  $        73.47      $  &  $\\textbf{75.58}^{**} $  &  $        72.09   $  &  $        67.60^{**} $  &  $        73.46      $  &  $\\textbf{75.49}^{**} $  \\\\\n",
      "&   Sadness  &  $        78.19   $  &  $        72.41^{**} $  &  $        76.06      $  &  $\\textbf{78.55}      $  &  $        78.22   $  &  $        72.40^{**} $  &  $        76.05      $  &  $\\textbf{78.58}      $  \\\\\n",
      "&   Openness  &  $        50.11   $  &  $        58.36^{**} $  &  $\\textbf{60.54}^{**} $  &  $        54.75^{**} $  &  $        50.60   $  &  $        58.36^{**} $  &  $\\textbf{60.56}^{**} $  &  $        54.59^{**} $  \\\\\n",
      "&   Conscientiousness  &  $        55.54   $  &  $        56.79      $  &  $\\textbf{61.59}^{**} $  &  $        57.44^*    $  &  $        55.84   $  &  $        56.78      $  &  $\\textbf{61.59}^{**} $  &  $        57.33      $  \\\\\n",
      "&   Extraversion  &  $        53.55   $  &  $        56.51^{**} $  &  $\\textbf{59.03}^{**} $  &  $        55.90^{**} $  &  $        53.38   $  &  $        56.51^{**} $  &  $\\textbf{59.02}^{**} $  &  $        55.90^{**} $  \\\\\n",
      "&   Agreeableness  &  $        51.67   $  &  $        57.81^{**} $  &  $\\textbf{58.12}^{**} $  &  $        54.04^{**} $  &  $        52.10   $  &  $        57.80^{**} $  &  $\\textbf{58.14}^{**} $  &  $        54.05^*    $  \\\\\n",
      "&   Neuroticism  &  $        48.94   $  &  $        58.60^{**} $  &  $\\textbf{59.86}^{**} $  &  $        49.68      $  &  $        49.04   $  &  $        58.60^{**} $  &  $\\textbf{59.86}^{**} $  &  $        49.73      $  \\\\\n",
      "&   Sentiment microblogs  &  $        69.30   $  &  $        70.88      $  &  $        72.37      $  &  $\\textbf{73.21}^{**} $  &  $        68.69   $  &  $        70.83      $  &  $        72.41^*    $  &  $\\textbf{73.08}^{**} $  \\\\\n",
      "&   Sarcasm  &  $        59.13   $  &  $        63.14^{**} $  &  $\\textbf{90.66}^{**} $  &  $        66.66^{**} $  &  $        56.82   $  &  $        66.29^{**} $  &  $\\textbf{90.70}^{**} $  &  $        69.45^{**} $  \\\\\n",
      "&   Sentiment  &  $        80.54   $  &  $        78.87      $  &  $\\textbf{88.74}^{**} $  &  $        84.09^{**} $  &  $        79.93   $  &  $        78.84      $  &  $\\textbf{88.75}^{**} $  &  $        83.69^{**} $  \\\\\n",
      "&   Suicide  &  $        89.46   $  &  $        84.75^{**} $  &  $\\textbf{98.43}^{**} $  &  $        93.46^{**} $  &  $        89.44   $  &  $        85.32^{**} $  &  $\\textbf{98.46}^{**} $  &  $        93.32^{**} $  \\\\\n",
      "&   Subjectivity  &  $        59.56   $  &  $        87.28^{**} $  &  $\\textbf{95.56}^{**} $  &  $        88.38^{**} $  &  $        58.59   $  &  $        87.19^{**} $  &  $\\textbf{95.51}^{**} $  &  $        88.19^{**} $  \\\\\n",
      "&   Toxic  &  $        87.37   $  &  $        81.85^{**} $  &  $        85.23      $  &  $\\textbf{89.29}      $  &  $        87.19   $  &  $        82.75^{**} $  &  $        86.01      $  &  $\\textbf{89.65}^*    $  \\\\\n",
      "&   Severe toxic  &  $        66.55   $  &  $\\textbf{87.65}^{**} $  &  $        80.07^{**} $  &  $        75.52^{**} $  &  $        80.65   $  &  $        82.48      $  &  $        84.78^*    $  &  $\\textbf{85.29}^{**} $  \\\\\n",
      "&   Obscene  &  $        83.45   $  &  $        85.40      $  &  $        84.83      $  &  $\\textbf{88.16}^{**} $  &  $        83.48   $  &  $        86.62^*    $  &  $        86.60^*    $  &  $\\textbf{86.78}^*    $  \\\\\n",
      "&   Threat  &  $        70.59   $  &  $        94.05^{**} $  &  $\\textbf{95.54}^{**} $  &  $        91.99^{**} $  &  $        80.12   $  &  $        73.99      $  &  $        87.43^{**} $  &  $\\textbf{91.51}^{**} $  \\\\\n",
      "&   Insult  &  $        80.14   $  &  $        84.65^{**} $  &  $\\textbf{87.25}^{**} $  &  $        80.70      $  &  $        83.21   $  &  $        84.64      $  &  $\\textbf{89.28}^{**} $  &  $        84.89      $  \\\\\n",
      "&   Identity hate  &  $        66.82   $  &  $        90.52^{**} $  &  $\\textbf{90.98}^{**} $  &  $        82.66^{**} $  &  $        78.61   $  &  $        81.61      $  &  $        86.16^{**} $  &  $\\textbf{87.88}^{**} $  \\\\\n",
      "&   Target res14  &  $        86.95   $  &  $        81.73^{**} $  &  $\\textbf{92.00}^{**} $  &  $        71.50^{**} $  &  $        76.18   $  &  $        81.18^{**} $  &  $\\textbf{91.54}^{**} $  &  $        73.23^{**} $  \\\\\n",
      "&   Target lap14  &  $        84.60   $  &  $        78.22^{**} $  &  $\\textbf{87.19}^{**} $  &  $        70.32^{**} $  &  $        77.62   $  &  $        82.21^{**} $  &  $\\textbf{90.64}^{**} $  &  $        75.94      $  \\\\\n",
      "&   Target res15  &  $\\textbf{84.57}   $  &  $        81.28^{**} $  &  $        73.02^{**} $  &  $        70.05^{**} $  &  $        78.56   $  &  $        72.28^{**} $  &  $\\textbf{85.13}^{**} $  &  $        73.81^{**} $  \\\\\n",
      "&   Polarity res14  &  $        85.13   $  &  $\\textbf{86.10}^*    $  &  $        71.85^{**} $  &  $        69.30^{**} $  &  $        49.96   $  &  $        49.72      $  &  $\\textbf{58.09}^*    $  &  $        48.26      $  \\\\\n",
      "&   Polarity lap14  &  $        82.23   $  &  $        72.57^{**} $  &  $\\textbf{90.22}^{**} $  &  $        67.63^{**} $  &  $        47.37   $  &  $        45.10      $  &  $\\textbf{58.23}^{**} $  &  $        44.54^*    $  \\\\\n",
      "&   Polarity res15  &  $        82.38   $  &  $        79.08^{**} $  &  $\\textbf{84.31}^{**} $  &  $        67.51^{**} $  &  $        47.79   $  &  $        36.84^{**} $  &  $\\textbf{48.96}      $  &  $        44.27      $  \\\\\n",
      "&   Opinion res14  &  $        91.04   $  &  $        81.61^{**} $  &  $\\textbf{93.26}^{**} $  &  $        80.93^{**} $  &  $        77.28   $  &  $        80.87      $  &  $        80.06      $  &  $\\textbf{83.02}^*    $  \\\\\n",
      "&   Opinion lap14  &  $\\textbf{89.43}   $  &  $        74.33^{**} $  &  $        73.81^{**} $  &  $        76.90^{**} $  &  $        73.51   $  &  $        66.43^*    $  &  $        76.73      $  &  $\\textbf{77.42}      $  \\\\\n",
      "&   Opinion res15  &  $\\textbf{89.32}   $  &  $        79.42^{**} $  &  $        89.16      $  &  $        78.10^{**} $  &  $        76.90   $  &  $        68.59      $  &  $\\textbf{77.31}      $  &  $        77.19      $  \\\\\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n\".join(latex_table))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac6b12f4",
   "metadata": {},
   "source": [
    "manual_opinions = [\n",
    "\n",
    " ('delicious (corresponding to \"flan\")', 'desserts include flan and sopaipillas', 'flan'),\n",
    "\n",
    " ('overpriced', 'over priced , overrated away !', 'over priced'),\n",
    " ('delicious', 'i usually get one the vietnamese beef noodle soup'),\n",
    " ('efficient service',\n",
    "  'my order is on my table even on a busy friday night within 10 minutes ( at the most ) of hanging up the phone'),\n",
    " ('expensive', \"i thought the food is n't cheap at all compared to chinatown\"),\n",
    " ('delicious (implied sentiment)',\n",
    "  'half a chicken with a mountain of rice and beans for 6.25'),\n",
    " ('delicious', 'this is literally a hot spot when it comes to the food'),\n",
    " ('mouth-watering',\n",
    "  \"el nidos one of the best restaurants in new york which i 've ever been to , has a great variety of tasty , mouth watering pizza 's\"),\n",
    " ('well-presented',\n",
    "  'had a lovely dinner in this dedicated seafood joint , food was well-prepared and -presented and the service was pleasant and prompt'),\n",
    " ('None of the words in the sentence express sentiment. Therefore, the sentiment of the sentence cannot be determined. Hence, the response is \"BACKGROUND\".',\n",
    "  'we live in the neighborhood and have been going there twice a week since the day they first opened'),\n",
    " ('chicken',\n",
    "  'i reccomend the fried pork dumplings , the orange chicken/beef , and the fried rice'),\n",
    " (\"doesn't meet\",\n",
    "  \"cuisine is billed as asian fusion - does n't meet the bill\"),\n",
    " ('delicious food', 'and the food , well the food will keep you coming back'),\n",
    " ('full-sized',\n",
    "  'and these are not small , wimpy fast food type burgers - these are real , full sized patties'),\n",
    " ('beautiful', 'the decor is what initially got me in the door'),\n",
    " ('cannot recommend',\n",
    "  \"maybe it 's that most of dumont 's patrons are younger and have lower expectations , but i can not recommend dumont\"),\n",
    " ('love',\n",
    "  \"for someone who used to hate indian food , baluchi 's has changed my mid\"),\n",
    " (\"wouldn't talk to me again\",\n",
    "  'it took a bigger bite from my wallet than my appetite - i would not reccomend this to anyone that i would want to talk to me again ! !'),\n",
    " ('expensive',\n",
    "  'but make sure you have enough room on your credit card as the bill will leave a big dent in your wallet'),\n",
    " ('delicious',\n",
    "  'they bring a sauce cart up to your table and offer you up to 7 or 8 choices of sauces for your steak ( i tried them all )'),\n",
    " ('recommended: duck',\n",
    "  'not only was the sushi fresh , they also served other entrees allowed each guest something to choose from and we all left happy ( try the duck !'),\n",
    " ('disappointed (implied)',\n",
    "  'the closest that i got was the cherry marscapone , but they were out of it that day'),\n",
    " ('irresistible',\n",
    "  \"my friends and i go once a week ( minimum , we live in the area ) and everytime ca n't resist walking out ready to burst\"),\n",
    " ('enjoyable',\n",
    "  'however , there is just something so great about being outdoors , in great landscaping , enjoying a casual drink that makes going to this place worthwhile'),\n",
    " ('better',\n",
    "  'if you are here as a pre-show meal , hop in a cab and take the extra 10 minutes to go to the uptown location'),\n",
    " ('delicious',\n",
    "  \"the best dessert , a chocolate and peanut butter tart , is n't particularly hawaiian , but it 's a small world when it comes to sweets\"),\n",
    " ('favorite',\n",
    "  'now we moved out of the state and every visit to nyc includes a stop here'),\n",
    " (\"would n't return to\",\n",
    "  \"we were seated next to a couple whose kitchen was being renovated and we agreed since we 're all foodies who eat out a lot - that this was n't a place we 'd return to\"),\n",
    " ('great menu options',\n",
    "  \"i just had my first visit to this place and ca n't wait to go back and slowly work my way through the menu\")]\n",
    "\n",
    "manual_opinions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ae790b9d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sarcasm\n",
      "subjectivity\n",
      "suicide\n",
      "sentiment\n",
      "personality\n",
      "toxicity\n",
      "   toxic  severe_toxic  obscene  threat  insult  identity_hate\n",
      "0  21936         35394    28634   36401   29170          35503\n",
      "1  14928          1470     8230     463    7694           1361\n",
      "   toxic  severe_toxic  obscene  threat  insult  identity_hate\n",
      "0  17872         28796    23305   29628   23763          28903\n",
      "1  12128          1204     6695     372    6237           1097\n",
      "   toxic  severe_toxic  obscene  threat  insult  identity_hate\n",
      "0   4064          6598     5329    6773    5407           6600\n",
      "1   2800           266     1535      91    1457            264\n",
      "(36864, 9) Index(['toxic', 'severe_toxic', 'obscene', 'threat', 'insult',\n",
      "       'identity_hate'],\n",
      "      dtype='object')\n",
      "   toxic  severe_toxic  obscene  threat  insult  identity_hate\n",
      "0  21936         35394    28634   36401   29170          35503\n",
      "1  14928          1470     8230     463    7694           1361\n",
      "   toxic  severe_toxic  obscene  threat  insult  identity_hate\n",
      "0  17872         28796    23305   29628   23763          28903\n",
      "1  12128          1204     6695     372    6237           1097\n",
      "   toxic  severe_toxic  obscene  threat  insult  identity_hate\n",
      "0   4064          6598     5329    6773    5407           6600\n",
      "1   2800           266     1535      91    1457            264\n",
      "(36864, 9) Index(['toxic', 'severe_toxic', 'obscene', 'threat', 'insult',\n",
      "       'identity_hate'],\n",
      "      dtype='object')\n",
      "   toxic  severe_toxic  obscene  threat  insult  identity_hate\n",
      "0  57888         63611    60287   63767   60551          63266\n",
      "1   6090           367     3691     211    3427            712\n",
      "   toxic  severe_toxic  obscene  threat  insult  identity_hate\n",
      "0    555           884      650     906     667            859\n",
      "1    445           116      350      94     333            141\n",
      "(1000, 9) Index(['toxic', 'severe_toxic', 'obscene', 'threat', 'insult',\n",
      "       'identity_hate'],\n",
      "      dtype='object')\n",
      "   toxic  severe_toxic  obscene  threat  insult  identity_hate\n",
      "0    537           860      631     871     647            832\n",
      "1    422            99      328      88     312            127\n",
      "(959, 9) Index(['toxic', 'severe_toxic', 'obscene', 'threat', 'insult',\n",
      "       'identity_hate'],\n",
      "      dtype='object')\n",
      "sentiment_ranking_microblogs\n",
      "sadness_emotion_intensity\n",
      "joy_emotion_intensity\n",
      "fear_emotion_intensity\n",
      "anger_emotion_intensity\n",
      "well_being_reddit_body\n",
      "(3123, 5) Index(['index', 'title', 'body', 'Body_Title', 'label'], dtype='object')\n",
      "(3123, 5) Index(['index', 'title', 'body', 'Body_Title', 'label'], dtype='object')\n",
      "(3123, 5) Index(['index', 'title', 'body', 'Body_Title', 'label'], dtype='object')\n",
      "well_being_reddit\n",
      "(3123, 5) Index(['index', 'title', 'body', 'Body_Title', 'label'], dtype='object')\n",
      "(3123, 5) Index(['index', 'title', 'body', 'Body_Title', 'label'], dtype='object')\n",
      "(3123, 5) Index(['index', 'title', 'body', 'Body_Title', 'label'], dtype='object')\n",
      "well_being_reddit_titles\n",
      "(5556, 3) Index(['index', 'title', 'label'], dtype='object')\n",
      "(5556, 3) Index(['index', 'title', 'label'], dtype='object')\n",
      "(5556, 3) Index(['index', 'title', 'label'], dtype='object')\n",
      "well_being_twitter\n",
      "(2051, 3) Index(['index', 'text', 'label'], dtype='object')\n",
      "(2051, 3) Index(['index', 'text', 'label'], dtype='object')\n",
      "(2051, 3) Index(['index', 'text', 'label'], dtype='object')\n",
      "well_being_twitter_full\n",
      "(8900, 4) Index(['index', 'text', 'hashtags', 'labels'], dtype='object')\n",
      "(8900, 4) Index(['index', 'text', 'hashtags', 'labels'], dtype='object')\n",
      "(8900, 4) Index(['index', 'text', 'hashtags', 'labels'], dtype='object')\n",
      "engagement\n",
      "aspect_res14_target\n",
      "aspect_lap14_target\n",
      "aspect_res15_target\n",
      "aspect_res14_polarity\n",
      "aspect_lap14_polarity\n",
      "aspect_res15_polarity\n",
      "aspect_res14_opinion\n",
      "aspect_lap14_opinion\n",
      "aspect_res15_opinion\n"
     ]
    }
   ],
   "source": [
    "out = []\n",
    "for key in reference.keys():\n",
    "    fn = globals()[f\"read_{key}\"]\n",
    "    print(key)\n",
    "    out.append([key, len(fn(\"train\")[0]), len(fn(\"valid\")[0]), len(fn(\"test\")[0])])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "3324a8c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sarcasm  &  18709  &  4000  &  4000\\\\ \n",
      "subjectivity  &  6000  &  2000  &  2000\\\\ \n",
      "suicide  &  23398  &  5611  &  2345\\\\ \n",
      "sentiment  &  100000  &  10000  &  2500\\\\ \n",
      "personality  &  5992  &  2000  &  1996\\\\ \n",
      "toxicity  &  30000  &  6864  &  959\\\\ \n",
      "sentiment_ranking_microblogs  &  1000  &  300  &  365\\\\ \n",
      "sadness_emotion_intensity  &  786  &  74  &  673\\\\ \n",
      "joy_emotion_intensity  &  823  &  79  &  714\\\\ \n",
      "fear_emotion_intensity  &  1147  &  110  &  995\\\\ \n",
      "anger_emotion_intensity  &  857  &  84  &  760\\\\ \n",
      "well_being_reddit_body  &  1511  &  458  &  935\\\\ \n",
      "well_being_reddit  &  1615  &  495  &  987\\\\ \n",
      "well_being_reddit_titles  &  3538  &  996  &  998\\\\ \n",
      "well_being_twitter  &  851  &  400  &  800\\\\ \n",
      "well_being_twitter_full  &  5900  &  1500  &  1500\\\\ \n",
      "engagement  &  30037  &  5000  &  4000\\\\ \n",
      "aspect_res14_target  &  2436  &  608  &  800\\\\ \n",
      "aspect_lap14_target  &  2439  &  609  &  800\\\\ \n",
      "aspect_res15_target  &  1052  &  263  &  685\\\\ \n",
      "aspect_res14_polarity  &  2436  &  608  &  800\\\\ \n",
      "aspect_lap14_polarity  &  2439  &  609  &  800\\\\ \n",
      "aspect_res15_polarity  &  1052  &  263  &  685\\\\ \n",
      "aspect_res14_opinion  &  2436  &  608  &  800\\\\ \n",
      "aspect_lap14_opinion  &  2439  &  609  &  800\\\\ \n",
      "aspect_res15_opinion  &  1052  &  263  &  685\n"
     ]
    }
   ],
   "source": [
    "print(\"\\\\\\\\ \\n\".join(\n",
    "    \"  &  \".join(map(str, x))\n",
    "    for x in out\n",
    "))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}