notebooks/valentin/First exploration.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "seventh-festival",
   "metadata": {},
   "source": [
    "# Imports & constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "exceptional-parliament",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:23:58.698186Z",
     "start_time": "2022-01-27T14:23:56.877728Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.naive_bayes import ComplementNB\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from evalstudent import metrics\n",
    "from evalstudent import utils\n",
    "from tqdm import tqdm\n",
    "from sklearn.model_selection import KFold\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "downtown-legend",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:23:58.741521Z",
     "start_time": "2022-01-27T14:23:58.701201Z"
    }
   },
   "outputs": [],
   "source": [
    "DATA_PATH = \"../../raw_data/train/\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "young-market",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Exploration"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "interim-amendment",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Loading data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "informational-outdoors",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:23:59.653686Z",
     "start_time": "2022-01-27T14:23:58.747839Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(\"../../raw_data/train.csv\", dtype = {\"discourse_id\": int, \"discourse_start\": int, \"discourse_end\": int})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "hearing-devices",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:23:59.708253Z",
     "start_time": "2022-01-27T14:23:59.656148Z"
    },
    "hidden": true,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>discourse_id</th>\n",
       "      <th>discourse_start</th>\n",
       "      <th>discourse_end</th>\n",
       "      <th>discourse_text</th>\n",
       "      <th>discourse_type</th>\n",
       "      <th>discourse_type_num</th>\n",
       "      <th>predictionstring</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>423A1CA112E2</td>\n",
       "      <td>1622627660524</td>\n",
       "      <td>8</td>\n",
       "      <td>229</td>\n",
       "      <td>Modern humans today are always on their phone....</td>\n",
       "      <td>Lead</td>\n",
       "      <td>Lead 1</td>\n",
       "      <td>1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>423A1CA112E2</td>\n",
       "      <td>1622627653021</td>\n",
       "      <td>230</td>\n",
       "      <td>312</td>\n",
       "      <td>They are some really bad consequences when stu...</td>\n",
       "      <td>Position</td>\n",
       "      <td>Position 1</td>\n",
       "      <td>45 46 47 48 49 50 51 52 53 54 55 56 57 58 59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>423A1CA112E2</td>\n",
       "      <td>1622627671020</td>\n",
       "      <td>313</td>\n",
       "      <td>401</td>\n",
       "      <td>Some certain areas in the United States ban ph...</td>\n",
       "      <td>Evidence</td>\n",
       "      <td>Evidence 1</td>\n",
       "      <td>60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>423A1CA112E2</td>\n",
       "      <td>1622627696365</td>\n",
       "      <td>402</td>\n",
       "      <td>758</td>\n",
       "      <td>When people have phones, they know about certa...</td>\n",
       "      <td>Evidence</td>\n",
       "      <td>Evidence 2</td>\n",
       "      <td>76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>423A1CA112E2</td>\n",
       "      <td>1622627759780</td>\n",
       "      <td>759</td>\n",
       "      <td>886</td>\n",
       "      <td>Driving is one of the way how to get around. P...</td>\n",
       "      <td>Claim</td>\n",
       "      <td>Claim 1</td>\n",
       "      <td>139 140 141 142 143 144 145 146 147 148 149 15...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             id   discourse_id  discourse_start  discourse_end  \\\n",
       "0  423A1CA112E2  1622627660524                8            229   \n",
       "1  423A1CA112E2  1622627653021              230            312   \n",
       "2  423A1CA112E2  1622627671020              313            401   \n",
       "3  423A1CA112E2  1622627696365              402            758   \n",
       "4  423A1CA112E2  1622627759780              759            886   \n",
       "\n",
       "                                      discourse_text discourse_type  \\\n",
       "0  Modern humans today are always on their phone....           Lead   \n",
       "1  They are some really bad consequences when stu...       Position   \n",
       "2  Some certain areas in the United States ban ph...       Evidence   \n",
       "3  When people have phones, they know about certa...       Evidence   \n",
       "4  Driving is one of the way how to get around. P...          Claim   \n",
       "\n",
       "  discourse_type_num                                   predictionstring  \n",
       "0             Lead 1  1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 1...  \n",
       "1         Position 1       45 46 47 48 49 50 51 52 53 54 55 56 57 58 59  \n",
       "2         Evidence 1    60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75  \n",
       "3         Evidence 2  76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 9...  \n",
       "4            Claim 1  139 140 141 142 143 144 145 146 147 148 149 15...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "upset-vinyl",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Class balance"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cordless-welsh",
   "metadata": {
    "hidden": true
   },
   "source": [
    "We have an important class imbalance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "demanding-genetics",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:23:59.763863Z",
     "start_time": "2022-01-27T14:23:59.711335Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Claim                   0.347959\n",
       "Evidence                0.316731\n",
       "Position                0.106859\n",
       "Concluding Statement    0.093594\n",
       "Lead                    0.064487\n",
       "Counterclaim            0.040314\n",
       "Rebuttal                0.030057\n",
       "Name: discourse_type, dtype: float64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df[\"discourse_type\"].value_counts()/train_df.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "pleased-saskatchewan",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Are discourse elements full sentences?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "broken-semester",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Discourse elements that are sentences or groups of sentences (ie: starts with an uppercase letter and ends with a mark). The real number is higher because some students forget uppercase letters or final marks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "professional-revision",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:23:59.864830Z",
     "start_time": "2022-01-27T14:23:59.767902Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5753778769586883"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences_elements = [\n",
    "    (text[0].isupper() or text[1].isupper())\n",
    "    and (text[-1] in \".?!\" or text[-2] in \".?!\")\n",
    "    for text in train_df[\"discourse_text\"]\n",
    "]\n",
    "sentences_elements.count(True)/len(sentences_elements)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "annual-explanation",
   "metadata": {
    "hidden": true
   },
   "source": [
    "Breakdown by discourse class:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "amazing-inside",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:24:00.050977Z",
     "start_time": "2022-01-27T14:23:59.871127Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Lead': 0.7219774314884471,\n",
       " 'Position': 0.510798365652766,\n",
       " 'Evidence': 0.7472539494989279,\n",
       " 'Claim': 0.43530911408540474,\n",
       " 'Concluding Statement': 0.5780821917808219,\n",
       " 'Counterclaim': 0.45212308750214886,\n",
       " 'Rebuttal': 0.45768964722158173}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences_ratio = {}\n",
    "for class_ in train_df[\"discourse_type\"].unique():\n",
    "    class_df = train_df[\"discourse_text\"][train_df[\"discourse_type\"] == class_]\n",
    "    sentences_elements = [\n",
    "        (text[0].isupper() or text[1].isupper())\n",
    "        and (text[-1] in \".?!\" or text[-2] in \".?!\")\n",
    "        for text in class_df\n",
    "    ]\n",
    "    sentences_ratio.update({class_: sentences_elements.count(True)/len(sentences_elements)})\n",
    "sentences_ratio"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fresh-disclosure",
   "metadata": {},
   "source": [
    "# Modeling: Super Naive Bayesline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "third-paintball",
   "metadata": {},
   "source": [
    "## Preliminary calculation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "difficult-grill",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-12-29T09:12:47.655042Z",
     "start_time": "2021-12-29T09:12:47.572057Z"
    }
   },
   "source": [
    "Average count of discourse classes per essay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "vital-spectrum",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:24:00.156464Z",
     "start_time": "2022-01-27T14:24:00.054952Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "discourse_type\n",
       "Claim                   3.363569\n",
       "Concluding Statement    1.006484\n",
       "Counterclaim            1.271198\n",
       "Evidence                2.939035\n",
       "Lead                    1.000430\n",
       "Position                1.003449\n",
       "Rebuttal                1.205392\n",
       "Name: discourse_id, dtype: float64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classes_ratio = train_df[[\"id\", \"discourse_id\", \"discourse_type\"]]\\\n",
    "    .groupby([\"id\", \"discourse_type\"]).count()\\\n",
    "    .groupby(\"discourse_type\").mean()\\\n",
    "    .squeeze()\n",
    "classes_ratio"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dried-december",
   "metadata": {},
   "source": [
    "The strategy is to identify the following numbers of discourse classes when we \"predict\" a new essay."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "civic-examination",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:24:00.215573Z",
     "start_time": "2022-01-27T14:24:00.159983Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "discourse_type\n",
       "Claim                   3\n",
       "Concluding Statement    1\n",
       "Counterclaim            1\n",
       "Evidence                3\n",
       "Lead                    1\n",
       "Position                1\n",
       "Rebuttal                1\n",
       "Name: discourse_id, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classes_ratio_rounded = classes_ratio.round().astype(int)\n",
    "classes_ratio_rounded"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "existing-offer",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "spare-vector",
   "metadata": {},
   "source": [
    "We use `ComplementNB` which is better suited than `MultinomialNB` for imbalanced classes (see Sklearn doc)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "adequate-provincial",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:24:00.269382Z",
     "start_time": "2022-01-27T14:24:00.222428Z"
    }
   },
   "outputs": [],
   "source": [
    "def bayesline_train(X, y):\n",
    "    vectorizer = TfidfVectorizer(lowercase=False)\n",
    "    X_vec = vectorizer.fit_transform(X)\n",
    "\n",
    "    model = ComplementNB()\n",
    "    model.fit(X_vec, y)\n",
    "    return model, vectorizer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "super-wilderness",
   "metadata": {},
   "source": [
    "## Infering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "medieval-assignment",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:24:00.312266Z",
     "start_time": "2022-01-27T14:24:00.274004Z"
    }
   },
   "outputs": [],
   "source": [
    "def bayesline_predict(essay_id, data_path, model, vectorizer):\n",
    "    essay_text = open(f'{data_path}{essay_id}.txt').read()\n",
    "    # For each sentence, we compute the probability of belonging to each class, and store this into a Dataframe\n",
    "    sentences = sent_tokenize(essay_text)\n",
    "    predictions_matrix = pd.DataFrame(model.predict_proba(vectorizer.transform(sentences)), columns=model.classes_)\n",
    "\n",
    "    # Now, we assign a defined number of sentences to each class using the numbers in `classes_ratio_rounded`.\n",
    "    # To do so, we take the sentence with the highest probability of belonging to a class, assign it to this class,\n",
    "    # and repeat until all classes have the defined number of sentences assigned.\n",
    "    prediction = []\n",
    "    classes_count = classes_ratio_rounded.copy()\n",
    "    sentences_nb = min(classes_count.sum(), len(sentences))\n",
    "    while len(prediction) < sentences_nb :\n",
    "        highest_score_class = predictions_matrix.max().idxmax()\n",
    "        highest_score_element = predictions_matrix[highest_score_class].idxmax()\n",
    "        prediction.append((highest_score_element, highest_score_class))\n",
    "        predictions_matrix.drop(highest_score_element, inplace=True)\n",
    "        classes_count[highest_score_class] -= 1\n",
    "        if classes_count[highest_score_class] == 0:\n",
    "            predictions_matrix.drop(columns=highest_score_class, inplace=True)\n",
    "    prediction\n",
    "\n",
    "    # To generate the submission DataFrame, we need to match sentences number and words index.\n",
    "    sentences_words_index = []\n",
    "    word_index = 0\n",
    "    for sentence in sentences:\n",
    "        sentence_length = len(sentence.split())\n",
    "        sentences_words_index.append(\" \".join([str(i) for i in range(word_index, word_index+sentence_length)]))\n",
    "        word_index += sentence_length\n",
    "    sentences_words_index\n",
    "\n",
    "    # Returning the submission DataFrame\n",
    "    submission = [[essay_id, element[1], sentences_words_index[element[0]]] for element in prediction]\n",
    "    return pd.DataFrame(submission, columns=[\"id\", \"class\", \"predictionstring\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "developing-captain",
   "metadata": {},
   "source": [
    "## Evaluating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "developmental-ethics",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:24:00.357494Z",
     "start_time": "2022-01-27T14:24:00.315036Z"
    }
   },
   "outputs": [],
   "source": [
    "def evaluate(test_df, model_prediction_method, data_path, model, vectorizer):\n",
    "    kaggle_scores = []\n",
    "    score_details = []\n",
    "    essay_ids = test_df[\"id\"].unique()\n",
    "    for essay_id in tqdm(essay_ids):\n",
    "        kaggle_score, score_detail = metrics.kaggle_score(\n",
    "            model_prediction_method(essay_id, data_path, model, vectorizer),\n",
    "            test_df[test_df[\"id\"] == essay_id],\n",
    "            return_details=True)\n",
    "        kaggle_scores.append(kaggle_score)\n",
    "        score_details.append(pd.DataFrame(score_detail))\n",
    "    kaggle_scores = pd.Series(kaggle_scores, index=essay_ids)\n",
    "    score_details = pd.concat(score_details, keys=essay_ids)\n",
    "    return (kaggle_scores, score_details)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "functioning-measurement",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:24:07.727954Z",
     "start_time": "2022-01-27T14:24:00.361259Z"
    }
   },
   "outputs": [],
   "source": [
    "# Training on the entire dataset for now\n",
    "X = train_df[\"discourse_text\"]\n",
    "y = train_df[\"discourse_type\"]\n",
    "model, vectorizer = bayesline_train(X,  y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "closing-christian",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:24:08.532736Z",
     "start_time": "2022-01-27T14:24:07.730189Z"
    },
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [00:00<00:00, 14.17it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(423A1CA112E2    0.000000\n",
       " A8445CABFECE    0.142857\n",
       " 6B4F7A0165B9    0.000000\n",
       " E05C7F5C1156    0.031746\n",
       " 50B3435E475B    0.190476\n",
       " DBF7EB6A9E02    0.238095\n",
       " 810B70E80E1D    0.231293\n",
       " CE98789F502B    0.057143\n",
       " A97DE0D49AEA    0.142857\n",
       " 48D3F4243F0F    0.057143\n",
       " dtype: float64,\n",
       "                         Lead  Position     Claim  Counterclaim  Rebuttal  \\\n",
       " 423A1CA112E2 precision   0.0       0.0  0.000000           0.0       0.0   \n",
       "              recall      0.0       0.0  0.000000           NaN       NaN   \n",
       "              f1          0.0       0.0  0.000000           0.0       0.0   \n",
       " A8445CABFECE precision   0.0       1.0  0.000000           0.0       0.0   \n",
       "              recall      NaN       1.0  0.000000           NaN       NaN   \n",
       "              f1          0.0       1.0  0.000000           0.0       0.0   \n",
       " 6B4F7A0165B9 precision   0.0       0.0  0.000000           0.0       0.0   \n",
       "              recall      0.0       0.0  0.000000           0.0       0.0   \n",
       "              f1          0.0       0.0  0.000000           0.0       0.0   \n",
       " E05C7F5C1156 precision   0.0       0.0  0.333333           0.0       0.0   \n",
       "              recall      0.0       0.0  0.166667           NaN       NaN   \n",
       "              f1          0.0       0.0  0.222222           0.0       0.0   \n",
       " 50B3435E475B precision   0.0       1.0  0.333333           0.0       0.0   \n",
       "              recall      0.0       1.0  0.333333           NaN       NaN   \n",
       "              f1          0.0       1.0  0.333333           0.0       0.0   \n",
       " DBF7EB6A9E02 precision   0.0       1.0  1.000000           0.0       0.0   \n",
       "              recall      0.0       1.0  0.500000           NaN       NaN   \n",
       "              f1          0.0       1.0  0.666667           0.0       0.0   \n",
       " 810B70E80E1D precision   0.0       1.0  0.333333           0.0       0.0   \n",
       "              recall      NaN       1.0  0.250000           NaN       NaN   \n",
       "              f1          0.0       1.0  0.285714           0.0       0.0   \n",
       " CE98789F502B precision   0.0       0.0  0.333333           0.0       0.0   \n",
       "              recall      0.0       0.0  0.500000           NaN       NaN   \n",
       "              f1          0.0       0.0  0.400000           0.0       0.0   \n",
       " A97DE0D49AEA precision   0.0       0.0  0.666667           0.0       0.0   \n",
       "              recall      NaN       0.0  0.666667           NaN       NaN   \n",
       "              f1          0.0       0.0  0.666667           0.0       0.0   \n",
       " 48D3F4243F0F precision   0.0       0.0  0.333333           0.0       0.0   \n",
       "              recall      0.0       0.0  0.500000           NaN       NaN   \n",
       "              f1          0.0       0.0  0.400000           0.0       0.0   \n",
       " \n",
       "                         Evidence  Concluding Statement  \n",
       " 423A1CA112E2 precision  0.000000                   0.0  \n",
       "              recall     0.000000                   0.0  \n",
       "              f1         0.000000                   0.0  \n",
       " A8445CABFECE precision  0.000000                   0.0  \n",
       "              recall     0.000000                   0.0  \n",
       "              f1         0.000000                   0.0  \n",
       " 6B4F7A0165B9 precision  0.000000                   0.0  \n",
       "              recall     0.000000                   0.0  \n",
       "              f1         0.000000                   0.0  \n",
       " E05C7F5C1156 precision  0.000000                   0.0  \n",
       "              recall     0.000000                   0.0  \n",
       "              f1         0.000000                   0.0  \n",
       " 50B3435E475B precision  0.000000                   0.0  \n",
       "              recall     0.000000                   0.0  \n",
       "              f1         0.000000                   0.0  \n",
       " DBF7EB6A9E02 precision  0.000000                   0.0  \n",
       "              recall     0.000000                   0.0  \n",
       "              f1         0.000000                   0.0  \n",
       " 810B70E80E1D precision  0.333333                   0.0  \n",
       "              recall     0.333333                   0.0  \n",
       "              f1         0.333333                   0.0  \n",
       " CE98789F502B precision  0.000000                   0.0  \n",
       "              recall     0.000000                   0.0  \n",
       "              f1         0.000000                   0.0  \n",
       " A97DE0D49AEA precision  0.333333                   0.0  \n",
       "              recall     0.333333                   0.0  \n",
       "              f1         0.333333                   0.0  \n",
       " 48D3F4243F0F precision  0.000000                   0.0  \n",
       "              recall     0.000000                   NaN  \n",
       "              f1         0.000000                   0.0  )"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Only predicting on a subset\n",
    "ids_subset = train_df[\"id\"].unique()[:10]\n",
    "subset_ = train_df.set_index(\"id\", drop=False).loc[ids_subset]\n",
    "evaluate(subset_, bayesline_predict, DATA_PATH, model, vectorizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "insured-durham",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-26T10:53:22.701689Z",
     "start_time": "2022-01-26T10:53:22.621429Z"
    }
   },
   "source": [
    "Generating submission file on a subset of the data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "crazy-chuck",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:24:08.762195Z",
     "start_time": "2022-01-27T14:24:08.535846Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 10/10 [00:00<00:00, 52.29it/s]\n"
     ]
    }
   ],
   "source": [
    "# Only submitting a subset\n",
    "submission = [bayesline_predict(essay_id, DATA_PATH, model, vectorizer) for essay_id in tqdm(ids_subset)]\n",
    "submission_df = pd.concat(submission)\n",
    "\n",
    "#submission_df.to_csv(\"submission.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "combined-boards",
   "metadata": {},
   "source": [
    "Cross-val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "posted-authorization",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T15:17:26.674225Z",
     "start_time": "2022-01-27T15:00:14.016360Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3119/3119 [03:22<00:00, 15.42it/s]\n",
      "100%|██████████| 3119/3119 [03:21<00:00, 15.49it/s]\n",
      "100%|██████████| 3119/3119 [03:22<00:00, 15.42it/s]\n",
      "100%|██████████| 3119/3119 [03:18<00:00, 15.68it/s]\n",
      "100%|██████████| 3118/3118 [03:19<00:00, 15.65it/s]\n"
     ]
    }
   ],
   "source": [
    "ids_subset_2 = train_df[\"id\"].unique()\n",
    "np.random.shuffle(ids_subset_2)\n",
    "\n",
    "# Cross val on the entire dataset\n",
    "train_small = train_df.set_index(\"id\", drop=False).loc[ids_subset_2]\n",
    "X_small = train_small[\"discourse_text\"]\n",
    "y_small = train_small[\"discourse_type\"]\n",
    "\n",
    "kf = KFold(n_splits=5)\n",
    "all_scores = []\n",
    "for train, test in kf.split(ids_subset_2):\n",
    "    X_train, y_train = (X_small.loc[ids_subset_2[train]], y_small.loc[ids_subset_2[train]])\n",
    "    test_df = train_small.loc[ids_subset_2[test]]\n",
    "    model_cv, vectorizer_cv = bayesline_train(X_train, y_train)\n",
    "    all_scores.append(evaluate(test_df, bayesline_predict, DATA_PATH, model_cv, vectorizer_cv))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "liable-spelling",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T15:26:05.435541Z",
     "start_time": "2022-01-27T15:26:05.390838Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.1534513670313835,\n",
       " 0.15258110749419776,\n",
       " 0.15195740790471243,\n",
       " 0.14920631059720968,\n",
       " 0.14989431062165415]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[scores.mean() for (scores, _) in all_scores]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "uniform-channels",
   "metadata": {},
   "source": [
    "# Unused utils"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "vital-fitting",
   "metadata": {},
   "source": [
    "`display_classes_2` uses the `discourse_text` field of `train_df`. It is generally better than `display_classes_3` (which uses `discourse_start/end`), but when a element is present several times in the essay text, things get messy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "toxic-nylon",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:24:16.215432Z",
     "start_time": "2022-01-27T14:24:16.150096Z"
    }
   },
   "outputs": [],
   "source": [
    "def display_classes_2(essay_id, train_df):\n",
    "    elements_df = train_df[train_df[\"id\"] == essay_id]\n",
    "    essay_text = open(f'../raw_data/train/{essay_id}.txt').read()\n",
    "    for i, element in elements_df.iterrows():\n",
    "        element_text = element[\"discourse_text\"].strip()\n",
    "        # The stripping above is needed to make sure the replace below works\n",
    "        if not element_text in essay_text:\n",
    "            return \"Formatting failed\"\n",
    "        essay_text = essay_text.replace(\n",
    "            element_text,\n",
    "            f\"|<span style='color:{CLASSES_COLORS[element['discourse_type']]}'>{element_text}</span>\"\n",
    "        )\n",
    "    essay_text = essay_text.replace(\"\\n\", \"<br>\")\n",
    "    color_labels = \" - \".join([\n",
    "        f\"<span style='color:{CLASSES_COLORS[class_]}'>{class_}</span>\"\n",
    "        for class_ in CLASSES_COLORS.keys()])\n",
    "    return HTML(color_labels + \"<br><br>\" + essay_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "inner-condition",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:24:16.268319Z",
     "start_time": "2022-01-27T14:24:16.218912Z"
    }
   },
   "outputs": [],
   "source": [
    "def format_discourse(text, type_):\n",
    "    if text:\n",
    "        text = text.replace(\"\\n\", \"<br>\")\n",
    "        return f\"|<span style='color:{CLASSES_COLORS.get(type_)}'>{text}</span>\"\n",
    "    return \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "alleged-singer",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-01-27T14:24:16.322424Z",
     "start_time": "2022-01-27T14:24:16.270688Z"
    }
   },
   "outputs": [],
   "source": [
    "def display_classes_3(essay_id, train_df):\n",
    "    elements_df = train_df[train_df[\"id\"] == essay_id]\n",
    "    essay_text = open(f'../raw_data/train/{essay_id}.txt').read()\n",
    "    formatted_elements = \"\"\n",
    "    char_pointer = 0\n",
    "    for i, element in elements_df.iterrows():\n",
    "        unlabelled_element = essay_text[char_pointer:element['discourse_start']]\n",
    "        char_pointer = element['discourse_end'] + 1\n",
    "        discourse_element = essay_text[element['discourse_start']:char_pointer]\n",
    "        formatted_elements +=\\\n",
    "            format_discourse(unlabelled_element, \"Unlabelled\") +\\\n",
    "            format_discourse(discourse_element, element['discourse_type'])\n",
    "    formatted_elements += essay_text[char_pointer:]\n",
    "    color_labels = \" \".join([format_discourse(class_, class_) for class_ in CLASSES_COLORS.keys()])\n",
    "    return HTML(color_labels + \"<br><br>\" + formatted_elements)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}