tools/4_extract_routines_from_transcripts.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook extracts routines of referring expressions that are \"fixed\", i.e. become shared or established amongst interlocutors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "import pathlib as pl\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from spacy.lang.en import English\n",
    "\n",
    "from read_utils import read_tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def natural_sort(l):\n",
    "    convert = lambda text: int(text) if text.isdigit() else text.lower()\n",
    "    alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ]\n",
    "    return sorted(l, key = alphanum_key)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define paths."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created ../processed_data/dialign_inputs\n",
      "Created ../processed_data/dialign_outputs\n",
      "Created ../processed_data/routines\n",
      "Created ../processed_data/utterances\n",
      "Created ../processed_data/tokens\n"
     ]
    }
   ],
   "source": [
    "# Inputs.\n",
    "data_dir = pl.Path('../data')\n",
    "transcripts_dir = data_dir.joinpath('transcripts')\n",
    "\n",
    "dialign_dir = pl.Path('dialign-1.0')\n",
    "dialign_jar_file = dialign_dir.joinpath('dialign.jar')\n",
    "\n",
    "output_dir = pl.Path('../outputs')\n",
    "interm_dir = output_dir.joinpath('intermediate')\n",
    "task_features_file = interm_dir.joinpath(\n",
    "    'log_features/justhink19_log_features_task_level.csv')\n",
    "\n",
    "# Outputs.\n",
    "processed_data_dir = pl.Path('../processed_data')\n",
    "dialign_inputs_dir = processed_data_dir.joinpath('dialign_inputs')\n",
    "dialign_outputs_dir = processed_data_dir.joinpath('dialign_outputs')\n",
    "routines_dir = processed_data_dir.joinpath('routines')\n",
    "utterances_dir = processed_data_dir.joinpath('utterances')\n",
    "tokens_dir = processed_data_dir.joinpath('tokens')\n",
    "\n",
    "dirs = [\n",
    "    dialign_inputs_dir, dialign_outputs_dir,\n",
    "    routines_dir, \n",
    "    utterances_dir, tokens_dir,\n",
    "]\n",
    "for d in dirs:\n",
    "    if not d.exists():\n",
    "        d.mkdir()\n",
    "        print('Created {}'.format(d))\n",
    "\n",
    "\n",
    "synthesis_dep_file = dialign_outputs_dir.joinpath(\n",
    "    'metrics-speaker-dependent.tsv')\n",
    "synthesis_indep_file = dialign_outputs_dir.joinpath(\n",
    "    'metrics-speaker-dependent.tsv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define task-specific referents."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10 ['basel', 'bern', 'davos', 'gallen', 'interlaken', 'luzern', 'montreux', 'neuchatel', 'zermatt', 'zurich']\n"
     ]
    }
   ],
   "source": [
    "node_words = {\n",
    "    'basel',\n",
    "    'luzern',\n",
    "    'zurich',\n",
    "    'bern',\n",
    "    'zermatt',\n",
    "    'interlaken',\n",
    "    'montreux',\n",
    "    'neuchatel',\n",
    "    'gallen',\n",
    "    'davos',\n",
    "}\n",
    "\n",
    "task_words = node_words\n",
    "\n",
    "print(len(task_words), sorted(task_words))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read transcripts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading transcript files from ../data/transcripts.\n",
      "transcript 10 files found.\n",
      "File justhink19_transcript_07 belongs to team  7\n",
      "File justhink19_transcript_08 belongs to team  8\n",
      "File justhink19_transcript_09 belongs to team  9\n",
      "File justhink19_transcript_10 belongs to team 10\n",
      "File justhink19_transcript_11 belongs to team 11\n",
      "File justhink19_transcript_17 belongs to team 17\n",
      "File justhink19_transcript_18 belongs to team 18\n",
      "File justhink19_transcript_20 belongs to team 20\n",
      "File justhink19_transcript_28 belongs to team 28\n",
      "File justhink19_transcript_47 belongs to team 47\n",
      "Transcript of  7 has  639 utterances\n",
      "Transcript of  8 has  669 utterances\n",
      "Transcript of  9 has  810 utterances\n",
      "Transcript of 10 has  469 utterances\n",
      "Transcript of 11 has  567 utterances\n",
      "Transcript of 17 has  325 utterances\n",
      "Transcript of 18 has  359 utterances\n",
      "Transcript of 20 has  507 utterances\n",
      "Transcript of 28 has  348 utterances\n",
      "Transcript of 47 has  396 utterances\n"
     ]
    }
   ],
   "source": [
    "transcript_dfs = read_tables(transcripts_dir, form='transcript')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Refine task and transcript durations."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Compute speaking durations from transcripts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{7: 1573.338,\n",
       " 8: 1570.4229999999998,\n",
       " 9: 2160.63,\n",
       " 10: 1384.955,\n",
       " 11: 1580.5079999999998,\n",
       " 17: 1075.073,\n",
       " 18: 1804.0870000000002,\n",
       " 20: 1155.259,\n",
       " 28: 669.247,\n",
       " 47: 1178.847}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "end_times = dict()\n",
    "\n",
    "for team_no, df in transcript_dfs.items():       \n",
    "    dff = df[df['utterance'] == '(omitted)']\n",
    "    if len(dff) > 0:\n",
    "        end_time = dff['start'].min()\n",
    "    else:\n",
    "        end_time = df.iloc[-1]['end']\n",
    "                \n",
    "    end_times[team_no] = end_time\n",
    "\n",
    "end_times"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Print the total transcribed duration in hours."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.9312130555555553"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "values = [td / 60 / 60 for td in end_times.values()]\n",
    "sum(values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Slice the transcripts by their inferred duration. \n",
    "There is sometimes more talk after the task ends, some of which was also transcribed, we omit that.\n",
    "This is specifically when the team fails i.e. time is up, and we intervene."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "for team_no in transcript_dfs:\n",
    "    df = transcript_dfs[team_no]\n",
    "    df = df[df.end <= end_times[team_no]]\n",
    "    transcript_dfs[team_no] = df\n",
    "    \n",
    "# # A quick check.\n",
    "# transcript_dfs[7].tail(), end_times[7]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate inputs for dialign to extract routines."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define a tokeniser. \n",
    "Create a tokeniser with the default settings for English, including punctuation rules and exceptions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = English()\n",
    "tokeniser = nlp.tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define a tokeniser method for dialign (as per dialign input format). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenise_utterances(df, tokeniser):\n",
    "    df = df.copy()\n",
    "    texts = list()\n",
    "    for u in df['utterance']:\n",
    "        tokens = tokeniser(u)\n",
    "        text = ' '.join([t.text for t in tokens])\n",
    "        texts.append(text)\n",
    "\n",
    "    df['utterance'] = texts\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define an exporter for dialign. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def export_for_dialign(df, file):\n",
    "    with open(str(file), 'w') as f:\n",
    "        for i, row in df.iterrows():\n",
    "            print('{}\\t{}'.format(row['interlocutor'], row['utterance']),\n",
    "                  file=f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rework the transcripts for dialign: obtain simpler transcripts (tokenised and interlocutors A & B only)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reworking the transcripts to input into dialign...\n",
      "Processing Team 7\n",
      "Processing Team 8\n",
      "Processing Team 9\n",
      "Processing Team 10\n",
      "Processing Team 11\n",
      "Processing Team 17\n",
      "Processing Team 18\n",
      "Processing Team 20\n",
      "Processing Team 28\n",
      "Processing Team 47\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "print('Reworking the transcripts to input into dialign...')\n",
    "utterance_dfs = dict()\n",
    "for team_no, df in transcript_dfs.items():\n",
    "    print('Processing Team {}'.format(team_no))\n",
    "    # Filter for interlocutors A and B only.\n",
    "    df = df[df['interlocutor'].isin(['A', 'B'])]\n",
    "\n",
    "    # Tokenise.\n",
    "    df = tokenise_utterances(df, tokeniser)\n",
    "\n",
    "    # Reset the utterance numbers.\n",
    "    df['utterance_no'] = range(len(df))\n",
    "\n",
    "    # Keep.\n",
    "    utterance_dfs[team_no] = df\n",
    "\n",
    "print('Done!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Export the transcripts in dialign input format."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exporting the transcripts in dialign input format...\n",
      "Written for team  7 to ../processed_data/dialign_inputs/justhink19_dialogue_07.tsv\n",
      "Written for team  8 to ../processed_data/dialign_inputs/justhink19_dialogue_08.tsv\n",
      "Written for team  9 to ../processed_data/dialign_inputs/justhink19_dialogue_09.tsv\n",
      "Written for team 10 to ../processed_data/dialign_inputs/justhink19_dialogue_10.tsv\n",
      "Written for team 11 to ../processed_data/dialign_inputs/justhink19_dialogue_11.tsv\n",
      "Written for team 17 to ../processed_data/dialign_inputs/justhink19_dialogue_17.tsv\n",
      "Written for team 18 to ../processed_data/dialign_inputs/justhink19_dialogue_18.tsv\n",
      "Written for team 20 to ../processed_data/dialign_inputs/justhink19_dialogue_20.tsv\n",
      "Written for team 28 to ../processed_data/dialign_inputs/justhink19_dialogue_28.tsv\n",
      "Written for team 47 to ../processed_data/dialign_inputs/justhink19_dialogue_47.tsv\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "print('Exporting the transcripts in dialign input format...')\n",
    "\n",
    "for team_no, df in utterance_dfs.items():\n",
    "    # Construct filename.\n",
    "    file = 'justhink19_dialogue_{:02d}.tsv'.format(team_no)\n",
    "    file = dialign_inputs_dir.joinpath(file)\n",
    "    \n",
    "    # Export table to file.\n",
    "    export_for_dialign(df, file)\n",
    "\n",
    "    print('Written for team {:2d} to {}'.format(team_no, file))\n",
    "\n",
    "print('Done!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run dialign."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running for dialogues...\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "cmd = 'java -jar {} -i {} -o {}'.format(\n",
    "    dialign_jar_file.resolve(), \n",
    "    dialign_inputs_dir.resolve(), \n",
    "    dialign_outputs_dir.resolve())\n",
    "# print(cmd)\n",
    "\n",
    "print('Running for dialogues...')\n",
    "!$cmd\n",
    "print('Done!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read routine tables.\n",
    "i.e. shared expression lexicons as termed by dialign."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read for team 07: 384 routines\n",
      "Read for team 08: 420 routines\n",
      "Read for team 09: 533 routines\n",
      "Read for team 10: 226 routines\n",
      "Read for team 11: 371 routines\n",
      "Read for team 17: 149 routines\n",
      "Read for team 18: 149 routines\n",
      "Read for team 20: 287 routines\n",
      "Read for team 28: 194 routines\n",
      "Read for team 47: 223 routines\n"
     ]
    }
   ],
   "source": [
    "routine_dfs = dict()\n",
    "\n",
    "for team_no in sorted(transcript_dfs):\n",
    "    routine_file = 'justhink19_dialogue_{:02d}_tsv-lexicon.tsv'.format(\n",
    "        team_no)\n",
    "    routine_file = dialign_outputs_dir.joinpath(routine_file)\n",
    "    df = pd.read_csv(str(routine_file), sep='\\t')\n",
    "    print('Read for team {:02d}: {} routines'.format(team_no, len(df)))\n",
    "\n",
    "    l = list()\n",
    "    for e in df['Surface Form']:\n",
    "        tokenised_e = [t.text for t in tokeniser(e)]\n",
    "        v = 0\n",
    "        for n in task_words:\n",
    "            if n in tokenised_e:\n",
    "                v += 1\n",
    "        l.append(v)\n",
    "    df.insert(3, 'task_spec_referent_count', l)\n",
    "\n",
    "    df['utterances'] = [[int(n) for n in seq.split(', ')]\n",
    "                        for seq in df['Turns']]\n",
    "\n",
    "    routine_dfs[team_no] = df\n",
    "\n",
    "# # Example/debugging.\n",
    "# team_no = 18\n",
    "# routine_dfs[team_no].head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter for routines with task-specific referents."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "for team_no, df in routine_dfs.items():\n",
    "    df = df[df.task_spec_referent_count > 0]\n",
    "    df = df.drop('task_spec_referent_count', axis=1)\n",
    "    routine_dfs[team_no] = df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rework routine instances with token positions."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Construct token tables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>team_no</th>\n",
       "      <th>utterance_no</th>\n",
       "      <th>token_no</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "      <th>interlocutor</th>\n",
       "      <th>utterance</th>\n",
       "      <th>token</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>42.525</td>\n",
       "      <td>43.004</td>\n",
       "      <td>A</td>\n",
       "      <td>okay .</td>\n",
       "      <td>okay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>42.525</td>\n",
       "      <td>43.004</td>\n",
       "      <td>A</td>\n",
       "      <td>okay .</td>\n",
       "      <td>.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>47.732</td>\n",
       "      <td>48.615</td>\n",
       "      <td>A</td>\n",
       "      <td>i am doing this .</td>\n",
       "      <td>i</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>47.732</td>\n",
       "      <td>48.615</td>\n",
       "      <td>A</td>\n",
       "      <td>i am doing this .</td>\n",
       "      <td>am</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>47.732</td>\n",
       "      <td>48.615</td>\n",
       "      <td>A</td>\n",
       "      <td>i am doing this .</td>\n",
       "      <td>doing</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   team_no  utterance_no  token_no   start     end interlocutor  \\\n",
       "1        7             0         0  42.525  43.004            A   \n",
       "1        7             0         1  42.525  43.004            A   \n",
       "2        7             1         2  47.732  48.615            A   \n",
       "2        7             1         3  47.732  48.615            A   \n",
       "2        7             1         4  47.732  48.615            A   \n",
       "\n",
       "           utterance  token  \n",
       "1             okay .   okay  \n",
       "1             okay .      .  \n",
       "2  i am doing this .      i  \n",
       "2  i am doing this .     am  \n",
       "2  i am doing this .  doing  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_dfs = dict()\n",
    "for team_no, df in utterance_dfs.items():\n",
    "    df = df.copy()\n",
    "\n",
    "    # Split the utterances into words, convert to a list.\n",
    "    df['token'] = [u.split() for u in df['utterance']]\n",
    "    # df = df.assign(**{'words': df['object'].str.split()})\n",
    "\n",
    "    # Transform each word to a row, preserving the other values in the row.\n",
    "    df = df.explode('token')\n",
    "\n",
    "    # Assign a subutterance no.\n",
    "    df.insert(2, 'token_no', range(len(df)))\n",
    "\n",
    "    token_dfs[team_no] = df\n",
    "\n",
    "df = token_dfs[7].copy()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_sub_list(sl, l):\n",
    "    # allows for multiple matches\n",
    "    # from https://stackoverflow.com/a/17870684\n",
    "    results = []\n",
    "    sll = len(sl)\n",
    "    for ind in (i for i, e in enumerate(l) if e == sl[0]):\n",
    "        if l[ind:ind+sll] == sl:\n",
    "            results.append((ind, ind+sll-1))\n",
    "\n",
    "    return results\n",
    "\n",
    "# # Try.\n",
    "# greeting = ['hello', 'my', 'name', 'is', 'bob',\n",
    "#             'how', 'are', 'you', 'my', 'name', 'is']\n",
    "# print(find_sub_list(['my', 'name', 'is'], greeting))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find routine expressions' subutterance numbers from utterance numbers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_start_indices(subutterance, u_no, u_df):\n",
    "    l = list()  # subutterance list to be built.\n",
    "\n",
    "    # Find the utterance (row) with that utterance no.\n",
    "    utterance_row = u_df[u_df.utterance_no == u_no]\n",
    "    # Make sure there is only one such row.\n",
    "    assert len(utterance_row) == 1, print(\n",
    "        'Multiple utterances found at {}'.format(u_no))\n",
    "    # Select the first (and only) row.\n",
    "    utterance_row = utterance_row.iloc[0]\n",
    "    # Get the utterance string at that row.\n",
    "    utterance = utterance_row['utterance']\n",
    "\n",
    "    # Find the occurrences of subutterance routine in the utterance.\n",
    "    indices = find_sub_list(subutterance.split(), utterance.split())\n",
    "    assert len(indices) != 0, print(\n",
    "        'Could not find subutterance \"{}\" at utterance \"{}\" ({})'.format(\n",
    "            subutterance, utterance, u_no))\n",
    "\n",
    "    # Get the token offset of the utterance.\n",
    "    offset = t_df[t_df.utterance_no == u_no].iloc[0]['token_no']\n",
    "\n",
    "    # Put the initial positions of the occurrences into a list.\n",
    "    for start, end in indices:\n",
    "        l.append(start + offset)\n",
    "\n",
    "    return l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finding routine token indices for team  7\n",
      "Finding routine token indices for team  8\n",
      "Finding routine token indices for team  9\n",
      "Finding routine token indices for team 10\n",
      "Finding routine token indices for team 11\n",
      "Finding routine token indices for team 17\n",
      "Finding routine token indices for team 18\n",
      "Finding routine token indices for team 20\n",
      "Finding routine token indices for team 28\n",
      "Finding routine token indices for team 47\n",
      "Done!\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Freq.</th>\n",
       "      <th>Free Freq.</th>\n",
       "      <th>Size</th>\n",
       "      <th>Surface Form</th>\n",
       "      <th>Establishment turn</th>\n",
       "      <th>Spanning</th>\n",
       "      <th>Priming</th>\n",
       "      <th>First Speaker</th>\n",
       "      <th>Turns</th>\n",
       "      <th>utterances</th>\n",
       "      <th>tokens</th>\n",
       "      <th>priming_token</th>\n",
       "      <th>establish_token</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>go to mount saint gallen .</td>\n",
       "      <td>508</td>\n",
       "      <td>145</td>\n",
       "      <td>2</td>\n",
       "      <td>A</td>\n",
       "      <td>364, 395, 508</td>\n",
       "      <td>[364, 395, 508]</td>\n",
       "      <td>[2023, 2164, 2776]</td>\n",
       "      <td>2023</td>\n",
       "      <td>2776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>mount montreux , mount montreux .</td>\n",
       "      <td>380</td>\n",
       "      <td>114</td>\n",
       "      <td>1</td>\n",
       "      <td>A</td>\n",
       "      <td>267, 380</td>\n",
       "      <td>[267, 380]</td>\n",
       "      <td>[1525, 2084]</td>\n",
       "      <td>1525</td>\n",
       "      <td>2084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>to mount saint gallen .</td>\n",
       "      <td>508</td>\n",
       "      <td>251</td>\n",
       "      <td>5</td>\n",
       "      <td>A</td>\n",
       "      <td>258, 287, 316, 364, 395, 508</td>\n",
       "      <td>[258, 287, 316, 364, 395, 508]</td>\n",
       "      <td>[1478, 1627, 1806, 2024, 2165, 2777]</td>\n",
       "      <td>1478</td>\n",
       "      <td>2777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>from mount davos to mount</td>\n",
       "      <td>559</td>\n",
       "      <td>21</td>\n",
       "      <td>2</td>\n",
       "      <td>B</td>\n",
       "      <td>539, 541, 559</td>\n",
       "      <td>[539, 541, 559]</td>\n",
       "      <td>[2933, 2949, 3046]</td>\n",
       "      <td>2933</td>\n",
       "      <td>3046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>mount davos to mount zermatt</td>\n",
       "      <td>559</td>\n",
       "      <td>329</td>\n",
       "      <td>1</td>\n",
       "      <td>B</td>\n",
       "      <td>259, 559, 587</td>\n",
       "      <td>[259, 559, 587]</td>\n",
       "      <td>[1483, 3047, 3197]</td>\n",
       "      <td>1483</td>\n",
       "      <td>3047</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Freq.  Free Freq.  Size                       Surface Form  \\\n",
       "0      3           3     6         go to mount saint gallen .   \n",
       "1      2           2     6  mount montreux , mount montreux .   \n",
       "2      6           3     5            to mount saint gallen .   \n",
       "3      3           3     5          from mount davos to mount   \n",
       "4      3           3     5       mount davos to mount zermatt   \n",
       "\n",
       "   Establishment turn  Spanning  Priming First Speaker  \\\n",
       "0                 508       145        2             A   \n",
       "1                 380       114        1             A   \n",
       "2                 508       251        5             A   \n",
       "3                 559        21        2             B   \n",
       "4                 559       329        1             B   \n",
       "\n",
       "                          Turns                      utterances  \\\n",
       "0                 364, 395, 508                 [364, 395, 508]   \n",
       "1                      267, 380                      [267, 380]   \n",
       "2  258, 287, 316, 364, 395, 508  [258, 287, 316, 364, 395, 508]   \n",
       "3                 539, 541, 559                 [539, 541, 559]   \n",
       "4                 259, 559, 587                 [259, 559, 587]   \n",
       "\n",
       "                                 tokens  priming_token  establish_token  \n",
       "0                    [2023, 2164, 2776]           2023             2776  \n",
       "1                          [1525, 2084]           1525             2084  \n",
       "2  [1478, 1627, 1806, 2024, 2165, 2777]           1478             2777  \n",
       "3                    [2933, 2949, 3046]           2933             3046  \n",
       "4                    [1483, 3047, 3197]           1483             3047  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for team_no, df in routine_dfs.items():\n",
    "    print('Finding routine token indices for team {:2d}'.format(\n",
    "        team_no))\n",
    "\n",
    "    u_df = utterance_dfs[team_no]\n",
    "    t_df = token_dfs[team_no]\n",
    "    tokens_list = list()\n",
    "    establish_list = list()\n",
    "    priming_list = list()\n",
    "    for i, row in df.iterrows():\n",
    "        subutterance = row['Surface Form']\n",
    "\n",
    "        # subutterance list to be built, for each row.\n",
    "        l = list()\n",
    "        for u_no in row['utterances']:  # for each utterance no\n",
    "            l += get_start_indices(subutterance, u_no, u_df)\n",
    "        tokens_list.append(l)\n",
    "\n",
    "        # priming token from the priming utterance no.\n",
    "        u_no = row['utterances'][0]\n",
    "        t = get_start_indices(subutterance, u_no, u_df)[0]\n",
    "        priming_list.append(t)\n",
    "\n",
    "        # establishment token from the establishment utterance no.\n",
    "        u_no = row['Establishment turn']\n",
    "        t = get_start_indices(subutterance, u_no, u_df)[0]\n",
    "        establish_list.append(t)\n",
    "\n",
    "    df['tokens'] = tokens_list\n",
    "    df['priming_token'] = priming_list\n",
    "    df['establish_token'] = establish_list\n",
    "\n",
    "\n",
    "print('Done!')\n",
    "\n",
    "routine_dfs[7].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export routine tables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exporting routine tables...\n",
      "Exported routines for  7 to ../processed_data/routines/justhink19_routines_07.csv\n",
      "Exported routines for  8 to ../processed_data/routines/justhink19_routines_08.csv\n",
      "Exported routines for  9 to ../processed_data/routines/justhink19_routines_09.csv\n",
      "Exported routines for 10 to ../processed_data/routines/justhink19_routines_10.csv\n",
      "Exported routines for 11 to ../processed_data/routines/justhink19_routines_11.csv\n",
      "Exported routines for 17 to ../processed_data/routines/justhink19_routines_17.csv\n",
      "Exported routines for 18 to ../processed_data/routines/justhink19_routines_18.csv\n",
      "Exported routines for 20 to ../processed_data/routines/justhink19_routines_20.csv\n",
      "Exported routines for 28 to ../processed_data/routines/justhink19_routines_28.csv\n",
      "Exported routines for 47 to ../processed_data/routines/justhink19_routines_47.csv\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "print('Exporting routine tables...')\n",
    "\n",
    "for team_no, df in routine_dfs.items():\n",
    "    # Construct filename.\n",
    "    file = 'justhink19_routines_{:02d}.csv'.format(team_no)\n",
    "    file = routines_dir.joinpath(file)\n",
    "\n",
    "    # Write the table to file.\n",
    "    df.to_csv(file, index=False, sep='\\t')\n",
    "\n",
    "    print('Exported routines for {:2d} to {}'.format(team_no, file))\n",
    "    \n",
    "print('Done!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export the simplified transcripts (\"utterances\")."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exporting tokenised filtered transcripts (utterances)\n",
      "Exported utterances for  7 to ../processed_data/utterances/justhink19_utterances_07.csv\n",
      "Exported utterances for  8 to ../processed_data/utterances/justhink19_utterances_08.csv\n",
      "Exported utterances for  9 to ../processed_data/utterances/justhink19_utterances_09.csv\n",
      "Exported utterances for 10 to ../processed_data/utterances/justhink19_utterances_10.csv\n",
      "Exported utterances for 11 to ../processed_data/utterances/justhink19_utterances_11.csv\n",
      "Exported utterances for 17 to ../processed_data/utterances/justhink19_utterances_17.csv\n",
      "Exported utterances for 18 to ../processed_data/utterances/justhink19_utterances_18.csv\n",
      "Exported utterances for 20 to ../processed_data/utterances/justhink19_utterances_20.csv\n",
      "Exported utterances for 28 to ../processed_data/utterances/justhink19_utterances_28.csv\n",
      "Exported utterances for 47 to ../processed_data/utterances/justhink19_utterances_47.csv\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "print('Exporting tokenised filtered transcripts (utterances)')\n",
    "for team_no, df in utterance_dfs.items():\n",
    "    file = 'justhink19_utterances_{:02d}.csv'.format(team_no)\n",
    "    file = utterances_dir.joinpath(file)\n",
    "\n",
    "    # Export table to file.\n",
    "    df.to_csv(file, index=False, float_format='%.3f', sep='\\t')\n",
    "\n",
    "    print('Exported utterances for {:2d} to {}'.format(team_no, file))\n",
    "\n",
    "print('Done!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Export the token tables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exported tokens for  7 to ../processed_data/tokens/justhink19_tokens_07.csv\n",
      "Exported tokens for  8 to ../processed_data/tokens/justhink19_tokens_08.csv\n",
      "Exported tokens for  9 to ../processed_data/tokens/justhink19_tokens_09.csv\n",
      "Exported tokens for 10 to ../processed_data/tokens/justhink19_tokens_10.csv\n",
      "Exported tokens for 11 to ../processed_data/tokens/justhink19_tokens_11.csv\n",
      "Exported tokens for 17 to ../processed_data/tokens/justhink19_tokens_17.csv\n",
      "Exported tokens for 18 to ../processed_data/tokens/justhink19_tokens_18.csv\n",
      "Exported tokens for 20 to ../processed_data/tokens/justhink19_tokens_20.csv\n",
      "Exported tokens for 28 to ../processed_data/tokens/justhink19_tokens_28.csv\n",
      "Exported tokens for 47 to ../processed_data/tokens/justhink19_tokens_47.csv\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "for team_no, df in token_dfs.items():\n",
    "    file = 'justhink19_tokens_{:02d}.csv'.format(team_no)\n",
    "    file = tokens_dir.joinpath(file)\n",
    "\n",
    "    # Export table to file.\n",
    "    df.to_csv(file, index=False, float_format='%.3f', sep='\\t')\n",
    "    \n",
    "    print('Exported tokens for {:2d} to {}'.format(team_no, file))\n",
    "    \n",
    "print('Done!')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (JUSThink-alignment)",
   "language": "python",
   "name": "justhink-alignment-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "274px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}