generating_and_annotating_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d69952a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: nltk in /gpfs/space/home/shuva/miniconda3/envs/gpt-for-data/lib/python3.8/site-packages (3.8.1)\n",
      "Requirement already satisfied: tqdm in /gpfs/space/home/shuva/miniconda3/envs/gpt-for-data/lib/python3.8/site-packages (from nltk) (4.65.0)\n",
      "Requirement already satisfied: click in /gpfs/space/home/shuva/miniconda3/envs/gpt-for-data/lib/python3.8/site-packages (from nltk) (7.1.2)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /gpfs/space/home/shuva/miniconda3/envs/gpt-for-data/lib/python3.8/site-packages (from nltk) (2023.3.23)\n",
      "Requirement already satisfied: joblib in /gpfs/space/home/shuva/miniconda3/envs/gpt-for-data/lib/python3.8/site-packages (from nltk) (1.2.0)\n"
     ]
    }
   ],
   "source": [
    "import openai\n",
    "import torch\n",
    "from transformers import AutoTokenizer, GPT2LMHeadModel, GenerationConfig\n",
    "from datasets import Dataset\n",
    "\n",
    "import os\n",
    "import csv\n",
    "import json\n",
    "import re\n",
    "import random\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "! pip install nltk\n",
    "from nltk.tokenize import TreebankWordTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fc8653cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL_PATH = \"estMed-gpt2_fine_tuned3\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ed6c99eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)\n",
    "model = GPT2LMHeadModel.from_pretrained(MODEL_PATH)\n",
    "generation_config = GenerationConfig(\n",
    "    max_new_tokens=400, do_sample=True, top_k=40, eos_token_id=model.config.eos_token_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3ab84b7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_text(beginning: str):\n",
    "    inputs = tokenizer(beginning, return_tensors=\"pt\")\n",
    "    with torch.no_grad():\n",
    "        outputs = model.generate(**inputs, generation_config=generation_config, pad_token_id=tokenizer.eos_token_id)\n",
    "    return tokenizer.batch_decode(outputs, skip_special_tokens=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b5d95dac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['protseduur, vanusegrupp 10, naine, C50-50, ultraheli DATE tsütostaatilise ravikuuri planeerimine ja manustamine, kuni 24 tundi DATE mammograafia']\n"
     ]
    }
   ],
   "source": [
    "test_text = generate_text(\"protseduur, vanusegrupp 10, naine, C50-50,\")\n",
    "print(test_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "89a3170f",
   "metadata": {},
   "outputs": [],
   "source": [
    "disease = 'C50-50'\n",
    "gender = 'naine'\n",
    "\n",
    "synth_path = 'test_data'\n",
    "\n",
    "different_beginnings = ['protseduur', 'anamnees']\n",
    "\n",
    "for prompt_genre in different_beginnings:\n",
    "    \n",
    "    texts = []\n",
    "    \n",
    "    for i in range(1):\n",
    "        prompt = \"{genre}, vanusegrupp {age}, {gender}, {disease},\".format(\n",
    "        genre = prompt_genre,\n",
    "        age = str(random.randint(1,14)),\n",
    "        gender = gender,\n",
    "        disease = disease\n",
    "        )\n",
    "        texts.append(generate_text(prompt))\n",
    "    \n",
    "    filename = synth_path + prompt_genre.replace(\" \", \"_\") + \"_test.csv\"\n",
    "    \n",
    "    with open(filename, 'w', newline='') as csvfile:\n",
    "        spamwriter = csv.writer(csvfile, delimiter='\\t',\n",
    "                                quotechar='|', quoting=csv.QUOTE_MINIMAL)\n",
    "        \n",
    "        for text in texts:\n",
    "            spamwriter.writerow(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "fb9657ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = synth_path + prompt_genre.replace(\" \", \"_\") + \"_test.csv\"\n",
    "    \n",
    "with open(filename, 'w', newline='') as csvfile:\n",
    "    spamwriter = csv.writer(csvfile, delimiter='\\t',\n",
    "                            quotechar='|', quoting=csv.QUOTE_MINIMAL)\n",
    "\n",
    "    for text in texts:\n",
    "        spamwriter.writerow(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48108321",
   "metadata": {},
   "source": [
    "### Check the originality of the generated texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "731b542b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from originality import lcs\n",
    "from transformers import AutoTokenizer\n",
    "from tqdm import trange"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "928b43ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_in_file(file):\n",
    "    if file.endswith('.csv'):\n",
    "        texts = pd.read_csv(file, sep='\\t')\n",
    "    if file.endswith('.tsv'):\n",
    "        texts = pd.read_csv(file, sep='\\t')\n",
    "    if file.endswith('.pkl'):\n",
    "        texts = pd.read_pickle(file)\n",
    "    \n",
    "    return texts\n",
    "\n",
    "def calculate_lcs(target_texts_file,\n",
    "                  reference_texts_file,\n",
    "                  tokenizer_folder,\n",
    "                  top_k,\n",
    "                  save_folder,\n",
    "                  save_file_name):\n",
    "        \n",
    "    if not os.path.exists(save_folder):\n",
    "        os.makedirs(save_folder)\n",
    "\n",
    "    # Read in the data\n",
    "    texts = read_in_file(target_texts_file)\n",
    "    target_texts = []\n",
    "    for i in range(texts.shape[0]):\n",
    "        text = texts.iloc[i,0]\n",
    "        text = ', '.join(text.split(', ')[4:])\n",
    "        target_texts.append(text)\n",
    "\n",
    "    references = read_in_file(reference_texts_file) # We assume that the references are already tokenized.\n",
    "\n",
    "    # Read in the tokenizer\n",
    "    tokenizer = AutoTokenizer.from_pretrained(tokenizer_folder)\n",
    "\n",
    "    # Tokenize the targets texts\n",
    "    targets = tokenizer(target_texts)['input_ids']\n",
    "\n",
    "    lcs_elements = np.zeros((len(targets), top_k), dtype=np.float32)\n",
    "    lcs_indices = np.zeros((len(targets), top_k), dtype=np.float32)\n",
    "\n",
    "    batch_size=500\n",
    "    batch_size2=32768\n",
    "\n",
    "    lcs_matrix = np.zeros((batch_size, len(references)), dtype=np.float32)\n",
    "\n",
    "    # Calculate the LCS in batches\n",
    "    for i in trange(int(np.ceil(len(targets)/batch_size))):\n",
    "        for k in trange(int(np.ceil(len(references)/batch_size2))):\n",
    "            if (i+1)*batch_size < len(targets):\n",
    "                lcs_matrix[:,k*batch_size2:(k+1)*batch_size2] = lcs.check_originality(targets[i*batch_size:(i+1)*batch_size], references[k*batch_size2:(k+1)*batch_size2])\n",
    "            else:\n",
    "                lcs_matrix[:len(targets)-i*batch_size,k*batch_size2:(k+1)*batch_size2] = lcs.check_originality(targets[i*batch_size:(i+1)*batch_size], references[k*batch_size2:(k+1)*batch_size2])\n",
    "        # Get the top k\n",
    "        top_indices = np.argsort(-lcs_matrix, axis=1)[:, :top_k]\n",
    "        top_elements = np.take_along_axis(lcs_matrix, top_indices, axis=1)\n",
    "        \n",
    "        if (i+1)*batch_size < len(targets):\n",
    "            lcs_elements[i*batch_size:(i+1)*batch_size,:] = top_elements\n",
    "            lcs_indices[i*batch_size:(i+1)*batch_size,:] = top_indices\n",
    "        else:\n",
    "            lcs_elements[i*batch_size:,:] = top_elements[:len(targets)-i*batch_size,:]\n",
    "            lcs_indices[i*batch_size:,:] = top_indices[:len(targets)-i*batch_size,:]\n",
    "\n",
    "    # Save the lcs\n",
    "    save_file = os.path.join(save_folder, '.'.join(save_file_name, 'npy'))\n",
    "    np.save(save_file, lcs_elements)\n",
    "\n",
    "    save_file = os.path.join(save_folder, f'{save_file_name}_selected.npy')\n",
    "    np.save(save_file, lcs_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d532cd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "lcs_folder = \"lcs_validated_ids/selected\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65adbee9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# We assume that the references in reference_texts_file are already tokenized.\n",
    "calculate_lcs(f\"generated_data_12okt/batch{batch_nr}/{document_type}.csv\", 'data/train_tokenized.pkl', MODEL_PATH, 50, lcs_folder, f\"batch{batch_nr}_{document_type}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33a4442f",
   "metadata": {},
   "source": [
    "## Reading in the correct LCS verified synthetic texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "f6f78f4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "lcs_folder = \"lcs_validated_ids/selected\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "24c92574",
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text_of_prefix(text):\n",
    "    text_ = text.split(\",\")\n",
    "    return \",\".join(text_[4:])[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "977587d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def return_okay_texts(batch_nr, document_type):\n",
    "    tekstid_all = []\n",
    "    tekstid_filtered = []\n",
    "    with open(\"generated_data_12okt/batch\" + str(batch_nr)+ \"/\" + str(document_type) + \".csv\") as csvfile:\n",
    "        lugeja = csv.reader(csvfile, delimiter='\\t',quotechar='|', quoting=csv.QUOTE_MINIMAL)\n",
    "        for row in lugeja:\n",
    "            tekstid_all.append(row)\n",
    "    \n",
    "    idx = np.load(\"lcs_validated_ids/selected/batch\" + str(batch_nr) + \"_\" + str(document_type) + \"_selected.npy\")\n",
    "    for index in idx:\n",
    "        tekstid_filtered.append(clean_text_of_prefix(tekstid_all[index][0]))\n",
    "    \n",
    "    return tekstid_filtered\n",
    "            \n",
    "\n",
    "filtered_texts_anamnesis = return_okay_texts(1, \"anamnees\") + return_okay_texts(2, \"anamnees\")\n",
    "filtered_texts_procedures = return_okay_texts(1, \"protseduur\") + return_okay_texts(2, \"protseduur\") + return_okay_texts(4, \"protseduur\") + return_okay_texts(5, \"protseduur\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "a53f110f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "542"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(filtered_texts_anamnesis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "49eb2184",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "480"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(filtered_texts_procedures)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "511c802b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'pöördus kontrolli. DATE vasaku rinna sektorresektsioon + snb, adjuvantravina rinna kiiritus + zoladex + tamoxifen. DATE konsiiliumi otsusega herceptin-ravi (tmx). DATE.14 mmgr, uh ja jnb. DATE vasaku rinna rekonstruktsioon tram lapiga. DATE konsiiliumi otsusega näidustatud adjuvantne keemia-, bioloogiline-, hormoontaastusravi. kuna kasvaja hormoonsõltuv, plaanis alustada ravi zoladex + tamoxifeniga. obj: rindades tihendeid ei palpeeri. NAME. palp. bilat.ii. tellitud ca15-3'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_texts_anamnesis[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3073c2bd",
   "metadata": {},
   "source": [
    "## Azure API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a51628b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# api setup, removed for public repo\n",
    "\n",
    "openai.api_type = \"azure\"\n",
    "\n",
    "with open('', 'r') as api_file:\n",
    "    openai.api_key=str(api_file.readline()).strip()\n",
    "    \n",
    "openai.api_base = \"\"\n",
    "openai.api_version = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "0c33f3cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts = []\n",
    "responses = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "11971e90",
   "metadata": {},
   "outputs": [],
   "source": [
    "def ask_openai(prompt: str) -> str:\n",
    "    try:\n",
    "        response = openai.ChatCompletion.create(\n",
    "            deployment_id = \"gpt-35-experiments-health\",\n",
    "            model = \"gpt-35-turbo\",\n",
    "            messages=[{\"role\": \"user\", \"content\": prompt}]\n",
    "        )\n",
    "\n",
    "    except:\n",
    "        response = openai.ChatCompletion.create(\n",
    "            deployment_id = \"gpt-35-experiments-health\",\n",
    "            model = \"gpt-35-turbo\",\n",
    "            messages=[{\"role\": \"user\", \"content\": prompt}]\n",
    "        )\n",
    "    responses.append(response['choices'][0]['message']['content'])\n",
    "    prompts.append(prompt)\n",
    "    return response['choices'][0]['message']['content']\n",
    "\n",
    "# ask_openai(\"What is your name?\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57a31e48",
   "metadata": {},
   "source": [
    "## Zero shot prompt example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "3f0137ce",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "In the text below, give the list of: drug named entity, procedure named entity, family history named entity. Words need to be in exactly the same format as in input text. Format the output in JSON with the following keys: DRUG for drug named entity, PROCEDURE for procedure named entity, FAMILY for family history named entity. Text below: \n",
      "\"pöördus kontrolli. DATE vasaku rinna sektorresektsioon + snb, adjuvantravina rinna kiiritus + zoladex + tamoxifen. DATE konsiiliumi otsusega herceptin-ravi (tmx). DATE.14 mmgr, uh ja jnb. DATE vasaku rinna rekonstruktsioon tram lapiga. DATE konsiiliumi otsusega näidustatud adjuvantne keemia-, bioloogiline-, hormoontaastusravi. kuna kasvaja hormoonsõltuv, plaanis alustada ravi zoladex + tamoxifeniga. obj: rindades tihendeid ei palpeeri. NAME. palp. bilat.ii. tellitud ca15-3\"\n"
     ]
    }
   ],
   "source": [
    "output_data = 'drug named entity, procedure named entity, family history named entity.'\n",
    "output_format = 'DRUG for drug named entity, PROCEDURE for procedure named entity, FAMILY for family history named entity.'\n",
    "\n",
    "base_prompt_zero_shot = \"In the text below, give the list of: \" + output_data + \" Words need to be in exactly the same format as in input text. Format the output in JSON with the following keys: \" + output_format + \" Text below: \"\n",
    "print(base_prompt_zero_shot + \"\\n\" + \"\\\"\" + filtered_texts_anamnesis[1] + \"\\\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "f6d4dd66",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompts = []\n",
    "responses = []\n",
    "\n",
    "test_sentences = [filtered_texts_anamnesis[1]]\n",
    "try:\n",
    "    for sent in test_sentences:\n",
    "        ask_openai(base_prompt_zero_shot + \"\\n\" + \"\\\"\" + sent + \"\\\"\")\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "0715c050",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['{\\n   \"DRUG\": [\"zoladex\", \"tamoxifen\", \"herceptin-ravi\"],\\n   \"PROCEDURE\": [\"sektorresektsioon\", \"snb\", \"kiiritus\", \"rinna rekonstruktsioon\", \"tram lapiga\", \"adjuvantne keemia-, bioloogiline-, hormoontaastusravi\"],\\n   \"FAMILY\": []\\n}']"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "responses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "aaff864d",
   "metadata": {},
   "outputs": [],
   "source": [
    "entities = ['DRUG', 'PROCEDURE', 'FAMILY']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "0736a4c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_prompt(clinical_text: str, response: str, entities):\n",
    "    \n",
    "    results = {key: [] for key in entities}\n",
    "    list_results = []\n",
    "    \n",
    "    json_data = json.loads(response)\n",
    "    # print(json_data)\n",
    "    \n",
    "    for entity in entities:\n",
    "        # entity = DISEASE\n",
    "        if entity in json_data:\n",
    "            for finding in json_data[entity]:\n",
    "                \n",
    "                if finding in ['none', 'None']:\n",
    "                    pass\n",
    "                \n",
    "                pattern = re.compile(finding.lower())\n",
    "                # print(pattern.finditer(clinical_text.lower()))\n",
    "                for match in pattern.finditer(clinical_text.lower()):\n",
    "                    start = match.start()\n",
    "                    end = match.end()\n",
    "                    results[entity].append((start, end))\n",
    "                    # print(start, end, finding, entity)\n",
    "                    dict_result = {'entity_type': entity, 'start_idx': start, 'end_idx': end, 'text': clinical_text[start:end]}\n",
    "                    if dict_result not in list_results:\n",
    "                        list_results.append(dict_result)\n",
    "                    \n",
    "    return list_results\n",
    "\n",
    "# results_ = [parse_prompt(test_sentences[testid], responses[testid], entities) for testid in range(len(responses))]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "746e6766",
   "metadata": {},
   "source": [
    "## Parse to training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "cc73a44d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def parsed_results_to_train(parsed_results, clinical_results, entities):\n",
    "    \n",
    "    tokens_and_tags = []\n",
    "    \n",
    "    res = list(TreebankWordTokenizer().span_tokenize(clinical_results))\n",
    "    dic = {k:v for k,v in zip(res, [clinical_results[i:j] for i, j in res])}\n",
    "    \n",
    "    for entry in dic:\n",
    "        \n",
    "        token = dic[entry]\n",
    "        tags = []\n",
    "        \n",
    "        # print(entry, dic[entry])\n",
    "        tokenized_span_start = entry[0]\n",
    "        tokenized_span_end = entry[1]\n",
    "        \n",
    "        for parse in parsed_results:\n",
    "            parse_span_start = parse['start_idx'] # 148\n",
    "            parse_span_end = parse['end_idx'] # 172\n",
    "            parse_entity = parse['entity_type'] # DISEASE\n",
    "            \n",
    "            if len(range(max(parse_span_start, tokenized_span_start), min(parse_span_end, tokenized_span_end))) > 0:\n",
    "                if parse_entity not in tags:\n",
    "                    tags.append(parse_entity)\n",
    "        \n",
    "        if len(tags) == 0:\n",
    "            tags.append('O')\n",
    "        \n",
    "        tokens_and_tags.append((token, tags))\n",
    "        \n",
    "    return tokens_and_tags\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "0ef51385",
   "metadata": {},
   "outputs": [],
   "source": [
    "successful_texts = []\n",
    "parsed_answers = []\n",
    "\n",
    "for ans_, og_text in zip(responses, test_sentences):\n",
    "    try:\n",
    "        parsed_answer = parse_prompt(og_text, ans_[ans_.find('{'):ans_.find('}')+1], entities)\n",
    "        \n",
    "        parsed_answers.append(parsed_answer)\n",
    "        successful_texts.append(og_text)\n",
    "    except:\n",
    "        print(\"----------\")\n",
    "        print(og_text)\n",
    "        print()\n",
    "        print(ans_)\n",
    "        print(\"-------\")\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "611da11a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['pöördus kontrolli. DATE vasaku rinna sektorresektsioon + snb, adjuvantravina rinna kiiritus + zoladex + tamoxifen. DATE konsiiliumi otsusega herceptin-ravi (tmx). DATE.14 mmgr, uh ja jnb. DATE vasaku rinna rekonstruktsioon tram lapiga. DATE konsiiliumi otsusega näidustatud adjuvantne keemia-, bioloogiline-, hormoontaastusravi. kuna kasvaja hormoonsõltuv, plaanis alustada ravi zoladex + tamoxifeniga. obj: rindades tihendeid ei palpeeri. NAME. palp. bilat.ii. tellitud ca15-3']"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "successful_texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "32ec774d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "   \"DRUG\": [\"zoladex\", \"tamoxifen\", \"herceptin-ravi\"],\n",
      "   \"PROCEDURE\": [\"sektorresektsioon\", \"snb\", \"kiiritus\", \"rinna rekonstruktsioon\", \"tram lapiga\", \"adjuvantne keemia-, bioloogiline-, hormoontaastusravi\"],\n",
      "   \"FAMILY\": []\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(responses[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "e9ec3674",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parsed_answers[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "878ef543",
   "metadata": {},
   "source": [
    "## export to csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "0f6887b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "results_for_csv = []\n",
    "\n",
    "print(len(parsed_answers))\n",
    "print(len(successful_texts))\n",
    "\n",
    "for text_, results_ in zip(successful_texts, parsed_answers):\n",
    "    train_format_ = parsed_results_to_train(results_, text_, entities)\n",
    "    results_for_csv.append(train_format_)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}