faithfulness_calculation_lime_old.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Implementation of Comprehensiveness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluating Faithfulness on our model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/h/u11/c0/00/ammcourt/miniconda3/envs/csc413/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import argparse\n",
    "from pprint import pprint\n",
    "\n",
    "import evaluate\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from torch import nn\n",
    "from datasets import load_dataset\n",
    "from peft import LoraConfig, TaskType, get_peft_model\n",
    "from transformers import (\n",
    "    AutoConfig,\n",
    "    AutoTokenizer,\n",
    "    AutoModelForSequenceClassification,\n",
    "    DataCollatorWithPadding,\n",
    "    EvalPrediction,\n",
    "    OPTForSequenceClassification,\n",
    "    Trainer,\n",
    "    TrainingArguments,\n",
    ")\n",
    "import wandb\n",
    "\n",
    "MODEL = \"facebook/opt-350m\"\n",
    "MAX_POSITION_EMBEDDINGS = 2048\n",
    "\n",
    "from dataclasses import dataclass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "torch.backends.cuda.matmul.allow_tf32 = True\n",
    "torch.backends.cudnn.allow_tf32 = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "CHECKPOINT_DIR = \"OPT-350m-mimic-full\"\n",
    "VAL_DATASET_PATH = \"data/val_9.csv\"\n",
    "CODE_PATH = \"data/icd9_codes.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load dataset\n",
    "tokenizer = AutoTokenizer.from_pretrained(MODEL, use_fast=True, device=device)\n",
    "\n",
    "code_labels = pd.read_csv(\"data/icd9_codes.csv\")\n",
    "dataset = load_dataset(\"csv\", data_files=VAL_DATASET_PATH)\n",
    "\n",
    "# Create class dictionaries\n",
    "classes = [class_ for class_ in code_labels[\"icd_code\"] if class_]\n",
    "class2id = {class_: id for id, class_ in enumerate(classes)}\n",
    "id2class = {id: class_ for class_, id in class2id.items()}\n",
    "\n",
    "\n",
    "def multi_labels_to_ids(labels: list[str]) -> list[float]:\n",
    "    ids = [0.0] * len(class2id)  # BCELoss requires float as target type\n",
    "    for label in labels:\n",
    "        ids[class2id[label]] = 1.0\n",
    "    return ids\n",
    "\n",
    "\n",
    "def preprocess_function(example):\n",
    "    result = tokenizer(\n",
    "        example[\"text\"], truncation=True, max_length=MAX_POSITION_EMBEDDINGS\n",
    "    )\n",
    "    result[\"labels\"] = [multi_labels_to_ids(eval(label)) for label in example[\"labels\"]]\n",
    "    return result\n",
    "\n",
    "\n",
    "# dataset = dataset.map(\n",
    "#     preprocess_function, load_from_cache_file=True, batched=True, num_proc=8\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of OPTForSequenceClassification were not initialized from the model checkpoint at facebook/opt-350m and are newly initialized: ['score.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "config, unused_kwargs = AutoConfig.from_pretrained(\n",
    "    MODEL,\n",
    "    num_labels=len(classes),\n",
    "    id2label=id2class,\n",
    "    label2id=class2id,\n",
    "    problem_type=\"multi_label_classification\",\n",
    "    return_unused_kwargs=True,\n",
    ")\n",
    "\n",
    "if unused_kwargs:\n",
    "    print(f\"Unused kwargs: {unused_kwargs}\")\n",
    "\n",
    "model = OPTForSequenceClassification.from_pretrained(\n",
    "    MODEL,\n",
    "    config=config,\n",
    ").to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_adapter(CHECKPOINT_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "untokenized_dataset = load_dataset(\"csv\", data_files=VAL_DATASET_PATH)\n",
    "\n",
    "print(untokenized_dataset['train'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = tokenizer(untokenized_dataset[\"train\"][0]['text'], return_tensors=\"pt\", truncation=True, max_length=MAX_POSITION_EMBEDDINGS).to(device)\n",
    "\n",
    "with torch.no_grad():\n",
    "    logits = model(**inputs).logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logits = logits.to('cpu')\n",
    "\n",
    "predicted_class_ids = torch.arange(0, logits.shape[-1])[\n",
    "    torch.sigmoid(logits).squeeze(dim=0) > 0.5\n",
    "]\n",
    "\n",
    "# Get the predicted class names\n",
    "for id in predicted_class_ids:\n",
    "    predicted_class = id2class[int(id)]\n",
    "    pprint(code_labels[code_labels.icd_code == predicted_class])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, logit in enumerate(logits[0]):\n",
    "    pprint(f'{classes[i]}, {logit}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lime\n",
    "from lime import lime_text\n",
    "from lime.lime_text import LimeTextExplainer\n",
    "from lime.lime_text import IndexedString\n",
    "import numpy as np\n",
    "import torch.nn.functional as F\n",
    "from time import time\n",
    "\n",
    "\n",
    "explainer = LimeTextExplainer(class_names=classes, bow=False)\n",
    "\n",
    "def predictor_opt(texts):\n",
    "    tk = tokenizer(texts, return_tensors=\"pt\",truncation=True, max_length=MAX_POSITION_EMBEDDINGS).to(device)\n",
    "    outputs = model(**tk)\n",
    "    tensor_logits = outputs[0]\n",
    "    probas = F.sigmoid(tensor_logits).detach().cpu().numpy()\n",
    "    return probas\n",
    "\n",
    "# used by the faithfulness function\n",
    "def predictor_model(texts, model, tokenizer):\n",
    "    tk = tokenizer(texts, return_tensors=\"pt\",truncation=True, max_length=MAX_POSITION_EMBEDDINGS).to(device)\n",
    "    outputs = model(**tk)\n",
    "    tensor_logits = outputs[0]\n",
    "    probas = F.sigmoid(tensor_logits).detach().cpu().numpy()\n",
    "    return probas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence = untokenized_dataset[\"train\"][2][\"text\"]\n",
    "n_samples = 10\n",
    "k = 5\n",
    "print(len(sentence))\n",
    "print(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'torch' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241m.\u001b[39mcuda\u001b[38;5;241m.\u001b[39mempty_cache()\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mno_grad():\n\u001b[1;32m      3\u001b[0m     exp_bert \u001b[38;5;241m=\u001b[39m explainer\u001b[38;5;241m.\u001b[39mexplain_instance(\n\u001b[1;32m      4\u001b[0m         sentence, predictor_opt, num_samples\u001b[38;5;241m=\u001b[39mn_samples, top_labels\u001b[38;5;241m=\u001b[39mk\n\u001b[1;32m      5\u001b[0m     )\n",
      "\u001b[0;31mNameError\u001b[0m: name 'torch' is not defined"
     ]
    }
   ],
   "source": [
    "torch.cuda.empty_cache()\n",
    "with torch.no_grad():\n",
    "    exp_bert = explainer.explain_instance(\n",
    "        sentence, predictor_opt, num_samples=n_samples, top_labels=k\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp_bert.show_in_notebook(text=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"['d-42731', 'd-V5861', 'd-42789', 'd-25000', 'd-4019', 'd-2724', 'd-53081']\""
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "untokenized_dataset[\"train\"][2]['labels']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lime.explanation\n",
    "import lime.lime_text\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "def lime_create_index_arrays(instances, pred_fn, explainer, n_samples=10, k_labels=5):\n",
    "    \"\"\"get the explanation for the given instances and generate index arrays for the rationale\"\"\"\n",
    "    indexed_strs = np.array([])\n",
    "    # get the index of the longest instance\n",
    "    longest_instance = max(instances, key=len)\n",
    "    longest_instance = lime.lime_text.IndexedString(longest_instance)\n",
    "    padding_len = len(longest_instance.as_np)\n",
    "    \n",
    "    \n",
    "    index_array = None\n",
    "    for i, instance in enumerate(instances):\n",
    "        indexed_str = lime.lime_text.IndexedString(instance)\n",
    "        torch.cuda.empty_cache()\n",
    "        with torch.no_grad():\n",
    "            exp = explainer.explain_instance(instances[0], pred_fn, num_samples=n_samples, top_labels=k_labels)\n",
    "        \n",
    "        \n",
    "        # create masked array from map\n",
    "        exp_map = exp.as_map()\n",
    "        # print(exp_map)\n",
    "        for label in exp_map.keys():\n",
    "            for item in exp_map[label]:\n",
    "                if index_array is None:\n",
    "                    index_array = np.array([[i, item[0]]])\n",
    "                else:\n",
    "                    # append to the index array so that np.take can be used to mask the data\n",
    "                    index_array = np.append(index_array, [[i, item[0]]], axis=0)\n",
    "                    #print(index_array)\n",
    "        \n",
    "        # pad and save\n",
    "        str_as_np = indexed_str.as_np\n",
    "        padding = np.full((padding_len - len(str_as_np)), [''], dtype=str)\n",
    "        str_as_np = np.append(str_as_np, padding)\n",
    "        \n",
    "        if indexed_strs.size == 0:\n",
    "            # pad indexed_str\n",
    "            indexed_strs = np.array([str_as_np])\n",
    "        else:\n",
    "            indexed_strs = np.append(indexed_strs, [str_as_np], axis=0)\n",
    "        \n",
    "    index_array_x = np.transpose(index_array)[0]\n",
    "    index_array_y = np.transpose(index_array)[1]\n",
    "    index_array = np.array([index_array_x, index_array_y])\n",
    "    \n",
    "    return indexed_strs, index_array\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The instances are formatted as a list of strings, where each string is one word used by lime. The rationales mask is a list of indices, where the first list refers to the index of the sample the label corresponds to and the second list is the index of string used in that rationale."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# preprocessing code for comprehensiveness test\n",
    "# generates the versions of the instances with rationale words removed, and the versions with all non rationale words removed\n",
    "import copy\n",
    "import numpy as np\n",
    "\n",
    "def remove_rationale_words(instances, rationales):\n",
    "    inst_rationale_removed = copy.deepcopy(instances)\n",
    "    \n",
    "    rationales_mask = np.zeros(instances.shape, dtype=bool)\n",
    "    \n",
    "    # set the values of the rational mask to true based on rationales in a vectorized manner\n",
    "    # the rationales are in the format [[instance_index_1, instance_index_2, ...], [word_index_1, word_index_2, ...]]\n",
    "    rationales_mask[rationales[0], rationales[1]] = True\n",
    "    \n",
    "    print(rationales_mask)\n",
    "    \n",
    "    # remove the rationale words from the instance in a vectorized manner. The rationale words are a mask, w\n",
    "    # do this for every instance at the same time using numpy, this is faster than looping through each instance. do not use a list comprehension here\n",
    "    inst_rationale_removed = np.where(rationales_mask, \" \", instances)\n",
    "    return inst_rationale_removed\n",
    "    \n",
    "def remove_other_words(instances, rationales):\n",
    "    inst_other_removed = copy.deepcopy(instances)\n",
    "    \n",
    "    # create version of index array where all indexes are added that are not in the rationalle\n",
    "    inverse_rationales_mask = np.ones(instances.shape, dtype=bool)\n",
    "    inverse_rationales_mask[rationales[0], rationales[1]] = False\n",
    "    \n",
    "    # remove the rationale words from the instance in a vectorized manner\n",
    "    # do this for every instance at the same time using numpy, this is faster than looping through each instance. do not use a list comprehension here\n",
    "    # replace each word with \"\" so that the length of the instance stays the same\n",
    "    inst_other_removed = np.where(inverse_rationales_mask, \" \", instances)\n",
    "    return inst_other_removed\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def calculate_comprehensiveness(predictions, instances_rationale_removed, model, tokenizer):\n",
    "    \"\"\" Calculate the comprehensiveness of the rationales\n",
    "\n",
    "    Args:\n",
    "        predictions (np.array(np.array(float))): List of predictions made with the base instances (no words removed) using the given model.\n",
    "        instances_rationale_removed (np.array(np.array(word))): List of rationales to compute the comprehensiveness for. This is formatted as a list of numpy arrays, where each array is an array of words.\n",
    "        model (model): The model to compute the comprehensiveness for.\n",
    "    \"\"\"\n",
    "    print(\"Calculating Comprehensiveness\")\n",
    "    \n",
    "    # pass the instances through the model - get the predictions\n",
    "    torch.cuda.empty_cache()\n",
    "    predictions_rationale_removed = predictor_model(instances_rationale_removed, model, tokenizer)\n",
    "    print(\"Predictions ratonale removed: \", predictions_rationale_removed)\n",
    "    \n",
    "    # calculate the euclidean distance between the probability of the predicted class and sum over multi labels\n",
    "    # logits are the classification scores for the opt model\n",
    "    # confidence_dif = predictions.logits - predictions_rationale_removed.logits\n",
    "    confidence_dif = predictions - predictions_rationale_removed\n",
    "    print(\"Confidence Dif: \", confidence_dif)\n",
    "    confidence_dif = np.linalg.norm(confidence_dif, axis=-1)\n",
    "    print(\"Confidence Dif - eudclidean distance: \", confidence_dif)\n",
    "    \n",
    "    # return the average confidence difference over the samples\n",
    "    return np.mean(confidence_dif, axis=-1)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Implementation of Sufficency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_sufficency(predictions, instances_other_removed, model, tokenizer):\n",
    "    \"\"\"Calculates the sufficiency of the rationales\n",
    "\n",
    "    Args:\n",
    "        predictions (np.array(np.array(float))): List of predictions made with the base instances (no words removed) using the given model.\n",
    "        instances_rationale_removed (np.array(np.array(indices))): List of rationales to compute the sufficency for. This is formatted as a list of numpy arrays, where each array acts as a mask, where a 1 indicates that the word is a rationale word.\n",
    "        model (model): The model to compute the sufficency for.\n",
    "    \"\"\"\n",
    "    print(\"Calculating Sufficiency\")\n",
    "    torch.cuda.empty_cache()\n",
    "    predictions_other_removed = predictor_model(instances_other_removed, model, tokenizer)\n",
    "    print(\"Predicitons other removed: \", predictions_other_removed)\n",
    "    \n",
    "    # calculate the euclidean distance between the predictions and the predictions with the other words removed\n",
    "    # logits are the classification scores\n",
    "    # confidence_dif = predictions.logits - predictions_other_removed.logits\n",
    "    confidence_dif = predictions - predictions_other_removed\n",
    "    print(\"Confidence Dif: \", confidence_dif)\n",
    "    confidence_dif = np.linalg.norm(confidence_dif, axis=-1)\n",
    "    print(\"Confidence Dif - eudclidean distance: \", confidence_dif)\n",
    "    \n",
    "    # return the average confidence difference\n",
    "    return np.mean(confidence_dif, axis=-1)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Implementation of Faithfullness "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_faithfulness(instances, instances_rationalle_removed, instances_other_removed, model, tokenizer):\n",
    "    \"\"\"Calculate the faithfulness of the rationales\n",
    "\n",
    "    Args:\n",
    "        instances (numpy(numpy(string))): List of instances to compute the faithfulness for. This is formatted as a list of numpy arrays of words.\n",
    "        instances_rationalle_removed (numpy(numpy(numpy(int)))): List of rationales to compute the faithfulness for. This is formatted as a list of numpy arrays, where each array acts as a mask, where a 1 indicates that the word is a rationale word. Each list is provided by one interpretability method.\n",
    "        instances_other_removed (numpy(numpy(int))): List of instances with all non rationale words removed to compute the faithfulness for. This is formatted as a list of numpy arrays, where each array acts as a mask, where a 1 indicates that the word is not a rationale word. Each list is provided by one interpretability method.\n",
    "        model (model): The model to compute the faithfulness for.\n",
    "    \"\"\"\n",
    "    # generate predictions\n",
    "    predictions = predictor_model(instances, model, tokenizer)\n",
    "    faithfulness_calc = []\n",
    "    \n",
    "    # for each method, calculate the sufficency and comprehensiveness\n",
    "    for i, instance in enumerate(instances_rationalle_removed):\n",
    "        print(\"Currently interpreting instance: \", i)\n",
    "        \n",
    "        print(instances_rationalle_removed[i])\n",
    "        sufficency = calculate_sufficency(predictions, instances_rationalle_removed[i], model, tokenizer)\n",
    "        print(\"Sufficency for iteration: \", sufficency)\n",
    "        \n",
    "        comprehensiveness = calculate_comprehensiveness(predictions, instances_other_removed[i], model, tokenizer)\n",
    "        print(\"Comprehensiveness for iteration: \", comprehensiveness)\n",
    "        \n",
    "        # calculate faithfulness\n",
    "        faithfulness = sufficency * comprehensiveness\n",
    "        print(\"Faithfulness for iteration: \", faithfulness)\n",
    "        faithfulness_calc.append(faithfulness)\n",
    "    \n",
    "    # return the minimum index of the faithfulness_calc to get the best method\n",
    "    return np.argmin(faithfulness_calc), faithfulness_calc\n",
    "        \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test code for faithfulness calculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a test with 10 instances for faithfulness evaluation\n",
    "# from transformers import AutoTokenizer\n",
    "\n",
    "# get the lime evaluations of each instance\n",
    "from faithfulness import calculate_faithfulness, remove_other_words, remove_rationale_words\n",
    "\n",
    "samples_start = 0\n",
    "samples_end = 2\n",
    "\n",
    "instances = untokenized_dataset[\"train\"][samples_start:samples_end][\"text\"]\n",
    "print(len(instances))\n",
    "\n",
    "print(instances)\n",
    "explainer = LimeTextExplainer(class_names=classes, bow=False)\n",
    "\n",
    "indexed_text, index_array_rationalle = lime_create_index_arrays(instances, predictor_opt, explainer)\n",
    "print(indexed_text)\n",
    "print(index_array_rationalle)\n",
    "\n",
    "# # remove the rationale words\n",
    "rationalle_removed = remove_rationale_words(indexed_text, index_array_rationalle)\n",
    "others_removed = remove_other_words(indexed_text, index_array_rationalle)\n",
    "\n",
    "print(rationalle_removed)\n",
    "print(others_removed)\n",
    "\n",
    "# # concatenate the rationalle_removed and others_removed to tokenize them\n",
    "rationalle_removed = [''.join(rationalle_removed[i].tolist()) for i in range(len(rationalle_removed))]\n",
    "others_removed = [''.join(others_removed[i].tolist()) for i in range(len(others_removed))]\n",
    "\n",
    "print(rationalle_removed)\n",
    "\n",
    "print(len(rationalle_removed))\n",
    "print(len(others_removed))\n",
    "\n",
    "# # the extra list is needed since the function expects a list of instances each coming from a different interpretability method\n",
    "ind, faith = calculate_faithfulness(instances, [rationalle_removed], [others_removed], model, tokenizer=tokenizer, predictor_model)\n",
    "print(ind)\n",
    "print(faith)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}