analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2c26a5a1-c589-41af-a777-973ac92179a4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/local/QCRI/ejeong/miniconda3/envs/vllm/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "2024-05-26 10:21:36,234\tINFO util.py:154 -- Missing packages: ['ipywidgets']. Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING 05-26 10:21:36 config.py:1086] Casting torch.bfloat16 to torch.float16.\n",
      "INFO 05-26 10:21:36 llm_engine.py:100] Initializing an LLM engine (v0.4.2) with config: model='../text-generation-webui/models/Qwen_CodeQwen1.5-7B-Chat/', speculative_config=None, tokenizer='../text-generation-webui/models/Qwen_CodeQwen1.5-7B-Chat/', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.float16, max_seq_len=8192, download_dir=None, load_format=LoadFormat.AUTO, tensor_parallel_size=1, disable_custom_all_reduce=False, quantization=None, enforce_eager=False, kv_cache_dtype=auto, quantization_param_path=None, device_config=cuda, decoding_config=DecodingConfig(guided_decoding_backend='outlines'), seed=0, served_model_name=../text-generation-webui/models/Qwen_CodeQwen1.5-7B-Chat/)\n",
      "INFO 05-26 10:21:37 utils.py:660] Found nccl from library /home/local/QCRI/ejeong/.config/vllm/nccl/cu12/libnccl.so.2.18.1\n",
      "INFO 05-26 10:21:38 selector.py:69] Cannot use FlashAttention-2 backend for Volta and Turing GPUs.\n",
      "INFO 05-26 10:21:38 selector.py:32] Using XFormers backend.\n",
      "INFO 05-26 10:21:46 model_runner.py:175] Loading model weights took 13.5516 GB\n",
      "INFO 05-26 10:21:48 gpu_executor.py:114] # GPU blocks: 13461, # CPU blocks: 4096\n",
      "INFO 05-26 10:21:51 model_runner.py:937] Capturing the model for CUDA graphs. This may lead to unexpected consequences if the model is not static. To run the model in eager mode, set 'enforce_eager=True' or use '--enforce-eager' in the CLI.\n",
      "INFO 05-26 10:21:51 model_runner.py:941] CUDA graphs can take additional 1~3 GiB memory per GPU. If you are running out of memory, consider decreasing `gpu_memory_utilization` or enforcing eager mode. You can also reduce the `max_num_seqs` as needed to decrease memory usage.\n",
      "INFO 05-26 10:22:00 model_runner.py:1017] Graph capturing finished in 9 secs.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processed prompts: 100%|██████████| 2/2 [00:06<00:00,  3.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "My name is Qwen. I'm a large language model created by Alibaba Cloud. I'm here to help you with any questions or tasks you have!\n",
      "--------------------------------------------------\n",
      "Sure, I'd be happy to explain how binary search works!\n",
      "\n",
      "Binary search is an algorithm that works by repeatedly dividing in half the portion of the array that could contain the item, until you've narrowed the possible locations to just one.\n",
      "\n",
      "Here's how it works:\n",
      "\n",
      "1. Start by comparing the middle item of the array to the target item.\n",
      "2. If the middle item is the target, then you've found the item, so return its index.\n",
      "3. If the target item is less than the middle item, then the target item must be in the left half of the array. So, narrow down the possible locations to just the left half of the array and repeat steps 1 and 2.\n",
      "4. If the target item is greater than the middle item, then the target item must be in the right half of the array. So, narrow down the possible locations to just the right half of the array and repeat steps 1 and 2.\n",
      "5. Continue narrowing down the possible locations until you've found the target item or until you've eliminated all possible locations.\n",
      "\n",
      "Binary search has a time complexity of O(log n), which means that it can perform in logarithmic time, making it much faster than linear search, which has a time complexity of O(n).\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from utils import *\n",
    "import os\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = '4'\n",
    "# res = inferSystemModelVllm('meta-llama_Meta-Llama-3-8B-Instruct/', ['introduce yourself', 'can you tell me how binary search works?'], \"you're mario from the super mario brothers\", mode='llama3')\n",
    "# res = inferSystemModelVllm('Qwen_CodeQwen1.5-7B-Chat/', ['introduce yourself', 'can you tell me how binary search works?'], \"you're mario from the super mario brothers\", mode='codeqwen')\n",
    "# res = inferSystemModelVllm('deepseek-ai_deepseek-coder-6.7b-instruct/', ['introduce yourself', 'can you tell me how binary search works?'], \"you're mario from the super mario brothers\", mode='deepseekcoder')\n",
    "# res = inferSystemModelVllm('Artigenz_Artigenz-Coder-DS-6.7B/', ['introduce yourself', 'can you tell me how binary search works?'], \"you're mario from the super mario brothers\", mode='artigenz')\n",
    "print(res[0])\n",
    "print('-'*50)\n",
    "print(res[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06cf21dd-f587-40ed-9c58-5d33b914d725",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1a0ebf1-989c-44f4-b3b1-b8e6859edd0a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b125fd5-8959-4cd3-b6c6-7907aada0729",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db6f577e-ac23-429b-bb4e-d3e6c3ba88f4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b8207cf-a4a2-450f-a314-82680d7edfcd",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5870db12-5cb5-4af2-9131-c1fa843edb7e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "556773ad-6c16-44d6-a114-d03e43e4715b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('cwe25.json', 'r') as f:\n",
    "    data = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8b7b13d3-4ff3-4055-bb64-2eacbd0f5a6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_cwe(possible_options, cwe25):\n",
    "    for cwe in cwe25:\n",
    "        for possible_id in possible_options:\n",
    "            possible_id = possible_id[4:]\n",
    "\n",
    "            if possible_id == cwe['id']:\n",
    "                return cwe['id'], cwe['name'], cwe['description']\n",
    "\n",
    "    return '', '', ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c12dcfe5-e35c-43a6-8bf2-bca7af79ed9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_cwe_id(possible_options, cwe25):\n",
    "    for cwe in cwe25:\n",
    "        for possible_id in possible_options:\n",
    "            possible_id = possible_id[4:]\n",
    "\n",
    "            if possible_id == cwe['id']:\n",
    "                return f\"CWE-{cwe['id']}\"\n",
    "\n",
    "    return ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1e06ed22-425f-4969-b763-12402197689f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('269',\n",
       " 'Improper Privilege Management',\n",
       " 'The product does not properly assign, modify, track, or check privileges for an actor, creating an unintended sphere of control for that actor.')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extract_cwe(ast.literal_eval(df.cwe_id.iloc[4]), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0ec19770-3193-4469-aa10-62f8254a77ed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2621, 14)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('vul7.csv')\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "081e602d-5ca1-457d-bbb4-d13a5e505346",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>method_name</th>\n",
       "      <th>code_before</th>\n",
       "      <th>code_after</th>\n",
       "      <th>func_before</th>\n",
       "      <th>func_after</th>\n",
       "      <th>diff</th>\n",
       "      <th>num_lines_added</th>\n",
       "      <th>num_lines_deleted</th>\n",
       "      <th>num_lines_in_file</th>\n",
       "      <th>num_tokens_in_file</th>\n",
       "      <th>repo</th>\n",
       "      <th>cve_id</th>\n",
       "      <th>cwe_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>open_utils.c</td>\n",
       "      <td>read_new_config_info</td>\n",
       "      <td>//////////////////////////////////////////////...</td>\n",
       "      <td>//////////////////////////////////////////////...</td>\n",
       "      <td>static int read_new_config_info (WavpackContex...</td>\n",
       "      <td>static int read_new_config_info (WavpackContex...</td>\n",
       "      <td>{'added': [(563, '    if (bytecnt &gt;= 2) {'), (...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>825.0</td>\n",
       "      <td>6305.0</td>\n",
       "      <td>https://github.com/dbry/WavPack</td>\n",
       "      <td>CVE-2016-10169</td>\n",
       "      <td>['CWE-125']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>route.c</td>\n",
       "      <td>inet_rtm_getroute</td>\n",
       "      <td>/*\\n * INET\\t\\tAn implementation of the TCP/IP...</td>\n",
       "      <td>/*\\n * INET\\t\\tAn implementation of the TCP/IP...</td>\n",
       "      <td>static int inet_rtm_getroute(struct sk_buff *i...</td>\n",
       "      <td>static int inet_rtm_getroute(struct sk_buff *i...</td>\n",
       "      <td>{'added': [(2766, '\\tif (rtm-&gt;rtm_flags &amp; RTM_...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2327.0</td>\n",
       "      <td>14667.0</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2017-13686</td>\n",
       "      <td>['CWE-476']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>main.c</td>\n",
       "      <td>mp4boxMain</td>\n",
       "      <td>/*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...</td>\n",
       "      <td>/*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...</td>\n",
       "      <td>int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...</td>\n",
       "      <td>int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...</td>\n",
       "      <td>{'added': [(5598, '\\t\\t\\tfprintf(stderr, \"File...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5787.0</td>\n",
       "      <td>43960.0</td>\n",
       "      <td>https://github.com/gpac/gpac</td>\n",
       "      <td>CVE-2020-23930</td>\n",
       "      <td>['CWE-476']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>udp.c</td>\n",
       "      <td>udpv6_sendmsg</td>\n",
       "      <td>/*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...</td>\n",
       "      <td>/*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...</td>\n",
       "      <td>int udpv6_sendmsg(struct sock *sk, struct msgh...</td>\n",
       "      <td>int udpv6_sendmsg(struct sock *sk, struct msgh...</td>\n",
       "      <td>{'added': [(1113, '\\tstruct ipv6_txoptions *op...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1221.0</td>\n",
       "      <td>7876.0</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2016-3841</td>\n",
       "      <td>['CWE-264', 'CWE-416']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>api.c</td>\n",
       "      <td>crypto_larval_lookup</td>\n",
       "      <td>/*\\n * Scatterlist Cryptographic API.\\n *\\n * ...</td>\n",
       "      <td>/*\\n * Scatterlist Cryptographic API.\\n *\\n * ...</td>\n",
       "      <td>struct crypto_alg *crypto_larval_lookup(const ...</td>\n",
       "      <td>struct crypto_alg *crypto_larval_lookup(const ...</td>\n",
       "      <td>{'added': [(219, '\\t\\trequest_module(\"crypto-%...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>424.0</td>\n",
       "      <td>2423.0</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2013-7421</td>\n",
       "      <td>['CWE-269']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      file_name           method_name  \\\n",
       "0  open_utils.c  read_new_config_info   \n",
       "1       route.c     inet_rtm_getroute   \n",
       "2        main.c            mp4boxMain   \n",
       "3         udp.c         udpv6_sendmsg   \n",
       "4         api.c  crypto_larval_lookup   \n",
       "\n",
       "                                         code_before  \\\n",
       "0  //////////////////////////////////////////////...   \n",
       "1  /*\\n * INET\\t\\tAn implementation of the TCP/IP...   \n",
       "2  /*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...   \n",
       "3  /*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...   \n",
       "4  /*\\n * Scatterlist Cryptographic API.\\n *\\n * ...   \n",
       "\n",
       "                                          code_after  \\\n",
       "0  //////////////////////////////////////////////...   \n",
       "1  /*\\n * INET\\t\\tAn implementation of the TCP/IP...   \n",
       "2  /*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...   \n",
       "3  /*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...   \n",
       "4  /*\\n * Scatterlist Cryptographic API.\\n *\\n * ...   \n",
       "\n",
       "                                         func_before  \\\n",
       "0  static int read_new_config_info (WavpackContex...   \n",
       "1  static int inet_rtm_getroute(struct sk_buff *i...   \n",
       "2  int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...   \n",
       "3  int udpv6_sendmsg(struct sock *sk, struct msgh...   \n",
       "4  struct crypto_alg *crypto_larval_lookup(const ...   \n",
       "\n",
       "                                          func_after  \\\n",
       "0  static int read_new_config_info (WavpackContex...   \n",
       "1  static int inet_rtm_getroute(struct sk_buff *i...   \n",
       "2  int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...   \n",
       "3  int udpv6_sendmsg(struct sock *sk, struct msgh...   \n",
       "4  struct crypto_alg *crypto_larval_lookup(const ...   \n",
       "\n",
       "                                                diff  num_lines_added  \\\n",
       "0  {'added': [(563, '    if (bytecnt >= 2) {'), (...              5.0   \n",
       "1  {'added': [(2766, '\\tif (rtm->rtm_flags & RTM_...              9.0   \n",
       "2  {'added': [(5598, '\\t\\t\\tfprintf(stderr, \"File...              1.0   \n",
       "3  {'added': [(1113, '\\tstruct ipv6_txoptions *op...              6.0   \n",
       "4  {'added': [(219, '\\t\\trequest_module(\"crypto-%...              2.0   \n",
       "\n",
       "   num_lines_deleted  num_lines_in_file  num_tokens_in_file  \\\n",
       "0                1.0              825.0              6305.0   \n",
       "1                2.0             2327.0             14667.0   \n",
       "2                1.0             5787.0             43960.0   \n",
       "3                2.0             1221.0              7876.0   \n",
       "4                2.0              424.0              2423.0   \n",
       "\n",
       "                                repo          cve_id                  cwe_id  \n",
       "0    https://github.com/dbry/WavPack  CVE-2016-10169             ['CWE-125']  \n",
       "1  https://github.com/torvalds/linux  CVE-2017-13686             ['CWE-476']  \n",
       "2       https://github.com/gpac/gpac  CVE-2020-23930             ['CWE-476']  \n",
       "3  https://github.com/torvalds/linux   CVE-2016-3841  ['CWE-264', 'CWE-416']  \n",
       "4  https://github.com/torvalds/linux   CVE-2013-7421             ['CWE-269']  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "43b08fe3-2eb4-4d71-a04c-5f8eab4a4921",
   "metadata": {},
   "outputs": [],
   "source": [
    "import ast\n",
    "df.cwe_id = df.cwe_id.apply(ast.literal_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e45928fe-b0f2-43c9-8932-73fd8db11e89",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['exp_cwe_id'] = df.cwe_id.apply(lambda x: extract_cwe_id(x, data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "68d6d55a-4b60-49fe-a070-93b9527de2e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "exp_cwe_id\n",
       "CWE-125    558\n",
       "CWE-119    480\n",
       "CWE-787    303\n",
       "CWE-20     284\n",
       "CWE-476    260\n",
       "CWE-416    238\n",
       "CWE-190    191\n",
       "CWE-362    149\n",
       "CWE-78      30\n",
       "CWE-269     28\n",
       "CWE-287     22\n",
       "CWE-276     21\n",
       "CWE-22      14\n",
       "CWE-863     12\n",
       "CWE-79       8\n",
       "CWE-352      6\n",
       "CWE-77       6\n",
       "CWE-89       5\n",
       "CWE-862      4\n",
       "CWE-94       1\n",
       "CWE-434      1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.exp_cwe_id.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5f16c30b-30aa-40ae-8d5b-a5b2b9d388c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_and_select_rows(df):\n",
    "    allowed_cwe_ids = ['CWE-125', 'CWE-119', 'CWE-787', 'CWE-20', 'CWE-476', 'CWE-416', 'CWE-190', 'CWE-362']\n",
    "    \n",
    "    df = df[df['exp_cwe_id'].isin(allowed_cwe_ids)]\n",
    "    df.reset_index(drop=True, inplace=True)\n",
    "    \n",
    "    result_df = pd.DataFrame()\n",
    "    for cwe_id in allowed_cwe_ids:\n",
    "        subset = df[df['exp_cwe_id'] == cwe_id]\n",
    "        subset_sorted = subset.nsmallest(125, 'num_tokens_in_file')\n",
    "        result_df = pd.concat([result_df, subset_sorted])\n",
    "    \n",
    "    result_df.reset_index(drop=True, inplace=True)\n",
    "    return result_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e1408eaf-75ed-41bb-a5db-487e06fc2795",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df = filter_and_select_rows(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8b0a8097-1ad8-4f21-8f12-2015e64b8e5b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000, 15)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1981d729-6cf3-442e-8bc1-fe410ae856d6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>method_name</th>\n",
       "      <th>code_before</th>\n",
       "      <th>code_after</th>\n",
       "      <th>func_before</th>\n",
       "      <th>func_after</th>\n",
       "      <th>diff</th>\n",
       "      <th>num_lines_added</th>\n",
       "      <th>num_lines_deleted</th>\n",
       "      <th>num_lines_in_file</th>\n",
       "      <th>num_tokens_in_file</th>\n",
       "      <th>repo</th>\n",
       "      <th>cve_id</th>\n",
       "      <th>cwe_id</th>\n",
       "      <th>exp_cwe_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>snmp-api.c</td>\n",
       "      <td>snmp_api_set_oid</td>\n",
       "      <td>/*\\n * Copyright (C) 2019 Yago Fontoura do Ros...</td>\n",
       "      <td>/*\\n * Copyright (C) 2019-2020 Yago Fontoura d...</td>\n",
       "      <td>snmp_api_set_oid(snmp_varbind_t *varbind, uint...</td>\n",
       "      <td>snmp_api_set_oid(snmp_varbind_t *varbind, snmp...</td>\n",
       "      <td>{'added': [(2, ' * Copyright (C) 2019-2020 Yag...</td>\n",
       "      <td>11.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>https://github.com/contiki-ng/contiki-ng</td>\n",
       "      <td>CVE-2020-12141</td>\n",
       "      <td>[CWE-125]</td>\n",
       "      <td>CWE-125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>snmp-api.c</td>\n",
       "      <td>snmp_api_set_string</td>\n",
       "      <td>/*\\n * Copyright (C) 2019 Yago Fontoura do Ros...</td>\n",
       "      <td>/*\\n * Copyright (C) 2019-2020 Yago Fontoura d...</td>\n",
       "      <td>snmp_api_set_string(snmp_varbind_t *varbind, u...</td>\n",
       "      <td>snmp_api_set_string(snmp_varbind_t *varbind, s...</td>\n",
       "      <td>{'added': [(2, ' * Copyright (C) 2019-2020 Yag...</td>\n",
       "      <td>11.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>https://github.com/contiki-ng/contiki-ng</td>\n",
       "      <td>CVE-2020-12141</td>\n",
       "      <td>[CWE-125]</td>\n",
       "      <td>CWE-125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>snmp-api.c</td>\n",
       "      <td>snmp_api_set_time_ticks</td>\n",
       "      <td>/*\\n * Copyright (C) 2019 Yago Fontoura do Ros...</td>\n",
       "      <td>/*\\n * Copyright (C) 2019-2020 Yago Fontoura d...</td>\n",
       "      <td>snmp_api_set_time_ticks(snmp_varbind_t *varbin...</td>\n",
       "      <td>snmp_api_set_time_ticks(snmp_varbind_t *varbin...</td>\n",
       "      <td>{'added': [(2, ' * Copyright (C) 2019-2020 Yag...</td>\n",
       "      <td>11.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>https://github.com/contiki-ng/contiki-ng</td>\n",
       "      <td>CVE-2020-12141</td>\n",
       "      <td>[CWE-125]</td>\n",
       "      <td>CWE-125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>heap_overflow.c</td>\n",
       "      <td>main</td>\n",
       "      <td>/**\\n * Test that the crafted TGA file doesn't...</td>\n",
       "      <td>/**\\n * Test that crafted TGA files don't trig...</td>\n",
       "      <td>int main()\\n{\\n    gdImagePtr im;\\n    char *b...</td>\n",
       "      <td>int main()\\n{\\n    check_file(\"heap_overflow_1...</td>\n",
       "      <td>{'added': [(2, \" * Test that crafted TGA files...</td>\n",
       "      <td>12.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>233.0</td>\n",
       "      <td>https://github.com/libgd/libgd</td>\n",
       "      <td>CVE-2016-6906</td>\n",
       "      <td>[CWE-125]</td>\n",
       "      <td>CWE-125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>grammar1.c</td>\n",
       "      <td>Ta3Grammar_FindDFA</td>\n",
       "      <td>\\n/* Grammar subroutines needed by parser */\\n...</td>\n",
       "      <td>\\n/* Grammar subroutines needed by parser */\\n...</td>\n",
       "      <td>Ta3Grammar_FindDFA(grammar *g, int type)\\n{\\n ...</td>\n",
       "      <td>Ta3Grammar_FindDFA(grammar *g, int type)\\n{\\n ...</td>\n",
       "      <td>{'added': [(28, '    abort();')], 'deleted': [...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>234.0</td>\n",
       "      <td>https://github.com/python/typed_ast</td>\n",
       "      <td>CVE-2019-19274</td>\n",
       "      <td>[CWE-125]</td>\n",
       "      <td>CWE-125</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         file_name              method_name  \\\n",
       "0       snmp-api.c         snmp_api_set_oid   \n",
       "1       snmp-api.c      snmp_api_set_string   \n",
       "2       snmp-api.c  snmp_api_set_time_ticks   \n",
       "3  heap_overflow.c                     main   \n",
       "4       grammar1.c       Ta3Grammar_FindDFA   \n",
       "\n",
       "                                         code_before  \\\n",
       "0  /*\\n * Copyright (C) 2019 Yago Fontoura do Ros...   \n",
       "1  /*\\n * Copyright (C) 2019 Yago Fontoura do Ros...   \n",
       "2  /*\\n * Copyright (C) 2019 Yago Fontoura do Ros...   \n",
       "3  /**\\n * Test that the crafted TGA file doesn't...   \n",
       "4  \\n/* Grammar subroutines needed by parser */\\n...   \n",
       "\n",
       "                                          code_after  \\\n",
       "0  /*\\n * Copyright (C) 2019-2020 Yago Fontoura d...   \n",
       "1  /*\\n * Copyright (C) 2019-2020 Yago Fontoura d...   \n",
       "2  /*\\n * Copyright (C) 2019-2020 Yago Fontoura d...   \n",
       "3  /**\\n * Test that crafted TGA files don't trig...   \n",
       "4  \\n/* Grammar subroutines needed by parser */\\n...   \n",
       "\n",
       "                                         func_before  \\\n",
       "0  snmp_api_set_oid(snmp_varbind_t *varbind, uint...   \n",
       "1  snmp_api_set_string(snmp_varbind_t *varbind, u...   \n",
       "2  snmp_api_set_time_ticks(snmp_varbind_t *varbin...   \n",
       "3  int main()\\n{\\n    gdImagePtr im;\\n    char *b...   \n",
       "4  Ta3Grammar_FindDFA(grammar *g, int type)\\n{\\n ...   \n",
       "\n",
       "                                          func_after  \\\n",
       "0  snmp_api_set_oid(snmp_varbind_t *varbind, snmp...   \n",
       "1  snmp_api_set_string(snmp_varbind_t *varbind, s...   \n",
       "2  snmp_api_set_time_ticks(snmp_varbind_t *varbin...   \n",
       "3  int main()\\n{\\n    check_file(\"heap_overflow_1...   \n",
       "4  Ta3Grammar_FindDFA(grammar *g, int type)\\n{\\n ...   \n",
       "\n",
       "                                                diff  num_lines_added  \\\n",
       "0  {'added': [(2, ' * Copyright (C) 2019-2020 Yag...             11.0   \n",
       "1  {'added': [(2, ' * Copyright (C) 2019-2020 Yag...             11.0   \n",
       "2  {'added': [(2, ' * Copyright (C) 2019-2020 Yag...             11.0   \n",
       "3  {'added': [(2, \" * Test that crafted TGA files...             12.0   \n",
       "4  {'added': [(28, '    abort();')], 'deleted': [...              1.0   \n",
       "\n",
       "   num_lines_deleted  num_lines_in_file  num_tokens_in_file  \\\n",
       "0               27.0               31.0               184.0   \n",
       "1               27.0               31.0               184.0   \n",
       "2               27.0               31.0               184.0   \n",
       "3                4.0               39.0               233.0   \n",
       "4                2.0               46.0               234.0   \n",
       "\n",
       "                                       repo          cve_id     cwe_id  \\\n",
       "0  https://github.com/contiki-ng/contiki-ng  CVE-2020-12141  [CWE-125]   \n",
       "1  https://github.com/contiki-ng/contiki-ng  CVE-2020-12141  [CWE-125]   \n",
       "2  https://github.com/contiki-ng/contiki-ng  CVE-2020-12141  [CWE-125]   \n",
       "3            https://github.com/libgd/libgd   CVE-2016-6906  [CWE-125]   \n",
       "4       https://github.com/python/typed_ast  CVE-2019-19274  [CWE-125]   \n",
       "\n",
       "  exp_cwe_id  \n",
       "0    CWE-125  \n",
       "1    CWE-125  \n",
       "2    CWE-125  \n",
       "3    CWE-125  \n",
       "4    CWE-125  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b7bd033d-286f-4474-bc5a-f8ea598c7aba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "exp_cwe_id\n",
       "CWE-125    125\n",
       "CWE-119    125\n",
       "CWE-787    125\n",
       "CWE-20     125\n",
       "CWE-476    125\n",
       "CWE-416    125\n",
       "CWE-190    125\n",
       "CWE-362    125\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df.exp_cwe_id.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "35a4bfa8-19a3-4d10-83b4-1f37be600dc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df.to_csv('test_vulC7.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c765acdf-c094-4091-b6ba-39a5f1d53dbd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>method_name</th>\n",
       "      <th>code_before</th>\n",
       "      <th>code_after</th>\n",
       "      <th>func_before</th>\n",
       "      <th>func_after</th>\n",
       "      <th>diff</th>\n",
       "      <th>num_lines_added</th>\n",
       "      <th>num_lines_deleted</th>\n",
       "      <th>num_lines_in_file</th>\n",
       "      <th>num_tokens_in_file</th>\n",
       "      <th>repo</th>\n",
       "      <th>cve_id</th>\n",
       "      <th>cwe_id</th>\n",
       "      <th>exp_cwe_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>open_utils.c</td>\n",
       "      <td>read_new_config_info</td>\n",
       "      <td>//////////////////////////////////////////////...</td>\n",
       "      <td>//////////////////////////////////////////////...</td>\n",
       "      <td>static int read_new_config_info (WavpackContex...</td>\n",
       "      <td>static int read_new_config_info (WavpackContex...</td>\n",
       "      <td>{'added': [(563, '    if (bytecnt &gt;= 2) {'), (...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>825.0</td>\n",
       "      <td>6305.0</td>\n",
       "      <td>https://github.com/dbry/WavPack</td>\n",
       "      <td>CVE-2016-10169</td>\n",
       "      <td>[CWE-125]</td>\n",
       "      <td>CWE-125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>route.c</td>\n",
       "      <td>inet_rtm_getroute</td>\n",
       "      <td>/*\\n * INET\\t\\tAn implementation of the TCP/IP...</td>\n",
       "      <td>/*\\n * INET\\t\\tAn implementation of the TCP/IP...</td>\n",
       "      <td>static int inet_rtm_getroute(struct sk_buff *i...</td>\n",
       "      <td>static int inet_rtm_getroute(struct sk_buff *i...</td>\n",
       "      <td>{'added': [(2766, '\\tif (rtm-&gt;rtm_flags &amp; RTM_...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2327.0</td>\n",
       "      <td>14667.0</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2017-13686</td>\n",
       "      <td>[CWE-476]</td>\n",
       "      <td>CWE-476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>main.c</td>\n",
       "      <td>mp4boxMain</td>\n",
       "      <td>/*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...</td>\n",
       "      <td>/*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...</td>\n",
       "      <td>int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...</td>\n",
       "      <td>int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...</td>\n",
       "      <td>{'added': [(5598, '\\t\\t\\tfprintf(stderr, \"File...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5787.0</td>\n",
       "      <td>43960.0</td>\n",
       "      <td>https://github.com/gpac/gpac</td>\n",
       "      <td>CVE-2020-23930</td>\n",
       "      <td>[CWE-476]</td>\n",
       "      <td>CWE-476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>udp.c</td>\n",
       "      <td>udpv6_sendmsg</td>\n",
       "      <td>/*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...</td>\n",
       "      <td>/*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...</td>\n",
       "      <td>int udpv6_sendmsg(struct sock *sk, struct msgh...</td>\n",
       "      <td>int udpv6_sendmsg(struct sock *sk, struct msgh...</td>\n",
       "      <td>{'added': [(1113, '\\tstruct ipv6_txoptions *op...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1221.0</td>\n",
       "      <td>7876.0</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2016-3841</td>\n",
       "      <td>[CWE-264, CWE-416]</td>\n",
       "      <td>CWE-416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>api.c</td>\n",
       "      <td>crypto_larval_lookup</td>\n",
       "      <td>/*\\n * Scatterlist Cryptographic API.\\n *\\n * ...</td>\n",
       "      <td>/*\\n * Scatterlist Cryptographic API.\\n *\\n * ...</td>\n",
       "      <td>struct crypto_alg *crypto_larval_lookup(const ...</td>\n",
       "      <td>struct crypto_alg *crypto_larval_lookup(const ...</td>\n",
       "      <td>{'added': [(219, '\\t\\trequest_module(\"crypto-%...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>424.0</td>\n",
       "      <td>2423.0</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2013-7421</td>\n",
       "      <td>[CWE-269]</td>\n",
       "      <td>CWE-269</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      file_name           method_name  \\\n",
       "0  open_utils.c  read_new_config_info   \n",
       "1       route.c     inet_rtm_getroute   \n",
       "2        main.c            mp4boxMain   \n",
       "3         udp.c         udpv6_sendmsg   \n",
       "4         api.c  crypto_larval_lookup   \n",
       "\n",
       "                                         code_before  \\\n",
       "0  //////////////////////////////////////////////...   \n",
       "1  /*\\n * INET\\t\\tAn implementation of the TCP/IP...   \n",
       "2  /*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...   \n",
       "3  /*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...   \n",
       "4  /*\\n * Scatterlist Cryptographic API.\\n *\\n * ...   \n",
       "\n",
       "                                          code_after  \\\n",
       "0  //////////////////////////////////////////////...   \n",
       "1  /*\\n * INET\\t\\tAn implementation of the TCP/IP...   \n",
       "2  /*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...   \n",
       "3  /*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...   \n",
       "4  /*\\n * Scatterlist Cryptographic API.\\n *\\n * ...   \n",
       "\n",
       "                                         func_before  \\\n",
       "0  static int read_new_config_info (WavpackContex...   \n",
       "1  static int inet_rtm_getroute(struct sk_buff *i...   \n",
       "2  int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...   \n",
       "3  int udpv6_sendmsg(struct sock *sk, struct msgh...   \n",
       "4  struct crypto_alg *crypto_larval_lookup(const ...   \n",
       "\n",
       "                                          func_after  \\\n",
       "0  static int read_new_config_info (WavpackContex...   \n",
       "1  static int inet_rtm_getroute(struct sk_buff *i...   \n",
       "2  int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...   \n",
       "3  int udpv6_sendmsg(struct sock *sk, struct msgh...   \n",
       "4  struct crypto_alg *crypto_larval_lookup(const ...   \n",
       "\n",
       "                                                diff  num_lines_added  \\\n",
       "0  {'added': [(563, '    if (bytecnt >= 2) {'), (...              5.0   \n",
       "1  {'added': [(2766, '\\tif (rtm->rtm_flags & RTM_...              9.0   \n",
       "2  {'added': [(5598, '\\t\\t\\tfprintf(stderr, \"File...              1.0   \n",
       "3  {'added': [(1113, '\\tstruct ipv6_txoptions *op...              6.0   \n",
       "4  {'added': [(219, '\\t\\trequest_module(\"crypto-%...              2.0   \n",
       "\n",
       "   num_lines_deleted  num_lines_in_file  num_tokens_in_file  \\\n",
       "0                1.0              825.0              6305.0   \n",
       "1                2.0             2327.0             14667.0   \n",
       "2                1.0             5787.0             43960.0   \n",
       "3                2.0             1221.0              7876.0   \n",
       "4                2.0              424.0              2423.0   \n",
       "\n",
       "                                repo          cve_id              cwe_id  \\\n",
       "0    https://github.com/dbry/WavPack  CVE-2016-10169           [CWE-125]   \n",
       "1  https://github.com/torvalds/linux  CVE-2017-13686           [CWE-476]   \n",
       "2       https://github.com/gpac/gpac  CVE-2020-23930           [CWE-476]   \n",
       "3  https://github.com/torvalds/linux   CVE-2016-3841  [CWE-264, CWE-416]   \n",
       "4  https://github.com/torvalds/linux   CVE-2013-7421           [CWE-269]   \n",
       "\n",
       "  exp_cwe_id  \n",
       "0    CWE-125  \n",
       "1    CWE-476  \n",
       "2    CWE-476  \n",
       "3    CWE-416  \n",
       "4    CWE-269  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "95369a5d-c711-4042-891a-5314983bc408",
   "metadata": {},
   "outputs": [],
   "source": [
    "allowed_cwe_ids = ['CWE-125', 'CWE-119', 'CWE-787', 'CWE-20', 'CWE-476', 'CWE-416', 'CWE-190', 'CWE-362']\n",
    "    \n",
    "df = df[df['exp_cwe_id'].isin(allowed_cwe_ids)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "8d6f9110-22de-4a66-bcbc-7f673fd00799",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d38fd4c9-0e0e-44c8-a3d6-eee0c8a163e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>method_name</th>\n",
       "      <th>code_before</th>\n",
       "      <th>code_after</th>\n",
       "      <th>func_before</th>\n",
       "      <th>func_after</th>\n",
       "      <th>diff</th>\n",
       "      <th>num_lines_added</th>\n",
       "      <th>num_lines_deleted</th>\n",
       "      <th>num_lines_in_file</th>\n",
       "      <th>num_tokens_in_file</th>\n",
       "      <th>repo</th>\n",
       "      <th>cve_id</th>\n",
       "      <th>cwe_id</th>\n",
       "      <th>exp_cwe_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>open_utils.c</td>\n",
       "      <td>read_new_config_info</td>\n",
       "      <td>//////////////////////////////////////////////...</td>\n",
       "      <td>//////////////////////////////////////////////...</td>\n",
       "      <td>static int read_new_config_info (WavpackContex...</td>\n",
       "      <td>static int read_new_config_info (WavpackContex...</td>\n",
       "      <td>{'added': [(563, '    if (bytecnt &gt;= 2) {'), (...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>825.0</td>\n",
       "      <td>6305.0</td>\n",
       "      <td>https://github.com/dbry/WavPack</td>\n",
       "      <td>CVE-2016-10169</td>\n",
       "      <td>[CWE-125]</td>\n",
       "      <td>CWE-125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>route.c</td>\n",
       "      <td>inet_rtm_getroute</td>\n",
       "      <td>/*\\n * INET\\t\\tAn implementation of the TCP/IP...</td>\n",
       "      <td>/*\\n * INET\\t\\tAn implementation of the TCP/IP...</td>\n",
       "      <td>static int inet_rtm_getroute(struct sk_buff *i...</td>\n",
       "      <td>static int inet_rtm_getroute(struct sk_buff *i...</td>\n",
       "      <td>{'added': [(2766, '\\tif (rtm-&gt;rtm_flags &amp; RTM_...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2327.0</td>\n",
       "      <td>14667.0</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2017-13686</td>\n",
       "      <td>[CWE-476]</td>\n",
       "      <td>CWE-476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>main.c</td>\n",
       "      <td>mp4boxMain</td>\n",
       "      <td>/*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...</td>\n",
       "      <td>/*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...</td>\n",
       "      <td>int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...</td>\n",
       "      <td>int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...</td>\n",
       "      <td>{'added': [(5598, '\\t\\t\\tfprintf(stderr, \"File...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5787.0</td>\n",
       "      <td>43960.0</td>\n",
       "      <td>https://github.com/gpac/gpac</td>\n",
       "      <td>CVE-2020-23930</td>\n",
       "      <td>[CWE-476]</td>\n",
       "      <td>CWE-476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>udp.c</td>\n",
       "      <td>udpv6_sendmsg</td>\n",
       "      <td>/*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...</td>\n",
       "      <td>/*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...</td>\n",
       "      <td>int udpv6_sendmsg(struct sock *sk, struct msgh...</td>\n",
       "      <td>int udpv6_sendmsg(struct sock *sk, struct msgh...</td>\n",
       "      <td>{'added': [(1113, '\\tstruct ipv6_txoptions *op...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1221.0</td>\n",
       "      <td>7876.0</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2016-3841</td>\n",
       "      <td>[CWE-264, CWE-416]</td>\n",
       "      <td>CWE-416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>mpeg4videodec.c</td>\n",
       "      <td>read_quant_matrix_ext</td>\n",
       "      <td>/*\\n * MPEG-4 decoder\\n * Copyright (c) 2000,2...</td>\n",
       "      <td>/*\\n * MPEG-4 decoder\\n * Copyright (c) 2000,2...</td>\n",
       "      <td>static void read_quant_matrix_ext(MpegEncConte...</td>\n",
       "      <td>static int read_quant_matrix_ext(MpegEncContex...</td>\n",
       "      <td>{'added': [(2870, 'static int read_quant_matri...</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2824.0</td>\n",
       "      <td>24222.0</td>\n",
       "      <td>https://github.com/FFmpeg/FFmpeg</td>\n",
       "      <td>CVE-2018-1999015</td>\n",
       "      <td>[CWE-125]</td>\n",
       "      <td>CWE-125</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         file_name            method_name  \\\n",
       "0     open_utils.c   read_new_config_info   \n",
       "1          route.c      inet_rtm_getroute   \n",
       "2           main.c             mp4boxMain   \n",
       "3            udp.c          udpv6_sendmsg   \n",
       "4  mpeg4videodec.c  read_quant_matrix_ext   \n",
       "\n",
       "                                         code_before  \\\n",
       "0  //////////////////////////////////////////////...   \n",
       "1  /*\\n * INET\\t\\tAn implementation of the TCP/IP...   \n",
       "2  /*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...   \n",
       "3  /*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...   \n",
       "4  /*\\n * MPEG-4 decoder\\n * Copyright (c) 2000,2...   \n",
       "\n",
       "                                          code_after  \\\n",
       "0  //////////////////////////////////////////////...   \n",
       "1  /*\\n * INET\\t\\tAn implementation of the TCP/IP...   \n",
       "2  /*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...   \n",
       "3  /*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...   \n",
       "4  /*\\n * MPEG-4 decoder\\n * Copyright (c) 2000,2...   \n",
       "\n",
       "                                         func_before  \\\n",
       "0  static int read_new_config_info (WavpackContex...   \n",
       "1  static int inet_rtm_getroute(struct sk_buff *i...   \n",
       "2  int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...   \n",
       "3  int udpv6_sendmsg(struct sock *sk, struct msgh...   \n",
       "4  static void read_quant_matrix_ext(MpegEncConte...   \n",
       "\n",
       "                                          func_after  \\\n",
       "0  static int read_new_config_info (WavpackContex...   \n",
       "1  static int inet_rtm_getroute(struct sk_buff *i...   \n",
       "2  int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...   \n",
       "3  int udpv6_sendmsg(struct sock *sk, struct msgh...   \n",
       "4  static int read_quant_matrix_ext(MpegEncContex...   \n",
       "\n",
       "                                                diff  num_lines_added  \\\n",
       "0  {'added': [(563, '    if (bytecnt >= 2) {'), (...              5.0   \n",
       "1  {'added': [(2766, '\\tif (rtm->rtm_flags & RTM_...              9.0   \n",
       "2  {'added': [(5598, '\\t\\t\\tfprintf(stderr, \"File...              1.0   \n",
       "3  {'added': [(1113, '\\tstruct ipv6_txoptions *op...              6.0   \n",
       "4  {'added': [(2870, 'static int read_quant_matri...             10.0   \n",
       "\n",
       "   num_lines_deleted  num_lines_in_file  num_tokens_in_file  \\\n",
       "0                1.0              825.0              6305.0   \n",
       "1                2.0             2327.0             14667.0   \n",
       "2                1.0             5787.0             43960.0   \n",
       "3                2.0             1221.0              7876.0   \n",
       "4                1.0             2824.0             24222.0   \n",
       "\n",
       "                                repo            cve_id              cwe_id  \\\n",
       "0    https://github.com/dbry/WavPack    CVE-2016-10169           [CWE-125]   \n",
       "1  https://github.com/torvalds/linux    CVE-2017-13686           [CWE-476]   \n",
       "2       https://github.com/gpac/gpac    CVE-2020-23930           [CWE-476]   \n",
       "3  https://github.com/torvalds/linux     CVE-2016-3841  [CWE-264, CWE-416]   \n",
       "4   https://github.com/FFmpeg/FFmpeg  CVE-2018-1999015           [CWE-125]   \n",
       "\n",
       "  exp_cwe_id  \n",
       "0    CWE-125  \n",
       "1    CWE-476  \n",
       "2    CWE-476  \n",
       "3    CWE-416  \n",
       "4    CWE-125  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef1e7492-8cd8-463a-ae5a-f8193bb9ba8a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a2fc7a5-e612-434b-b918-3dafa1ec9d72",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1500c5e-c998-445c-bd03-3dd8f8236ced",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d128316-85b0-4009-9299-ec96394f0826",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abaca27a-b4b8-44b2-9221-561df097f808",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9ea80d0-aa29-4866-87ae-0de38ebe8856",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6aa6654b-cf78-44b8-a964-e93e98df460e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "def calculate_metrics(df):\n",
    "    tp = 0\n",
    "    fp = 0\n",
    "    tn = 0\n",
    "    fn = 0\n",
    "\n",
    "    for i in range(df.shape[0]):\n",
    "        res_before = str(df['res_func_before'].iloc[i]).lower()\n",
    "        res_after = str(df['res_func_after'].iloc[i]).lower()\n",
    "\n",
    "        if 'yes' in res_before:\n",
    "            tp += 1\n",
    "        else:\n",
    "            fn += 1\n",
    "\n",
    "        if 'yes' in res_after:\n",
    "            fp += 1\n",
    "        else:\n",
    "            tn += 1\n",
    "\n",
    "    return tp, fp, tn, fn\n",
    "\n",
    "model_map = {\n",
    "    'codeLlama': '[meta-llama/CodeLlama-7b-hf](https://huggingface.co/meta-llama/CodeLlama-7b-hf)',\n",
    "    'codeQwen': '[Qwen/CodeQwen1.5-7B-Chat](https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat)',\n",
    "    'deepSeekCoder': '[deepseek-ai/deepseek-coder-6.7b-instruct](https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct)',\n",
    "    'gemma': '[google/gemma-7b](https://huggingface.co/google/gemma-7b)',\n",
    "    'llama3': '[meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct)',\n",
    "    'mistral': '[mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2)',\n",
    "    'qwen': '[Qwen/Qwen1.5-7B-Chat](https://huggingface.co/Qwen/Qwen1.5-7B-Chat)',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b9ffb108-f08c-4969-9ca0-1f2d498e9aa9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Model Name | TP | FP | TN | FN |\n",
      "|------------|----|----|----|----|\n",
      "| [meta-llama/CodeLlama-7b-hf](https://huggingface.co/meta-llama/CodeLlama-7b-hf) | 0 | 0 | 2621 | 2621 |\n",
      "| [Qwen/CodeQwen1.5-7B-Chat](https://huggingface.co/Qwen/CodeQwen1.5-7B-Chat) | 0 | 0 | 2621 | 2621 |\n",
      "| [deepseek-ai/deepseek-coder-6.7b-instruct](https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct) | 29 | 28 | 2593 | 2592 |\n",
      "| [google/gemma-7b](https://huggingface.co/google/gemma-7b) | 22 | 22 | 2599 | 2599 |\n",
      "| [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) | 0 | 0 | 2621 | 2621 |\n",
      "| [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) | 29 | 28 | 2593 | 2592 |\n",
      "| [Qwen/Qwen1.5-7B-Chat](https://huggingface.co/Qwen/Qwen1.5-7B-Chat) | 0 | 0 | 2621 | 2621 |\n",
      "\n"
     ]
    }
   ],
   "source": [
    "table = \"| Model Name | TP | FP | TN | FN |\\n\"\n",
    "table += \"|------------|----|----|----|----|\\n\"\n",
    "\n",
    "csv_files = [file for file in os.listdir() if file.endswith('.csv') and 'res0_vulC7' in file and 'test' not in file]\n",
    "\n",
    "csv_files.sort()\n",
    "\n",
    "for csv_file in csv_files:\n",
    "    model_name = csv_file.split('vulC7_')[1].split('.csv')[0]\n",
    "    \n",
    "    df = pd.read_csv(csv_file)\n",
    "    tp, fp, tn, fn = calculate_metrics(df)\n",
    "    table += f\"| {model_map[model_name]} | {tp} | {fp} | {tn} | {fn} |\\n\"\n",
    "\n",
    "print(table)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22f4314e-1c00-4a53-84b8-c847875dd7c5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "0e169de1-71a4-458d-b75d-f6a81a17f45f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000, 5)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv('kat_res/new_res_D1_llama3.csv')\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "1e77b6c4-db60-4289-bdf7-c62c394dffc7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>res_before</th>\n",
       "      <th>res_after</th>\n",
       "      <th>pred_before</th>\n",
       "      <th>pred_after</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Yes, the code contains instances of the securi...</td>\n",
       "      <td>Yes, the code contains an instance of the secu...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Yes, the code contains an instance of the secu...</td>\n",
       "      <td>Yes, this code contains an out-of-bounds read ...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Yes, the code contains an instance of the secu...</td>\n",
       "      <td>Yes, this code is vulnerable to an Out-of-boun...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Yes, this code is vulnerable to an out-of-boun...</td>\n",
       "      <td>Yes, the code contains instances of Out-of-bou...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Yes, the code contains an instance of the Out-...</td>\n",
       "      <td>Yes, this code contains an instance of Out-of-...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                         res_before  \\\n",
       "0   0  Yes, the code contains instances of the securi...   \n",
       "1   1  Yes, the code contains an instance of the secu...   \n",
       "2   2  Yes, the code contains an instance of the secu...   \n",
       "3   3  Yes, this code is vulnerable to an out-of-boun...   \n",
       "4   4  Yes, the code contains an instance of the Out-...   \n",
       "\n",
       "                                           res_after  pred_before  pred_after  \n",
       "0  Yes, the code contains an instance of the secu...            1           1  \n",
       "1  Yes, this code contains an out-of-bounds read ...            1           1  \n",
       "2  Yes, this code is vulnerable to an Out-of-boun...            1           1  \n",
       "3  Yes, the code contains instances of Out-of-bou...            1           1  \n",
       "4  Yes, this code contains an instance of Out-of-...            1           1  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "9e1464a4-f9d4-43c4-87f7-cdbca2adcb15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Yes, this code is vulnerable to an Out-of-bounds Read.\n",
      "\n",
      "Explanation:\n",
      "In the loop where the current dfa and state are fetched, the code checks if the state is in an accept-only state. If it is, the code pops the stack until it is no longer in an accept-only state. This code does not check if the state is out of bounds, which could lead to a buffer overflow or out-of-bounds read. \n",
      "\n",
      "Additionally, the code does not check if the dfa or state pointers are valid before accessing them. This could also lead to a buffer overflow or out-of-bounds read. \n",
      "\n",
      "It's also worth noting that the code has some other issues, such as lack of error handling, and potentially uninitialized variables. \n",
      "\n",
      "It's recommended to fix these issues and also add proper bounds checking to prevent any potential security vulnerabilities. \n",
      "\n",
      "Also, the code is using some deprecated functions and features, it's recommended to update it to use the latest and safest methods and features. \n",
      "\n",
      "It's also recommended to use a static analyzer or a code review tool to help identify potential security vulnerabilities in the code. \n"
     ]
    }
   ],
   "source": [
    "print(df.res_before.iloc[108])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "689ecd31-6e26-4958-b3f0-fedf6a63352a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import *\n",
    "tp, fp, tn, fn, na_b, na_a = calculate_metrics(df, 'pred_before', 'pred_after')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d72118ee-84c4-4a7e-883d-03f3939b56e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(912, 914, 7, 12, 76, 79)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tp, fp, tn, fn, na_b, na_a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7d2d004e-2eb1-46c0-a18b-c8d29d310767",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAz8AAAIQCAYAAACv2NAUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAABaD0lEQVR4nO3deXwNd////2f2RFZRScSSoGqNKopYqiqoS5VWFdU21vZSS1GUT+20dFFKW6rXVUtx6UJVtVVBtfallqqqJdbWviSxJiTv3x++mZ+ThCSIg3ncb7dzu+XMvGfmNeuZ55kzExdjjBEAAAAA3ONcnV0AAAAAANwOhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB/gDjJ06FC5uLg4dIuMjFS7du2cU9AdbNmyZXJxcdGyZcty3Pbrr7/O+8Jge1ntx3ea21Hjvn375OLioqlTp97W6d5Kd9pxJqtlCiB3CD/AVaZOnSoXFxfr5e3trfDwcDVq1Ejjx4/XmTNnMg1z+PBh9e/fX/Xq1ZO/v3+OPyjvJZ9++qnq1q2r0NBQeXl5qXjx4mrfvr327dt3W+uYNWuWxo0bd1unCedw9ro+f/68hg4dart9HXm37aV//mzYsOGWj/t269WrlypXrqzg4GDly5dPZcuW1dChQ3X27FlnlwYQfoCsDB8+XJ9//rkmTpyo7t27S5J69uypqKgo/f777w5td+zYobffflv//POPoqKinFGu023atEnFixdXv379NHHiRD3//PP68ccf9fDDD+vQoUN5Ms1HHnlEFy5c0COPPGJ1c/YJMW4fZ6/r8+fPa9iwYVmGn4EDB+rChQu3v6hccFaNd8OyuRrHmRuzfv161alTR8OGDdMHH3ygevXqafTo0Xr88ceVlpbm7PJgc+7OLgC4EzVu3FhVq1a13g8YMEBLly7VE088oSeffFLbt2+Xj4+PJKlKlSo6efKkgoOD9fXXX6tly5bOKttpPv7440zdmjdvrqpVq2r69Onq37//LZ+mq6urvL29b/l4c+rcuXPy9fV12vSRcxcvXpSnp6dcXW/P933u7u5yd7+zP16dVePdsGyu5uzjzN1qxYoVmbqVLFlSffr00bp161SjRg0nVAVcwZUfIIcee+wxDRo0SPv379eMGTOs7v7+/goODr5tdZw6dUp9+vRRVFSU/Pz8FBAQoMaNG2vLli0O7dJ/f/7ll19q2LBhKly4sPz9/fXMM88oMTFRycnJ6tmzp0JCQuTn56f27dsrOTnZYRxTpkzRY489ppCQEHl5ealcuXKaOHFijuqMjIyUJCUkJFy33dNPP63KlSs7dGvatKlcXFw0f/58q9vatWvl4uKiH3/80WH+0r95f/TRR/X9999r//791s8W02tIl5aWpjfffFNFihSRt7e36tevr927d2c7L+n3Kfz555967rnnlD9/ftWuXdvqP2PGDFWpUkU+Pj4KDg5W69atdfDgQYdx7Nq1Sy1atFBYWJi8vb1VpEgRtW7dWomJiVYbFxcXdevWTTNnzlTp0qXl7e2tKlWq6Ndff81U06ZNm9S4cWMFBATIz89P9evX15o1axzapP+MZuXKlerdu7cKFiwoX19fPfXUUzp+/LhD2w0bNqhRo0a677775OPjo+LFi6tDhw6Zlt+4ceNUvnx5eXt7KzQ0VC+//LJOnz6d7TKUpL/++kvPPPOMgoOD5e3trapVqzqs42PHjqlgwYJ69NFHZYyxuu/evVu+vr5q1aqVpOuv6/TtYvbs2Ro4cKAKFy6sfPnyKSkpKcf7jnQlMA0dOlQPPPCAvL29VahQIT399NOKj4/Xvn37VLBgQUnSsGHDrBqGDh0qKev7Wi5fvqwRI0aoZMmS8vLyUmRkpP7v//4v0z4XGRmpJ554QitWrFC1atXk7e2tEiVKaPr06dku3/T7Qd577z2NHTtWERER8vHxUd26dfXHH384tM2qxtxsf//88486dOhg/dS1fPny+uyzz7Kt8XrTnTdvnipUqGCNb+HChbdsunfDceZG/f7772rXrp1KlCghb29vhYWFqUOHDjp58qRDu/Rlv3PnTj3//PMKDAxUwYIFNWjQIBljdPDgQTVr1kwBAQEKCwvTmDFjHIZPSUnR4MGDVaVKFQUGBsrX11d16tTRzz//nKM6c/qZAOS1u+frF+AO8MILL+j//u//tGjRInXu3NkpNezZs0fz5s1Ty5YtVbx4cR09elSffPKJ6tatqz///FPh4eEO7UeNGiUfHx/1799fu3fv1oQJE+Th4SFXV1edPn1aQ4cO1Zo1azR16lQVL15cgwcPtoadOHGiypcvryeffFLu7u767rvv9MorrygtLU1du3bNVNvJkyeVmpqqAwcOaPjw4ZKk+vXrX3d+6tSpo2+//VZJSUkKCAiQMUYrV66Uq6urli9frieffFKStHz5crm6uqpWrVpZjueNN95QYmKi/v77b40dO1aS5Ofn59Bm9OjRcnV1VZ8+fZSYmKh33nlHbdu21dq1a7NZ6le0bNlSpUqV0ltvvWWdnL/55psaNGiQnn32WXXq1EnHjx/XhAkT9Mgjj2jTpk0KCgpSSkqKGjVqpOTkZHXv3l1hYWH6559/tGDBAiUkJCgwMNCaxi+//KIvvvhCPXr0kJeXlz7++GM9/vjjWrdunSpUqCBJ2rZtm+rUqaOAgAD169dPHh4e+uSTT/Too4/ql19+UfXq1R3q7t69u/Lnz68hQ4Zo3759GjdunLp166YvvvhC0pXQ0bBhQxUsWFD9+/dXUFCQ9u3bp7lz5zqM5+WXX9bUqVPVvn179ejRQ3v37tWHH36oTZs2aeXKlfLw8Ljmstu2bZtq1aqlwoULq3///vL19dWXX36p5s2ba86cOXrqqacUEhKiiRMnqmXLlpowYYJ69OihtLQ0tWvXTv7+/tYVxpys6xEjRsjT01N9+vRRcnKyPD099eeff+Zo30lNTdUTTzyhJUuWqHXr1nr11Vd15swZxcXF6Y8//lBMTIwmTpyoLl266KmnntLTTz8tSapYseI1579Tp06aNm2annnmGb322mtau3atRo0ape3bt+ubb75xaLt7924988wz6tixo2JjY/XZZ5+pXbt2qlKlisqXL3/NaaSbPn26zpw5o65du+rixYv64IMP9Nhjj2nr1q0KDQ297rA52f6OHj2qGjVqWKGlYMGC+vHHH9WxY0clJSWpZ8+e2daY0YoVKzR37ly98sor8vf31/jx49WiRQsdOHBABQoUuOnp3k3HmdyKi4vTnj171L59e4WFhWnbtm2aPHmytm3bpjVr1mQKm61atVLZsmU1evRoff/99xo5cqSCg4P1ySef6LHHHtPbb7+tmTNnqk+fPnr44Yetn/wlJSXpP//5j9q0aaPOnTvrzJkz+u9//6tGjRpp3bp1qlSpksN0Ll++rISEBKWkpOiPP/7QwIED5e/vr2rVquXJcgByzACwTJkyxUgy69evv2abwMBA89BDD2XZ76uvvjKSzM8//3xD0x8yZIjJuFtGRESY2NhY6/3FixdNamqqQ5u9e/caLy8vM3z4cKvbzz//bCSZChUqmJSUFKt7mzZtjIuLi2ncuLHDOKKjo01ERIRDt/Pnz2eqsVGjRqZEiRJZ1u/l5WUkGUmmQIECZvz48dedX2OMWb9+vZFkfvjhB2OMMb///ruRZFq2bGmqV69utXvyyScdlnv6/F29rJs0aZJpHq5uW7ZsWZOcnGx1/+CDD4wks3Xr1uvWmL5e2rRp49B93759xs3Nzbz55psO3bdu3Wrc3d2t7ps2bTKSzFdffXXd6aQvuw0bNljd9u/fb7y9vc1TTz1ldWvevLnx9PQ08fHxVrdDhw4Zf39/88gjj1jd0rfnmJgYk5aWZnXv1auXcXNzMwkJCcYYY7755ptst/vly5cbSWbmzJkO3RcuXJhl94zq169voqKizMWLF61uaWlppmbNmqZUqVIObdu0aWPy5ctndu7cad59910jycybN8+hTXbrukSJEpm235zuO5999pmRZN5///1M409fjsePHzeSzJAhQzK1ybgfb9682UgynTp1cmjXp08fI8ksXbrU6hYREWEkmV9//dXqduzYMePl5WVee+21TNPKOC+SjI+Pj/n777+t7mvXrjWSTK9eva5ZozE53/46duxoChUqZE6cOOEwfOvWrU1gYKC13NPrmTJlSrbT9fT0NLt377a6bdmyxUgyEyZMyPV0s3I3HGdy8vmT1TLNar7/97//ZdqO0pf9Sy+9ZHW7fPmyKVKkiHFxcTGjR4+2up8+fdr4+Pg4fPZcvnzZYb7S24WGhpoOHTpkqmH16tXWNiXJlC5d+oY/G4FbiZ+9Abnk5+eX5VPfbhcvLy/r3oXU1FSdPHlSfn5+Kl26tDZu3Jip/YsvvujwjXz16tVljMn0k6bq1avr4MGDunz5stUt/b4mSUpMTNSJEydUt25d7dmzx+HnWul+/PFH/fDDDxozZoyKFSumc+fOZTs/Dz30kPz8/Kyf1ixfvlxFihTRiy++qI0bN+r8+fMyxmjFihWqU6dOtuO7nvbt28vT09N6nz6+PXv25Gj4f//73w7v586dq7S0ND377LM6ceKE9QoLC1OpUqWsn4OkX9n56aefdP78+etOIzo6WlWqVLHeFytWTM2aNdNPP/2k1NRUpaamatGiRWrevLlKlChhtStUqJCee+45rVixQklJSQ7jfOmllxy+/a1Tp45SU1O1f/9+SVJQUJAkacGCBbp06VKWdX311VcKDAxUgwYNHOa1SpUq8vPzu+5PX06dOqWlS5fq2Wef1ZkzZ6xhT548qUaNGmnXrl36559/rPYffvihAgMD9cwzz2jQoEF64YUX1KxZs+sut4xiY2Mdtl8p5/vOnDlzdN9991kPO7najTym+YcffpAk9e7d26H7a6+9Jkn6/vvvHbqXK1fOYVsvWLCgSpcunePttHnz5ipcuLD1vlq1aqpevbpVx/Vkt/0ZYzRnzhw1bdpUxhiHbaFRo0ZKTEzM8jiUnZiYGJUsWdJ6X7FiRQUEBFjzfLPTvZuOM7l19XZ+8eJFnThxwrqnJqtl0qlTJ+tvNzc3Va1aVcYYdezY0eoeFBSUaZtzc3Oz5istLU2nTp3S5cuXVbVq1SynU65cOcXFxWnevHnq16+ffH19edob7gj87A3IpbNnzyokJMRp009LS9MHH3ygjz/+WHv37lVqaqrVL/3nIVcrVqyYw/v0E/GiRYtm6p6WlqbExERrPCtXrtSQIUO0evXqTCftiYmJDj/XkqR69epJuvLAiGbNmqlChQry8/NTt27drjk/bm5uio6O1vLlyyVdOSmpU6eOateurdTUVK1Zs0ahoaE6derUTZ+UZFwW+fPnl6Qc37NSvHhxh/e7du2SMUalSpXKsn166CxevLh69+6t999/XzNnzlSdOnX05JNPWr+7v1pW43rggQd0/vx56z6d8+fPq3Tp0pnalS1bVmlpaTp48KDDz6Oym++6deuqRYsWGjZsmMaOHatHH31UzZs313PPPScvLy9rXhMTE6+57R87dizL7tKVn3EZYzRo0CANGjTomsOnn7AHBwdr/PjxatmypUJDQzV+/PhrjvtaMq4rKef7Tnx8vEqXLn3Lbszfv3+/XF1ddf/99zt0DwsLU1BQkBVC02VcX9KVdZbT7fRa29CXX355w8Omb3+urq5KSEjQ5MmTNXny5CzHcb1t4Vqym+fjx4/f1HTvpuNMbp06dUrDhg3T7NmzMy2DrL6kyuozwdvbW/fdd1+m7hnvG5o2bZrGjBmjv/76y+GLkqz2t4CAAMXExEiSmjVrplmzZqlZs2bauHGjHnzwwdzNJHALEX6AXPj777+VmJiY6STmdnrrrbc0aNAgdejQQSNGjFBwcLBcXV3Vs2fPLB8h6ubmluV4rtXd/L97WeLj41W/fn2VKVNG77//vooWLSpPT0/98MMPGjt2bLaPKy1ZsqQeeughzZw587rhR5Jq166tN998UxcvXtTy5cv1xhtvKCgoSBUqVNDy5cut+xRu9qQku3nOTsYrCWlpadbN0VmN++p7AcaMGaN27drp22+/1aJFi9SjRw+NGjVKa9asUZEiRXIxF7mX3Xyn/2PGNWvW6LvvvtNPP/2kDh06aMyYMVqzZo38/PyUlpamkJAQzZw5M8txpT8AICvp20qfPn3UqFGjLNtk3Kd++uknSVdOGP/++2/r6lROZVxXUu73nVstp1eNbnY7zUvpy+n5559XbGxslm2ud+/TtWQ3z7diunfLcSa3nn32Wa1atUp9+/ZVpUqVrP31Wo+Vzqq+nNQ8Y8YMtWvXTs2bN1ffvn0VEhIiNzc3jRo1SvHx8dnW+fTTT+uFF17Q7NmzCT9wKsIPkAuff/65JF3zBO52+Prrr1WvXj3997//deiekJCQ6Zu7m/Hdd98pOTlZ8+fPd/imMKdP9pGkCxcuZHqaVVbq1KmjlJQU/e9//9M///xjnXw88sgj1knJAw88kO3N2rf7P8eXLFlSxhgVL15cDzzwQLbto6KiFBUVpYEDB2rVqlWqVauWJk2apJEjR1ptdu3alWm4nTt3Kl++fFbAyJcvn3bs2JGp3V9//SVXV9dMV/VyqkaNGqpRo4befPNNzZo1S23bttXs2bPVqVMnlSxZUosXL1atWrWyDBbXk/7zPA8PD+ub4OtZuHCh/vOf/6hfv36aOXOmYmNjtXbtWocrMTeyrnO675QsWVJr167VpUuXrvkQh9xMPyIiQmlpadq1a5fKli1rdT969KgSEhIUERGRyzm5vmttQxmfSpabYa/e/vz9/ZWampqjdXmrFCxY8Kane7ceZ67n9OnTWrJkiYYNG+bwsJqs1uPN+vrrr1WiRAnNnTvXYRkMGTIkR8MnJydbvy4AnIl7foAcWrp0qUaMGKHixYurbdu2TqvDzc0t0zeIX331lcM9E7dqOpLjN3+JiYmaMmWKQ7vLly9n+XOOdevWaevWrQ7/L+laqlevLg8PD7399tsKDg62frJVp04drVmzRr/88kuOvo319fW9rR+sTz/9tNzc3DRs2LBM68QYY/1kJCkpyeFeKulKEHJ1dc0UDlevXu3w+/mDBw/q22+/VcOGDeXm5iY3Nzc1bNhQ3377rfbt22e1O3r0qGbNmqXatWsrICAgV/Nx+vTpTPWnP7kpvb5nn31WqampGjFiRKbh05/qdC0hISF69NFH9cknn+jw4cOZ+l/92O2EhAR16tRJ1apV01tvvaX//Oc/2rhxo9566y2HYW5kXed032nRooVOnDihDz/8MNM40ofPly+fVW92/vWvf0lSpn+M+f7770uSmjRpkqP6c2revHkO87Ru3TqtXbtWjRs3znbYnGx/LVq00Jw5czI9PltSpkeo3yq3Yrp363HmerI6TkuZt7W8mtbatWu1evVqh3YJCQlZ3jv4n//8R5Jy9JkA5CWu/ABZ+PHHH/XXX3/p8uXLOnr0qJYuXaq4uDhFRERo/vz5mf7pXfo399u2bZN05QpR+j95Gzhw4C2t7YknntDw4cPVvn171axZU1u3btXMmTMdbn6/FRo2bChPT081bdpUL7/8ss6ePatPP/1UISEhDiewZ8+eVdGiRdWqVSuVL19evr6+2rp1q6ZMmaLAwMBr3uNxtXz58qlKlSpas2aN9b83pCvfyJ47d07nzp3L0UlJlSpV9MUXX6h37956+OGH5efnp6ZNm974QshGyZIlNXLkSA0YMED79u1T8+bN5e/vr7179+qbb77RSy+9pD59+mjp0qXq1q2bWrZsqQceeECXL1/W559/bp3QXa1ChQpq1KiRw6OGpSv/TybdyJEjFRcXp9q1a+uVV16Ru7u7PvnkEyUnJ+udd97J9XxMmzZNH3/8sZ566imVLFlSZ86c0aeffqqAgADrxL1u3bp6+eWXNWrUKG3evFkNGzaUh4eHdu3apa+++koffPCBnnnmmWtO46OPPlLt2rUVFRWlzp07q0SJEjp69KhWr16tv//+2/pfO6+++qpOnjypxYsXy83NTY8//rg6deqkkSNHqlmzZtbPZW5kXed033nxxRc1ffp09e7dW+vWrVOdOnV07tw5LV68WK+88oqaNWsmHx8flStXTl988YUeeOABBQcHq0KFCtbjoK/24IMPKjY2VpMnT1ZCQoLq1q2rdevWadq0aWrevLl1r9ytcv/996t27drq0qWLkpOTNW7cOBUoUED9+vXLdticbH+jR4/Wzz//rOrVq6tz584qV66cTp06pY0bN2rx4sU6derULZ2fWzXdu+U489lnn2X5P45effXVTN0CAgL0yCOP6J133tGlS5dUuHBhLVq0SHv37r1l9aR74oknNHfuXD311FNq0qSJ9u7dq0mTJqlcuXIODzJYtmyZevTooWeeeUalSpVSSkqKli9frrlz56pq1ap6/vnnb3ltQK7cpqfKAXeF9EeNpr88PT1NWFiYadCggfnggw9MUlJSlsNdPUzGV27k9FHXr732milUqJDx8fExtWrVMqtXrzZ169Y1devWtdqlP3Y14+OVr/U41fRpHz9+3Oo2f/58U7FiRePt7W0iIyPN22+/bT0GeO/evcYYY5KTk82rr75qKlasaAICAoyHh4eJiIgwHTt2tNrkRN++fY0k8/bbbzt0v//++40kh8c6Xz1/Vz869ezZs+a5554zQUFBRpL1ONprLYusHhublayWzdXmzJljateubXx9fY2vr68pU6aM6dq1q9mxY4cxxpg9e/aYDh06mJIlSxpvb28THBxs6tWrZxYvXuwwHkmma9euZsaMGaZUqVLGy8vLPPTQQ1k+Hnbjxo2mUaNGxs/Pz+TLl8/Uq1fPrFq1yqHNtdZ1xmW3ceNG06ZNG1OsWDHj5eVlQkJCzBNPPOHwyON0kydPNlWqVDE+Pj7G39/fREVFmX79+plDhw5ddxkaY0x8fLx58cUXTVhYmPHw8DCFCxc2TzzxhPn666+NMcZ8++23RpIZM2aMw3BJSUkmIiLCPPjgg9Zj23O7ro3J+b5jzJXHB7/xxhumePHixsPDw4SFhZlnnnnGYTtctWqVqVKlivH09HR47HVW+/GlS5fMsGHDrPEVLVrUDBgwwOHR38Zc2d+bNGmSqfasaswofXt+9913zZgxY0zRokWNl5eXqVOnjtmyZYtD22s9cjqn29/Ro0dN165dTdGiRa3lU79+fTN58uRM9eTkUdddu3bNNI2Mx76cTvd67uTjTMbPn4yvgwcPZjmuv//+2zz11FMmKCjIBAYGmpYtW5pDhw5lehT7tY5jsbGxxtfXN1M9devWNeXLl7fep6WlmbfeestERERY28aCBQtMbGysw6O/d+/ebV588UVTokQJ4+PjY7y9vU358uXNkCFDzNmzZ6+7DIDbwcWYO+AOSgCAXFxc1LVr1yx/bgVkZ9++fSpevLjeffdd9enTJ9fDs/0BsAPu+QEAAABgC9zzA9wGiYmJunDhwnXbhIWF3aZqAAAA7InwA9wGr776qqZNm3bdNvwCFQAAIG9xzw9wG/z55586dOjQddvczv+ZAQAAYEeEHwAAAAC2wAMPAAAAANjCXXnPT1pamg4dOiR/f3/rn5QBAAAAsB9jjM6cOaPw8HC5ul7/2s5dGX4OHTqkokWLOrsMAAAAAHeIgwcPqkiRItdtc1eGH39/f0lXZjAgIMDJ1QAAAABwlqSkJBUtWtTKCNdzV4af9J+6BQQEEH4AAAAA5Oh2GB54AAAAAMAWCD8AAAAAbIHwAwAAAMAWCD8AAAAAbIHwAwAAAMAWCD8AAAAAbIHwA9zFzpw5o549eyoiIkI+Pj6qWbOm1q9fb/WfO3euGjZsqAIFCsjFxUWbN2++5riMMWrcuLFcXFw0b968vC8eAADgNiP83MNu9sT41KlT6t69u0qXLi0fHx8VK1ZMPXr0UGJi4m2eE1xLp06dFBcXp88//1xbt25Vw4YNFRMTo3/++UeSdO7cOdWuXVtvv/12tuMaN25cjp6PDwAAcLe6K//JKXKmU6dO+uOPP/T5558rPDxcM2bMUExMjP78808VLlzYOjF+9tln1blz50zDHzp0SIcOHdJ7772ncuXKaf/+/fr3v/+tQ4cO6euvv3bCHOFqFy5c0Jw5c/Ttt9/qkUcekSQNHTpU3333nSZOnKiRI0fqhRdekCTt27fvuuPavHmzxowZow0bNqhQoUJ5XToAAIBTEH7uUbfixLhChQqaM2eO9b5kyZJ688039fzzz+vy5ctyd2fzcabLly8rNTVV3t7eDt19fHy0YsWKHI/n/Pnzeu655/TRRx8pLCzsVpcJAABwx+Bnb/eoW3VinFFiYqICAgIIPncAf39/RUdHa8SIETp06JBSU1M1Y8YMrV69WocPH87xeHr16qWaNWuqWbNmeVgtAACA8xF+7lG36sT4aidOnNCIESP00ksv3eJqcaM+//xzGWNUuHBheXl5afz48WrTpo1cXXO2a8+fP19Lly7VuHHj8rZQAACAOwDh5x52syfGV0tKSlKTJk1Urlw5DR069NYXixtSsmRJ/fLLLzp79qwOHjyodevW6dKlSypRokSOhl+6dKni4+MVFBQkd3d364peixYt9Oijj+Zh5QAAALcfv126h6WfGJ87d05JSUkqVKiQWrVqleMT43RnzpzR448/Ln9/f33zzTfy8PDIo4pxo3x9feXr66vTp0/rp59+0jvvvJOj4fr3769OnTo5dIuKitLYsWPVtGnTvCgVAADAaQg/NnCjJ8bSlSs+jRo1kpeXl+bPn5/pHiI4108//SRjjEqXLq3du3erb9++KlOmjNq3by/pyuPKDxw4oEOHDkmSduzYIUkKCwtzeGVUrFgxFS9e/PbNCAAAwG1A+LmH3eyJcVJSkho2bKjz589rxowZSkpKUlJSkiSpYMGCcnNzc86MwZKYmKgBAwbo77//VnBwsFq0aKE333zTujo3f/58a31LUuvWrSVJQ4YM4eeLAADAdlyMMcbZReRWUlKSAgMDrSePIWtffvlllifGgYGBkqSpU6c6nBinSz8xXrZsmerVq5fluPfu3avIyMi8LB8AAADIVm6yAeEHAAAAwF0rN9mAp70BAAAAsAXu+QFyamigsyvA0ERnVwAAAO5ihJ9bILL/984uAZL2jW7i7BIAAABwB+NnbwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsgfADAAAAwBYIPwAAAABsIVfhJzU1VYMGDVLx4sXl4+OjkiVLasSIETLGWG2MMRo8eLAKFSokHx8fxcTEaNeuXQ7jOXXqlNq2bauAgAAFBQWpY8eOOnv27K2ZIwAAAADIQq7Cz9tvv62JEyfqww8/1Pbt2/X222/rnXfe0YQJE6w277zzjsaPH69JkyZp7dq18vX1VaNGjXTx4kWrTdu2bbVt2zbFxcVpwYIF+vXXX/XSSy/durkCAAAAgAzcc9N41apVatasmZo0aSJJioyM1P/+9z+tW7dO0pWrPuPGjdPAgQPVrFkzSdL06dMVGhqqefPmqXXr1tq+fbsWLlyo9evXq2rVqpKkCRMm6F//+pfee+89hYeH38r5AwAAAABJubzyU7NmTS1ZskQ7d+6UJG3ZskUrVqxQ48aNJUl79+7VkSNHFBMTYw0TGBio6tWra/Xq1ZKk1atXKygoyAo+khQTEyNXV1etXbs2y+kmJycrKSnJ4QUAAAAAuZGrKz/9+/dXUlKSypQpIzc3N6WmpurNN99U27ZtJUlHjhyRJIWGhjoMFxoaavU7cuSIQkJCHItwd1dwcLDVJqNRo0Zp2LBhuSkVAAAAABzk6srPl19+qZkzZ2rWrFnauHGjpk2bpvfee0/Tpk3Lq/okSQMGDFBiYqL1OnjwYJ5ODwAAAMC9J1dXfvr27av+/furdevWkqSoqCjt379fo0aNUmxsrMLCwiRJR48eVaFChazhjh49qkqVKkmSwsLCdOzYMYfxXr58WadOnbKGz8jLy0teXl65KRUAAAAAHOTqys/58+fl6uo4iJubm9LS0iRJxYsXV1hYmJYsWWL1T0pK0tq1axUdHS1Jio6OVkJCgn777TerzdKlS5WWlqbq1avf8IwAAAAAwPXk6spP06ZN9eabb6pYsWIqX768Nm3apPfff18dOnSQJLm4uKhnz54aOXKkSpUqpeLFi2vQoEEKDw9X8+bNJUlly5bV448/rs6dO2vSpEm6dOmSunXrptatW/OkNwAAAAB5JlfhZ8KECRo0aJBeeeUVHTt2TOHh4Xr55Zc1ePBgq02/fv107tw5vfTSS0pISFDt2rW1cOFCeXt7W21mzpypbt26qX79+nJ1dVWLFi00fvz4WzdXAAAAAJCBizHGOLuI3EpKSlJgYKASExMVEBDg7HIU2f97Z5cASftGN8nbCQwNzNvxI3tDE51dAQAAuMPkJhvk6p4fAAAAALhbEX4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAtEH4AAAAA2ALhBwAAAIAt5Dr8/PPPP3r++edVoEAB+fj4KCoqShs2bLD6G2M0ePBgFSpUSD4+PoqJidGuXbscxnHq1Cm1bdtWAQEBCgoKUseOHXX27NmbnxsAAAAAuIZchZ/Tp0+rVq1a8vDw0I8//qg///xTY8aMUf78+a0277zzjsaPH69JkyZp7dq18vX1VaNGjXTx4kWrTdu2bbVt2zbFxcVpwYIF+vXXX/XSSy/durkCAAAAgAzcc9P47bffVtGiRTVlyhSrW/Hixa2/jTEaN26cBg4cqGbNmkmSpk+frtDQUM2bN0+tW7fW9u3btXDhQq1fv15Vq1aVJE2YMEH/+te/9N577yk8PPxWzBcAAAAAOMjVlZ/58+eratWqatmypUJCQvTQQw/p008/tfrv3btXR44cUUxMjNUtMDBQ1atX1+rVqyVJq1evVlBQkBV8JCkmJkaurq5au3ZtltNNTk5WUlKSwwsAAAAAciNX4WfPnj2aOHGiSpUqpZ9++kldunRRjx49NG3aNEnSkSNHJEmhoaEOw4WGhlr9jhw5opCQEIf+7u7uCg4OttpkNGrUKAUGBlqvokWL5qZsAAAAAMhd+ElLS1PlypX11ltv6aGHHtJLL72kzp07a9KkSXlVnyRpwIABSkxMtF4HDx7M0+kBAAAAuPfkKvwUKlRI5cqVc+hWtmxZHThwQJIUFhYmSTp69KhDm6NHj1r9wsLCdOzYMYf+ly9f1qlTp6w2GXl5eSkgIMDhBQAAAAC5kavwU6tWLe3YscOh286dOxURESHpysMPwsLCtGTJEqt/UlKS1q5dq+joaElSdHS0EhIS9Ntvv1ltli5dqrS0NFWvXv2GZwQAAAAAridXT3vr1auXatasqbfeekvPPvus1q1bp8mTJ2vy5MmSJBcXF/Xs2VMjR45UqVKlVLx4cQ0aNEjh4eFq3ry5pCtXih5//HHr53KXLl1St27d1Lp1a570BgAAACDP5Cr8PPzww/rmm280YMAADR8+XMWLF9e4cePUtm1bq02/fv107tw5vfTSS0pISFDt2rW1cOFCeXt7W21mzpypbt26qX79+nJ1dVWLFi00fvz4WzdXAAAAAJCBizHGOLuI3EpKSlJgYKASExPviPt/Ivt/7+wSIGnf6CZ5O4GhgXk7fmRvaKKzKwAAAHeY3GSDXN3zAwAAAAB3K8IPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwBcIPAAAAAFsg/AAAAACwhZsKP6NHj5aLi4t69uxpdbt48aK6du2qAgUKyM/PTy1atNDRo0cdhjtw4ICaNGmifPnyKSQkRH379tXly5dvphQAAAAAuK4bDj/r16/XJ598oooVKzp079Wrl7777jt99dVX+uWXX3To0CE9/fTTVv/U1FQ1adJEKSkpWrVqlaZNm6apU6dq8ODBNz4XAAAAAJCNGwo/Z8+eVdu2bfXpp58qf/78VvfExET997//1fvvv6/HHntMVapU0ZQpU7Rq1SqtWbNGkrRo0SL9+eefmjFjhipVqqTGjRtrxIgR+uijj5SSknJr5goAAAAAMrih8NO1a1c1adJEMTExDt1/++03Xbp0yaF7mTJlVKxYMa1evVqStHr1akVFRSk0NNRq06hRIyUlJWnbtm03Ug4AAAAAZMs9twPMnj1bGzdu1Pr16zP1O3LkiDw9PRUUFOTQPTQ0VEeOHLHaXB180vun98tKcnKykpOTrfdJSUm5LRsAAACAzeXqys/Bgwf16quvaubMmfL29s6rmjIZNWqUAgMDrVfRokVv27QBAAAA3BtyFX5+++03HTt2TJUrV5a7u7vc3d31yy+/aPz48XJ3d1doaKhSUlKUkJDgMNzRo0cVFhYmSQoLC8v09Lf09+ltMhowYIASExOt18GDB3NTNgAAAADkLvzUr19fW7du1ebNm61X1apV1bZtW+tvDw8PLVmyxBpmx44dOnDggKKjoyVJ0dHR2rp1q44dO2a1iYuLU0BAgMqVK5fldL28vBQQEODwAgAAAIDcyNU9P/7+/qpQoYJDN19fXxUoUMDq3rFjR/Xu3VvBwcEKCAhQ9+7dFR0drRo1akiSGjZsqHLlyumFF17QO++8oyNHjmjgwIHq2rWrvLy8btFsAQAAAICjXD/wIDtjx46Vq6urWrRooeTkZDVq1Egff/yx1d/NzU0LFixQly5dFB0dLV9fX8XGxmr48OG3uhQAAAAAsLgYY4yzi8itpKQkBQYGKjEx8Y74CVxk/++dXQIk7RvdJG8nMDQwb8eP7A1NdHYFAADgDpObbHBD/+cHAAAAAO42hB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtkD4AQAAAGALhB8AAAAAtpCr8DNq1Cg9/PDD8vf3V0hIiJo3b64dO3Y4tLl48aK6du2qAgUKyM/PTy1atNDRo0cd2hw4cEBNmjRRvnz5FBISor59++ry5cs3PzcAAAAAcA25Cj+//PKLunbtqjVr1iguLk6XLl1Sw4YNde7cOatNr1699N133+mrr77SL7/8okOHDunpp5+2+qempqpJkyZKSUnRqlWrNG3aNE2dOlWDBw++dXMFAAAAABm4GGPMjQ58/PhxhYSE6JdfftEjjzyixMREFSxYULNmzdIzzzwjSfrrr79UtmxZrV69WjVq1NCPP/6oJ554QocOHVJoaKgkadKkSXr99dd1/PhxeXp6ZjvdpKQkBQYGKjExUQEBATda/i0T2f97Z5cASftGN8nbCQwNzNvxI3tDE51dAQAAuMPkJhvc1D0/iYlXTkSCg4MlSb/99psuXbqkmJgYq02ZMmVUrFgxrV69WpK0evVqRUVFWcFHkho1aqSkpCRt27Yty+kkJycrKSnJ4QUAAAAAuXHD4SctLU09e/ZUrVq1VKFCBUnSkSNH5OnpqaCgIIe2oaGhOnLkiNXm6uCT3j+9X1ZGjRqlwMBA61W0aNEbLRsAAACATd1w+Onatav++OMPzZ49+1bWk6UBAwYoMTHReh08eDDPpwkAAADg3uJ+IwN169ZNCxYs0K+//qoiRYpY3cPCwpSSkqKEhASHqz9Hjx5VWFiY1WbdunUO40t/Glx6m4y8vLzk5eV1I6UCAAAAgKRcXvkxxqhbt2765ptvtHTpUhUvXtyhf5UqVeTh4aElS5ZY3Xbs2KEDBw4oOjpakhQdHa2tW7fq2LFjVpu4uDgFBASoXLlyNzMvAAAAAHBNubry07VrV82aNUvffvut/P39rXt0AgMD5ePjo8DAQHXs2FG9e/dWcHCwAgIC1L17d0VHR6tGjRqSpIYNG6pcuXJ64YUX9M477+jIkSMaOHCgunbtytUdAAAAAHkmV+Fn4sSJkqRHH33UofuUKVPUrl07SdLYsWPl6uqqFi1aKDk5WY0aNdLHH39stXVzc9OCBQvUpUsXRUdHy9fXV7GxsRo+fPjNzQkAAAAAXEeuwk9O/iWQt7e3PvroI3300UfXbBMREaEffvghN5MGAAAAgJtyU//nBwAAAADuFoQfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAABgC4QfAAAAALZA+AEAAADyQGRkpFxcXDK9unbtKkmKj4/XU089pYIFCyogIEDPPvusjh496uSq722EHwAAgNssu5NiSVq9erUee+wx+fr6KiAgQI888oguXLjgxKqRW+vXr9fhw4etV1xcnCSpZcuWOnfunBo2bCgXFxctXbpUK1euVEpKipo2baq0tDQnV37vcnd2AQAAAHazfv16paamWu//+OMPNWjQQC1btpR0Jfg8/vjjGjBggCZMmCB3d3dt2bJFrq58b303KViwoMP70aNHq2TJkqpbt67i4uK0b98+bdq0SQEBAZKkadOmKX/+/Fq6dKliYmKcUfI9j/ADAABwm13vpFiSevXqpR49eqh///5Wm9KlS9/WGnFrpaSkaMaMGerdu7dcXFyUnJwsFxcXeXl5WW28vb3l6uqqFStWEH7yCF8fAAAAOFH6SXGHDh3k4uKiY8eOae3atQoJCVHNmjUVGhqqunXrasWKFc4uFTdh3rx5SkhIULt27SRJNWrUkK+vr15//XWdP39e586dU58+fZSamqrDhw87t9h7GOEHAADAiTKeFO/Zs0eSNHToUHXu3FkLFy5U5cqVVb9+fe3atcuJleJm/Pe//1Xjxo0VHh4u6crVv6+++krfffed/Pz8FBgYqISEBFWuXJmfN+YhfvYGAADgRBlPitNvdn/55ZfVvn17SdJDDz2kJUuW6LPPPtOoUaOcVituzP79+7V48WLNnTvXoXvDhg0VHx+vEydOyN3dXUFBQQoLC1OJEiWcVOm9j/ADAADgJFmdFBcqVEiSVK5cOYe2ZcuW1YEDB25rfbg1pkyZopCQEDVp0iTL/vfdd58kaenSpTp27JiefPLJ21merXBNDQAAwEmyOimOjIxUeHi4duzY4dB2586dioiIuN0l4ialpaVpypQpio2Nlbu743WHKVOmaM2aNYqPj9eMGTPUsmVL9erVi4db5CGu/AAAADjBtU6KXVxc1LdvXw0ZMkQPPvigKlWqpGnTpumvv/7S119/7cSKcSMWL16sAwcOqEOHDpn67dixQwMGDNCpU6cUGRmpN954Q7169XJClfZB+AEAAHCC650U9+zZUxcvXlSvXr106tQpPfjgg4qLi1PJkiWdUCluRsOGDWWMybLf6NGjNXr06Ntckb0RfgAAAJzgeifFktS/f3+H//MD4OZxzw8AAAAAW+DKDwAAAO46Y1o94ewSbO+1LxY4u4Rc48oPAAAAAFvgyg8AALinfPTvpc4uwfa6TnrM2SUAWeLKDwAAAABbIPwAAAAAsAXCDwAAAABbIPwAAAAAsAXCDwAAAABbIPwAAAAAsAXCDwAAAABbIPwAAAAAsAXCDwAAAABbIPwAAAAAsAXCDwAAAABbIPwAAAAAsAXCDwAAAABbIPwAAAAAsAXCDwAAAABbIPwAAAAAsAXCDwAAd5hff/1VTZs2VXh4uFxcXDRv3jyr36VLl/T6668rKipKvr6+Cg8P14svvqhDhw45r2AAuEsQfgAAuMOcO3dODz74oD766KNM/c6fP6+NGzdq0KBB2rhxo+bOnasdO3boySefdEKlAHB3cXd2AQCAmxMZGan9+/dn6v7KK69kefKMO1/jxo3VuHHjLPsFBgYqLi7OoduHH36oatWq6cCBAypWrNjtKBEA7kqEHwC4y61fv16pqanW+z/++EMNGjRQy5YtnVgVbqfExES5uLgoKCjI2aUAwB2N8AMAd7mCBQs6vB89erRKliypunXrOqki3E4XL17U66+/rjZt2iggIMDZ5QDAHY17fgDgHpKSkqIZM2aoQ4cOcnFxcXY5yGOXLl3Ss88+K2OMJk6c6OxyAOCOx5UfALiHzJs3TwkJCWrXrp2zS0EeSw8++/fv19KlS7nqAwA5QPgBgHvIf//7XzVu3Fjh4eHOLgV5KD347Nq1Sz///LMKFCjg7JIA4K5A+AGAe8T+/fu1ePFizZ0719ml4CadPXtWu3fvtt7v3btXmzdvVnBwsAoVKqRnnnlGGzdu1IIFC5SamqojR45IkoKDg+Xp6emssgHgjkf4AYB7xJQpUxQSEqImTZo4uxTcpA0bNqhevXrW+969e0uSYmNjNXToUM2fP1+SVKlSJYfhfv75Zz366KO3q0wAuOsQfgDgHpCWlqYpU6YoNjZW7u4c2u92jz76qIwx1+x/vX4AgGvjaW8AcA9YvHixDhw4oA4dOji7FAAA7lh8PQgA94CGDRtyNQAAgGwQfgAAtrK9TFlnl2B7Zf/a7uwSANgUP3sDAAAAYAtODT8fffSRIiMj5e3trerVq2vdunXOLAcAAADAPcxpP3v74osv1Lt3b02aNEnVq1fXuHHj1KhRI+3YsUMhISHOKguAjUVNi3J2Cba3NXars0sAANzDnHbl5/3331fnzp3Vvn17lStXTpMmTVK+fPn02WefOaskAAAAAPcwp1z5SUlJ0W+//aYBAwZY3VxdXRUTE6PVq1dnap+cnKzk5GTrfWJioiQpKSkp74vNgbTk884uAboN20MyT9Jyujxex6kXUvN0/Mje7Tiun01lPTtbXq/nCynn8nT8yN7t2JcvXrqU59PA9d0p5+LpdeTkqadOCT8nTpxQamqqQkNDHbqHhobqr7/+ytR+1KhRGjZsWKbuRYsWzbMacfcJHOfsCpDnRgc6uwLkscAurGNbCGQ93+v6TnF2BbgdBn5zZ+3LZ86cUWA2x5e74lHXAwYMUO/eva33aWlpOnXqlAoUKCAXFxcnVnZvSEpKUtGiRXXw4EEFBAQ4uxzkAdaxPbCe732sY3tgPd/7WMe3ljFGZ86cUXh4eLZtnRJ+7rvvPrm5ueno0aMO3Y8ePaqwsLBM7b28vOTl5eXQLSgoKC9LtKWAgAB2wHsc69geWM/3PtaxPbCe732s41snuys+6ZzywANPT09VqVJFS5YssbqlpaVpyZIlio6OdkZJAAAAAO5xTvvZW+/evRUbG6uqVauqWrVqGjdunM6dO6f27ds7qyQAAAAA9zCnhZ9WrVrp+PHjGjx4sI4cOaJKlSpp4cKFmR6CgLzn5eWlIUOGZPppIe4drGN7YD3f+1jH9sB6vvexjp3HxeTkmXAAAAAAcJdz2j85BQAAAIDbifADAAAAwBYIPwAAAABsgfCDm/boo4+qZ8+e120zderUe+Z/Mxlj9NJLLyk4OFguLi7avHnzddvv27cvR+3uBjmdl5xsE7g3uLi4aN68ec4uI8fsvP/erHvpOO4st2oZHjlyRA0aNJCvry/r5B6T22MUco/wk0fsdPI3d+5cjRgxwnofGRmpcePGObRp1aqVdu7ceZsryxsLFy7U1KlTtWDBAh0+fFgVKlRwdkm3TdGiRR3medmyZXJxcVFCQoJDu4zbxN3GTvtvTg0dOlSVKlXK1P3w4cNq3Ljx7S/oBtl5/82Ne/04frcbO3asDh8+rM2bN7NO7jFZHaPuti+Z7nROe9S13RljlJqaKnf3u38VBAcHZ9vGx8dHPj4+t6GavBcfH69ChQqpZs2azi7ltnNzc1NYWFi27XKyTdzN7qX992blZHu4k9h5/71Z99Jx/G4XHx+vKlWqqFSpUjc8jpSUFHl6et7CqnAr5OUx6tKlS/Lw8Ljl473rGNxysbGxRpLDa8qUKUaS+eGHH0zlypWNh4eH+fnnn01sbKxp1qyZw/CvvvqqqVu3rvU+NTXVvPXWWyYyMtJ4e3ubihUrmq+++ipHtfz8889GklmwYIGJiooyXl5epnr16mbr1q0O7b7++mtTrlw54+npaSIiIsx7773n0P+jjz4y999/v/Hy8jIhISGmRYsWVr+6deuaV1991fo747wbY8yUKVNMYGCgMcaYHTt2GElm+/btDtN4//33TYkSJaz3W7duNY8//rjx9fU1ISEh5vnnnzfHjx/P0XznlYzrNiIiwvz444+mVq1aJjAw0AQHB5smTZqY3bt3W8Ps3bvXSDKbNm0yxhhz6tQp89xzz5n77rvPeHt7m/vvv9989tlnVvsDBw6Yli1bmsDAQJM/f37z5JNPmr179+a4vmbNmpmhQ4ea++67z/j7+5uXX37ZJCcnW20uXrxounfvbgoWLGi8vLxMrVq1zLp166z+16vv6nlJ//vqV2xsrDHGcZsYMGCAqVatWqZaK1asaIYNG2a9//TTT02ZMmWMl5eXKV26tPnoo49yNM+32p24/y5evNhUqVLF+Pj4mOjoaPPXX385tJs3b5556KGHjJeXlylevLgZOnSouXTpktV/+/btplatWsbLy8uULVvWxMXFGUnmm2++sdr069fPlCpVyvj4+JjixYubgQMHmpSUFGOMseY/4zIxxjiMJzo62vTr18+htmPHjhl3d3fzyy+/GGOubH+vvfaaCQ8PN/ny5TPVqlUzP//8c46Wx826W/bfd99914SFhZng4GDzyiuvWOvBmJwtv8mTJ5siRYoYHx8f07x5czNmzBjr+GuMMbt37zZPPvmkCQkJMb6+vqZq1aomLi7O6n+vH8ezU7duXdO9e3fTt29fkz9/fhMaGmqGDBli9R8zZoypUKGCyZcvnylSpIjp0qWLOXPmTI7Gnb4Mv/nmG+sztWHDhubAgQMO7a63T0dERGR53N2/f7958sknja+vr/H39zctW7Y0R44cscY5ZMgQ8+CDD5pPP/3UREZGGhcXF2OMMadPnzYdO3a0PjPq1atnNm/efBNL8O6Wl+v/xIkTpnXr1iY8PNz4+PiYChUqmFmzZln9szpGZVzfERERVvvsjv2SzMcff2yaNm1q8uXL5zAfdkb4yQMJCQkmOjradO7c2Rw+fNgcPnzYLF682EgyFStWNIsWLTK7d+82J0+ezNHJ08iRI02ZMmXMwoULTXx8vJkyZYrx8vIyy5Yty7aW9JOnsmXLmkWLFpnff//dPPHEEyYyMtL6QN2wYYNxdXU1w4cPNzt27DBTpkwxPj4+1snN+vXrjZubm5k1a5bZt2+f2bhxo/nggw+saVx9onvy5ElTpEgRM3z4cGvejXH80DTGmKpVq5qBAwc61FqlShWr2+nTp03BggXNgAEDzPbt283GjRtNgwYNTL169XKyCvJMQkKCGT58uClSpIg5fPiwOXbsmPn666/NnDlzzK5du8ymTZtM06ZNTVRUlElNTTXGZD556tq1q6lUqZJZv3692bt3r4mLizPz5883xhiTkpJiypYtazp06GB+//138+eff5rnnnvOlC5d2iHAXEtsbKzx8/MzrVq1Mn/88YdZsGCBKViwoPm///s/q02PHj1MeHi4+eGHH8y2bdtMbGysyZ8/vzl58mS29V09L5cvXzZz5swxksyOHTvM4cOHTUJCgjHGcZv4448/jCSHE8r0brt27TLGGDNjxgxTqFAhM2fOHLNnzx4zZ84cExwcbKZOnXoTa+vG3In7b/Xq1c2yZcvMtm3bTJ06dUzNmjWtNr/++qsJCAgwU6dONfHx8WbRokUmMjLSDB061BhjzOXLl03p0qVNgwYNzObNm83y5ctNtWrVMoWfESNGmJUrV5q9e/ea+fPnm9DQUPP2228bY4w5f/68ee2110z58uWtZXL+/HljjGP4+fDDD02xYsVMWlqaNd4JEyY4dOvUqZOpWbOm+fXXX83u3bvNu+++a7y8vMzOnTtzvI5u1N2w/wYEBJh///vfZvv27ea7774z+fLlM5MnT7baZLf8VqxYYVxdXc27775rduzYYT766CMTHBzscPzdvHmzmTRpktm6davZuXOnGThwoPH29jb79+83xtz7x/Hs1K1b1wQEBJihQ4eanTt3mmnTphkXFxezaNEiY4wxY8eONUuXLjV79+41S5YsMaVLlzZdunTJ0binTJliPDw8TNWqVc2qVavMhg0bTLVq1XK1Tx87dsw8/vjj5tlnn7WOu6mpqaZSpUqmdu3aZsOGDWbNmjWmSpUqDseiIUOGGF9fX/P444+bjRs3mi1bthhjjImJiTFNmzY169evNzt37jSvvfaaKVCggPWZYDd5uf7//vtv8+6775pNmzaZ+Ph4M378eOPm5mbWrl1rjMn6GHXs2DHrC6f0bsZkv50Yc+X4HBISYj777DMTHx9v7eN2R/jJI1ef/Bnz/5/EzJs3z6FddidPFy9eNPny5TOrVq1yaNOxY0fTpk2bbOtIn+7s2bOtbidPnjQ+Pj7miy++MMYY89xzz5kGDRo4DNe3b19Trlw5Y4wxc+bMMQEBASYpKSlH8xoREWHGjh3r0Cbjh+bYsWNNyZIlrfcZv0UcMWKEadiwocM4Dh48aJ1oO9PYsWMdvnnJ6Pjx40aSdXUt48lT06ZNTfv27bMc9vPPPzelS5d2OHlMTk42Pj4+5qeffsq2ttjYWBMcHGzOnTtndZs4caLx8/Mzqamp5uzZs8bDw8PMnDnT6p+SkmLCw8PNO++8k219Geclffs6ffq0Q7uM28SDDz5ohg8fbr0fMGCAqV69uvW+ZMmSDt9+GXNlG4iOjs52nvPCnbb/Ll682Or2/fffG0nmwoULxhhj6tevb9566y2H4T7//HNTqFAhY4wxP/74o3F3d7dOYI0xWV75yejdd981VapUsd6nf2uc0dXjSb/K8+uvv1r9o6Ojzeuvv26MufLNtJubm/nnn38cxlG/fn0zYMCA6yyJW+dO338jIiLM5cuXrW4tW7Y0rVq1MsbkbPm1atXKNGnSxKF/27ZtHY6/WSlfvryZMGGC9f5eP45fT926dU3t2rUduj388MPWdpzRV199ZQoUKJCjcadfRV2zZo3Vbfv27UaSdQKc3T5tjDHNmjWzrvgYY8yiRYuMm5ubwxWkbdu2GUnWlf0hQ4YYDw8P6+TZGGOWL19uAgICzMWLFx2mV7JkSfPJJ5/kaJ7uNXm5/rPSpEkT89prr1nvszpGZXW8zsl2Isn07Nnzhmu7V/HAg9usatWquWq/e/dunT9/Xg0aNJCfn5/1mj59uuLj43M8nujoaOvv4OBglS5dWtu3b5ckbd++XbVq1XJoX6tWLe3atUupqalq0KCBIiIiVKJECb3wwguaOXOmzp8/n6v5yKh169bat2+f1qxZI0maOXOmKleurDJlykiStmzZop9//tlhntP75Wa+b4ddu3apTZs2KlGihAICAhQZGSlJOnDgQJbtu3TpotmzZ6tSpUrq16+fVq1aZfXbsmWLdu/eLX9/f2u+g4ODdfHixRzP94MPPqh8+fJZ76Ojo3X27FkdPHhQ8fHxunTpksP69vDwULVq1azt4Xr13ai2bdtq1qxZkq7cL/O///1Pbdu2lSSdO3dO8fHx6tixo8P6Hjly5B23rp21/1asWNH6u1ChQpKkY8eOSbqyzQwfPtxh/J07d9bhw4d1/vx57dixQ0WLFnW4N6datWqZpvHFF1+oVq1aCgsLk5+fnwYOHHjNbfhaChYsqIYNG2rmzJmSpL1792r16tXWut66datSU1P1wAMPONT7yy+/OG1d32n7b/ny5eXm5ma9L1SokLWuc7L8duzYkWn9Znx/9uxZ9enTR2XLllVQUJD8/Py0ffv2XK/ve+k4ntHV+5zkuB4WL16s+vXrq3DhwvL399cLL7ygkydP5vhz0d3dXQ8//LD1vkyZMgoKCrKOwdnt01nZvn27ihYtqqJFi1rdypUr5zBeSYqIiFDBggWt91u2bNHZs2dVoEABh+nt3bv3jl9HeSmv1n9qaqpGjBihqKgoBQcHy8/PTz/99FOu9z0p59tJbj+37IC7dW8zX19fh/eurq4yxjh0u3TpkvX32bNnJUnff/+9Chcu7NDOy8srj6p05O/vr40bN2rZsmVatGiRBg8erKFDh2r9+vU3/IjNsLAwPfbYY5o1a5Zq1KihWbNmqUuXLlb/s2fPqmnTpnr77bczDZt+8nenaNq0qSIiIvTpp58qPDxcaWlpqlChglJSUrJs37hxY+3fv18//PCD4uLiVL9+fXXt2lXvvfeezp49qypVqlgnj1e7+gMrL12vvhvVpk0bvf7669q4caMuXLiggwcPqlWrVpL+/238008/VfXq1R2Gu/ok8E7grP336htUXVxcJElpaWnWNIYNG6ann34603De3t45Gn96QBk2bJgaNWqkwMBAzZ49W2PGjMlxjenatm2rHj16aMKECZo1a5aioqIUFRVl1erm5qbffvst07r18/PL9bRuhTtt/814M7KLi4vDur4Vy69Pnz6Ki4vTe++9p/vvv18+Pj565plnrjnP13IvHcczutZ62Ldvn5544gl16dJFb775poKDg7VixQp17NhRKSkpDl883ahbsU9fS8Zj2NmzZ1WoUCEtW7YsU1s7P0I7r9b/u+++qw8++EDjxo1TVFSUfH191bNnz1zve1LOt5OM6xyEnzzj6emp1NTUbNsVLFhQf/zxh0O3zZs3WzteuXLl5OXlpQMHDqhu3bo3XM+aNWtUrFgxSdLp06e1c+dOlS1bVpJUtmxZrVy50qH9ypUr9cADD1gfsO7u7oqJiVFMTIyGDBmioKAgLV26NMudLqfz3rZtW/Xr109t2rTRnj171Lp1a6tf5cqVNWfOHEVGRt7RT9Q6efKkduzYoU8//VR16tSRJK1YsSLb4QoWLKjY2FjFxsaqTp066tu3r9577z1VrlxZX3zxhUJCQhQQEHBDNW3ZskUXLlywnsq0Zs0a+fn5qWjRorrvvvvk6emplStXKiIiQtKVk/X169c7PNr5WvVllP6koOzWd5EiRVS3bl3NnDlTFy5cUIMGDRQSEiJJCg0NVXh4uPbs2WNdIXC2O23/vZ7KlStrx44duv/++7PsX7p0aR08eFBHjx5VaGioJGn9+vUObVatWqWIiAi98cYbVrf9+/c7tMnpMmnWrJleeuklLVy4ULNmzdKLL75o9XvooYeUmpqqY8eOWfuLM92J++/15GT5lS5dOtP6zfh+5cqVateunZ566ilJV06i9u3b59DGTsfx3Pjtt9+UlpamMWPGyNX1yo9nvvzyy1yN4/Lly9qwYYN1RW7Hjh1KSEiwPpOz26ezUrZsWR08eFAHDx60rv78+eefSkhIULly5a45XOXKlXXkyBG5u7tbVz1xbTe7/leuXKlmzZrp+eefl3TlS6ydO3dedx1JV8JYxv3xRrYTXMHP3vJIZGSk1q5dq3379unEiRPWN3cZPfbYY9qwYYOmT5+uXbt2aciQIQ4nU/7+/urTp4969eqladOmKT4+Xhs3btSECRM0bdq0HNczfPhwLVmyRH/88YfatWun++67T82bN5ckvfbaa1qyZIlGjBihnTt3atq0afrwww/Vp08fSdKCBQs0fvx4bd68Wfv379f06dOVlpam0qVLX3Pef/31V/3zzz86ceLENWt6+umndebMGXXp0kX16tVTeHi41a9r1646deqU2rRpo/Xr1ys+Pl4//fST2rdvn6MP5Nslf/78KlCggCZPnqzdu3dr6dKl6t2793WHGTx4sL799lvt3r1b27Zt04IFC6wPvbZt2+q+++5Ts2bNtHz5cu3du1fLli1Tjx499Pfff+eoppSUFHXs2FF//vmnfvjhBw0ZMkTdunWTq6urfH191aVLF/Xt21cLFy7Un3/+qc6dO+v8+fPq2LFjtvVlFBERIRcXFy1YsEDHjx+3rnRkpW3btpo9e7a++uqrTCFn2LBhGjVqlMaPH6+dO3dq69atmjJlit5///0czfOtdqftv9czePBgTZ8+XcOGDdO2bdu0fft2zZ49WwMHDpQkNWjQQCVLllRsbKx+//13rVy50uqXfhWpVKlSOnDggGbPnq34+HiNHz9e33zzTaZlsnfvXm3evFknTpxQcnJylvX4+vqqefPmGjRokLZv3642bdpY/R544AG1bdtWL774oubOnau9e/dq3bp1GjVqlL7//vtbsjxy407cf68nJ8uve/fu+uGHH/T+++9r165d+uSTT/Tjjz9a61q6sr7nzp2rzZs3a8uWLXruuecybeN2Oo7nxv33369Lly5pwoQJ2rNnjz7//HNNmjQpV+Pw8PBQ9+7dtXbtWv32229q166datSoYYWh7PbprMTExCgqKkpt27bVxo0btW7dOr344ouqW7fudX/2FBMTo+joaDVv3lyLFi3Svn37tGrVKr3xxhvasGFDrubLDm52/ZcqVUpxcXFatWqVtm/frpdffllHjx7NdrjIyEgtWbJER44c0enTpyXd2HaC/8fJ9xzds3bs2GFq1KhhfHx8HB6Vm/HGcGOMGTx4sAkNDTWBgYGmV69eplu3bg5PaElLSzPjxo0zpUuXNh4eHqZgwYKmUaNG1qNjryf9hunvvvvOlC9f3nh6eppq1apZT3lJl/6oaw8PD1OsWDHz7rvvWv2WL19u6tata/Lnz298fHxMxYoVrYclGJP55vDVq1ebihUrGi8vrywfkXq1Z5991khyeFRsup07d5qnnnrKBAUFGR8fH1OmTBnTs2dPh5uJnSHjzYhxcXGmbNmyxsvLy1SsWNEsW7bM4ebEjDdMjxgxwpQtW9b4+PiY4OBg06xZM7Nnzx5rfIcPHzYvvviiue+++4yXl5cpUaKE6dy5s0lMTMy2tvQb8AcPHmwKFChg/Pz8TOfOnR1uZr1w4YLp3r27Nf6Mj7q+Xn0Z58UYY4YPH27CwsKMi4tLlo+6Tnf69Gnj5eVl8uXLl+VjQWfOnGkqVapkPD09Tf78+c0jjzxi5s6dm+0854U7bf+9erqbNm0ykhwen7xw4UJTs2ZN4+PjYwICAky1atUcnhCW/qhrT09PU6ZMGfPdd98ZSWbhwoVWm759+1rbTKtWrczYsWMd9tmLFy+aFi1amKCgoGs+6jrdDz/8YCSZRx55JNM8paSkmMGDB5vIyEjj4eFhChUqZJ566inz+++/Z7s8boW7Yf+9WsanB+Zk+U2ePNkULlzYetT1yJEjTVhYmNV/7969pl69esbHx8cULVrUfPjhh7Y7jl9PVsevqx8w8P7775tChQoZHx8f06hRIzN9+vRrHh8ySl+Gc+bMMSVKlDBeXl4mJiYm01O4stunMz7wwJicP+o6o6SkJNO9e3cTHh5uPDw8TNGiRU3btm0zPX7bLvJy/Z88edI0a9bM+Pn5mZCQEDNw4EDz4osvOuz3WT3wYP78+eb+++837u7uDv2y206yOj7DGBdjMvxgHfeUZcuWqV69ejp9+rStf79rF+3atVNCQgL/CRrXtXLlStWuXVu7d+9WyZIlnV0O8ljnzp31119/afny5c4uBQCc7t74ES4A4Jq++eYb+fn5qVSpUtq9e7deffVV1apVi+Bzj3rvvffUoEED+fr66scff9S0adP08ccfO7ssALgjcM/PXe7f//63w2MOr379+9//dnZ5uMWuta79/Pz4VvcudLv23zNnzqhr164qU6aM2rVrp4cffljffvvtLRs/cuZ27b/r1q1TgwYNFBUVpUmTJmn8+PHq1KnTLRs/rq1x48bXXMdvvfWWs8tDHmP93x342dtd7tixY0pKSsqyX0BAgPVELdwbdu/efc1+hQsXtp7whrsD+6+9sP/e+/755x9duHAhy37BwcEKDg6+zRXhdmL93x0IPwAAAABsgZ+9AQAAALAFwg8AAAAAWyD8AAAAALAFwg8AAAAAWyD8AAAAALAFwg8AAAAAWyD8AAAAALAFwg8AAAAAW/j/AEzLc5gfDvdFAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "values = [tp, fp, tn, fn, na_b, na_a]\n",
    "labels = ['true_positive', 'false_positive', 'true_negative', 'false_negative', 'na_before', 'na_after']\n",
    "\n",
    "plt.figure(figsize=(10, 6))\n",
    "bars = plt.bar(labels, values, color=plt.get_cmap('tab10').colors)\n",
    "\n",
    "plt.title('D1_llama3 with response extraction pipeline with Llama3')\n",
    "\n",
    "for bar in bars:\n",
    "    yval = bar.get_height()\n",
    "    plt.text(bar.get_x() + bar.get_width()/2, yval + 1, int(yval), ha='center', va='bottom')\n",
    "\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a462163d-a893-4d84-882a-e314f4d247ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Experiment | TP | FP | TN | FN | NA_B | NA_A\n",
      "|------------|----|----|----|----|----|----|\n",
      "| D1_llama3 | 912 | 914 | 7 | 12 |76 |79 |\n",
      "| D2_llama3 | 2 | 2 | 162 | 108 |890 |836 |\n",
      "| R1_llama3 | 333 | 319 | 258 | 256 |411 |423 |\n",
      "| R2_llama3 | 271 | 199 | 525 | 452 |277 |276 |\n",
      "| S1_llama3 | 689 | 720 | 210 | 225 |86 |70 |\n",
      "| S2_llama3 | 650 | 692 | 146 | 164 |186 |162 |\n",
      "| S3_llama3 | 713 | 702 | 150 | 153 |134 |148 |\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "table = \"| Experiment | TP | FP | TN | FN | NA_B | NA_A\\n\"\n",
    "table += \"|------------|----|----|----|----|----|----|\\n\"\n",
    "\n",
    "csv_files = [file for file in os.listdir('kat_res/') if file.endswith('.csv') and 'new' in file]\n",
    "\n",
    "csv_files.sort()\n",
    "\n",
    "for csv_file in csv_files:\n",
    "    experiment_name = csv_file.split('new_res_')[1].split('.csv')[0]\n",
    "    \n",
    "    df = pd.read_csv(f\"kat_res/{csv_file}\")\n",
    "    tp, fp, tn, fn, na_b, na_a = calculate_metrics(df, 'pred_before', 'pred_after')\n",
    "    table += f\"| {experiment_name} | {tp} | {fp} | {tn} | {fn} |{na_b} |{na_a} |\\n\"\n",
    "\n",
    "print(table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "b082eb80-46dc-42b9-be36-23a154e30200",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['res_D1_artigenz.csv',\n",
       " 'res_D1_codeQwen.csv',\n",
       " 'res_D1_deepSeekCoder.csv',\n",
       " 'res_D1_llama3.csv',\n",
       " 'res_D2_artigenz.csv',\n",
       " 'res_D2_codeQwen.csv',\n",
       " 'res_D2_deepSeekCoder.csv',\n",
       " 'res_D2_llama3.csv',\n",
       " 'res_R1_artigenz.csv',\n",
       " 'res_R1_codeQwen.csv',\n",
       " 'res_R1_deepSeekCoder.csv',\n",
       " 'res_R1_llama3.csv',\n",
       " 'res_R2_artigenz.csv',\n",
       " 'res_R2_codeQwen.csv',\n",
       " 'res_R2_deepSeekCoder.csv',\n",
       " 'res_R2_llama3.csv',\n",
       " 'res_S1_artigenz.csv',\n",
       " 'res_S1_codeQwen.csv',\n",
       " 'res_S1_deepSeekCoder.csv',\n",
       " 'res_S1_llama3.csv',\n",
       " 'res_S2_artigenz.csv',\n",
       " 'res_S2_codeQwen.csv',\n",
       " 'res_S2_deepSeekCoder.csv',\n",
       " 'res_S2_llama3.csv',\n",
       " 'res_S3_artigenz.csv',\n",
       " 'res_S3_codeQwen.csv',\n",
       " 'res_S3_deepSeekCoder.csv',\n",
       " 'res_S3_llama3.csv']"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "csv_files = [file for file in os.listdir('kat_res/') if file.endswith('.csv') and 'new' not in file]\n",
    "csv_files.sort()\n",
    "csv_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "619eb9ed-6151-4310-8952-3509cc3dcfe1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>res_before</th>\n",
       "      <th>res_after</th>\n",
       "      <th>pred_before</th>\n",
       "      <th>pred_after</th>\n",
       "      <th>res_before_1</th>\n",
       "      <th>res_after_1</th>\n",
       "      <th>pred_before_1</th>\n",
       "      <th>pred_after_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Yes, this code does contain an out-of-bounds r...</td>\n",
       "      <td>Yes, the code does contain instances of the Ou...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Yes, this code does contain an out-of-bounds r...</td>\n",
       "      <td>Yes, the code does contain instances of the se...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Yes, this code contains instances of the secur...</td>\n",
       "      <td>Yes, this code does contain instances of the s...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Yes, it does. This code is vulnerable to a com...</td>\n",
       "      <td>Yes, this code does contain instances of the s...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Yes, this code does contain an instance of the...</td>\n",
       "      <td>Yes, this code does contain instances of the O...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                         res_before  \\\n",
       "0   0  Yes, this code does contain an out-of-bounds r...   \n",
       "1   1  Yes, this code does contain an out-of-bounds r...   \n",
       "2   2  Yes, this code contains instances of the secur...   \n",
       "3   3  Yes, it does. This code is vulnerable to a com...   \n",
       "4   4  Yes, this code does contain an instance of the...   \n",
       "\n",
       "                                           res_after  pred_before  pred_after  \\\n",
       "0  Yes, the code does contain instances of the Ou...            1           1   \n",
       "1  Yes, the code does contain instances of the se...            1           1   \n",
       "2  Yes, this code does contain instances of the s...            1           1   \n",
       "3  Yes, this code does contain instances of the s...            1           1   \n",
       "4  Yes, this code does contain instances of the O...            1           1   \n",
       "\n",
       "  res_before_1 res_after_1  pred_before_1  pred_after_1  \n",
       "0          NaN         NaN             -1            -1  \n",
       "1          NaN         NaN             -1            -1  \n",
       "2          NaN         NaN             -1            -1  \n",
       "3          NaN         NaN             -1            -1  \n",
       "4          NaN         NaN             -1            -1  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(f'kat_res/{csv_files[0]}')\n",
    "df.head()\n",
    "df.drop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "a323ac60-4094-4ab3-b0bc-aceb9fb87320",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['pred_before'].iloc[0] == 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "6ed6bc4a-4656-463d-9c26-f504807b58c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import pandas as pd\n",
    "\n",
    "def get_prediction(text):\n",
    "    text = str(text).lower().split()\n",
    "    text = [re.sub(r'\\W+', '', word) for word in text]\n",
    "    \n",
    "    if text[0] == 'yes':\n",
    "        return 1\n",
    "    elif text[0] == 'no':\n",
    "        return 0\n",
    "    # elif 'yes' in text and 'no' in text:\n",
    "    #     return -1\n",
    "    # elif 'yes' in text:\n",
    "    #     return 1\n",
    "    # elif 'no' in text:\n",
    "    #     return 0\n",
    "    else:\n",
    "        return -1\n",
    "\n",
    "for csv_file in csv_files:\n",
    "    df = pd.read_csv(f\"kat_res/{csv_file}\")\n",
    "    pred_before = []\n",
    "    pred_after = []\n",
    "    \n",
    "    for i in range(df.shape[0]):\n",
    "        if df['pred_before'].iloc[i] == 1:\n",
    "            pred_before.append(1)\n",
    "        elif df['pred_before'].iloc[i] == 0:\n",
    "            pred_before.append(0)\n",
    "        else:\n",
    "            pred_before.append(get_prediction(df['res_before_1'].iloc[i]))\n",
    "\n",
    "        if df['pred_after'].iloc[i] == 1:\n",
    "            pred_after.append(1)\n",
    "        elif df['pred_after'].iloc[i] == 0:\n",
    "            pred_after.append(0)\n",
    "        else:\n",
    "            pred_after.append(get_prediction(df['res_after_1'].iloc[i]))\n",
    "        \n",
    "    df['pred_before_1'] = pred_before\n",
    "    df['pred_after_1'] = pred_after\n",
    "\n",
    "    df.to_csv(f\"kat_res/{csv_file}\", index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "81e964b5-a727-4fc3-a4e9-936619cc3836",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Experiment | TP | FP | TN | FN | NA_B | NA_A\n",
      "|------------|----|----|----|----|----|----|\n",
      "| D1_artigenz | 973 | 977 | 23 | 27 |0 |0 |\n",
      "| D1_codeQwen | 927 | 929 | 70 | 73 |0 |1 |\n",
      "| D1_deepSeekCoder | 915 | 912 | 88 | 85 |0 |0 |\n",
      "| D1_llama3 | 988 | 992 | 8 | 12 |0 |0 |\n",
      "| D2_artigenz | 860 | 818 | 179 | 138 |2 |3 |\n",
      "| D2_codeQwen | 831 | 842 | 154 | 169 |0 |4 |\n",
      "| D2_deepSeekCoder | 753 | 704 | 295 | 242 |5 |1 |\n",
      "| D2_llama3 | 949 | 891 | 107 | 50 |1 |2 |\n",
      "| R1_artigenz | 695 | 702 | 298 | 305 |0 |0 |\n",
      "| R1_codeQwen | 605 | 578 | 422 | 395 |0 |0 |\n",
      "| R1_deepSeekCoder | 609 | 604 | 396 | 391 |0 |0 |\n",
      "| R1_llama3 | 727 | 720 | 279 | 273 |0 |1 |\n",
      "| R2_artigenz | 573 | 550 | 447 | 426 |1 |3 |\n",
      "| R2_codeQwen | 604 | 573 | 427 | 396 |0 |0 |\n",
      "| R2_deepSeekCoder | 504 | 487 | 511 | 495 |1 |2 |\n",
      "| R2_llama3 | 629 | 472 | 528 | 371 |0 |0 |\n",
      "| S1_artigenz | 751 | 740 | 260 | 249 |0 |0 |\n",
      "| S1_codeQwen | 629 | 600 | 397 | 369 |2 |3 |\n",
      "| S1_deepSeekCoder | 337 | 328 | 672 | 661 |2 |0 |\n",
      "| S1_llama3 | 772 | 787 | 213 | 228 |0 |0 |\n",
      "| S2_artigenz | 998 | 995 | 5 | 2 |0 |0 |\n",
      "| S2_codeQwen | 60 | 59 | 941 | 940 |0 |0 |\n",
      "| S2_deepSeekCoder | 358 | 382 | 618 | 642 |0 |0 |\n",
      "| S2_llama3 | 978 | 974 | 26 | 22 |0 |0 |\n",
      "| S3_artigenz | 990 | 988 | 12 | 10 |0 |0 |\n",
      "| S3_codeQwen | 51 | 62 | 938 | 949 |0 |0 |\n",
      "| S3_deepSeekCoder | 463 | 483 | 517 | 535 |2 |0 |\n",
      "| S3_llama3 | 947 | 947 | 53 | 53 |0 |0 |\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from utils import *\n",
    "\n",
    "table = \"| Experiment | TP | FP | TN | FN | NA_B | NA_A\\n\"\n",
    "table += \"|------------|----|----|----|----|----|----|\\n\"\n",
    "\n",
    "# csv_files = [file for file in os.listdir('kat_res/') if file.endswith('.csv') and 'new' in file]\n",
    "\n",
    "# csv_files.sort()\n",
    "# print(csv_files)\n",
    "\n",
    "for csv_file in csv_files:\n",
    "    experiment_name = csv_file.split('res_')[1].split('.csv')[0]\n",
    "    \n",
    "    df = pd.read_csv(f\"kat_res/{csv_file}\")\n",
    "    tp, fp, tn, fn, na_b, na_a = calculate_metrics(df, 'pred_before_1', 'pred_after_1')\n",
    "    table += f\"| {experiment_name} | {tp} | {fp} | {tn} | {fn} |{na_b} |{na_a} |\\n\"\n",
    "\n",
    "print(table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "3b0beb3b-0c89-46bd-aa9d-7c57fc877887",
   "metadata": {},
   "outputs": [],
   "source": [
    "def classification_metrics(tp, fp, tn, fn):\n",
    "    accuracy = (tp + tn) / (tp + fp + tn + fn) if (tp + fp + tn + fn) > 0 else 0\n",
    "    precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n",
    "    recall = tp / (tp + fn) if (tp + fn) > 0 else 0\n",
    "    f1_score = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0\n",
    "    \n",
    "    return {\n",
    "        \"accuracy\": accuracy,\n",
    "        \"precision\": precision,\n",
    "        \"recall\": recall,\n",
    "        \"f1_score\": f1_score\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "69d3546a-054e-4532-be33-03a108269064",
   "metadata": {},
   "outputs": [],
   "source": [
    "table = \"| Experiment | Accuracy | Precision | Recall | F1 score\\n\"\n",
    "table += \"|------------|----|----|----|----|\\n\"\n",
    "\n",
    "# csv_files = [file for file in os.listdir('kat_res/') if file.endswith('.csv') and 'new' in file]\n",
    "\n",
    "# csv_files.sort()\n",
    "# print(csv_files)\n",
    "exp_name = []\n",
    "acc = []\n",
    "pre = []\n",
    "rec = []\n",
    "f1s = []\n",
    "\n",
    "for csv_file in csv_files:\n",
    "    experiment_name = csv_file.split('res_')[1].split('.csv')[0]\n",
    "    exp_name.append(experiment_name)\n",
    "    \n",
    "    df = pd.read_csv(f\"kat_res/{csv_file}\")\n",
    "    tp, fp, tn, fn, na_b, na_a = calculate_metrics(df, 'pred_before_1', 'pred_after_1')\n",
    "    res = classification_metrics(tp, fp, tn, fn)\n",
    "    acc.append(res['accuracy'])\n",
    "    pre.append(res['precision'])\n",
    "    rec.append(res['recall'])\n",
    "    f1s.append(res['f1_score'])\n",
    "    \n",
    "    table += f\"| {experiment_name} | {round(res['accuracy'], 3)} | {round(res['precision'], 3)} | {round(res['recall'], 3)} | {round(res['f1_score'], 3)} |\\n\"\n",
    "\n",
    "df = pd.DataFrame({'experiment_name': pd.Series(exp_name), 'accuracy': pd.Series(acc), 'precision': pd.Series(pre), 'recall': pd.Series(rec), 'f1_score': pd.Series(f1s)})\n",
    "df.to_csv('classification_metric.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "2d9787c4-c962-49b5-8056-2c633becf235",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28\n"
     ]
    }
   ],
   "source": [
    "exp_name = []\n",
    "acc_tp = []\n",
    "acc_fp = []\n",
    "acc_tn = []\n",
    "acc_fn = []\n",
    "acc_na_b = []\n",
    "acc_na_a = []\n",
    "acc = []\n",
    "pre = []\n",
    "rec = []\n",
    "f1s = []\n",
    "\n",
    "rec_df = pd.read_csv('classification_metric.csv')\n",
    "\n",
    "id_name='overall'\n",
    "\n",
    "for csv_file in csv_files:\n",
    "    experiment_name = csv_file.split('res_')[1].split('.csv')[0]\n",
    "    exp_name.append(experiment_name)\n",
    "    \n",
    "    df = pd.read_csv(f\"kat_res/{csv_file}\")\n",
    "    sub_df = df\n",
    "    tp, fp, tn, fn, na_b, na_a = calculate_metrics(sub_df, 'pred_before_1', 'pred_after_1')\n",
    "    \n",
    "    res = classification_metrics(tp, fp, tn, fn)\n",
    "\n",
    "    acc_tp.append(tp)\n",
    "    acc_fp.append(fp)\n",
    "    acc_tn.append(tn)\n",
    "    acc_fn.append(fn)\n",
    "    acc_na_b.append(na_b)\n",
    "    acc_na_a.append(na_a)\n",
    "    \n",
    "    acc.append(res['accuracy'])\n",
    "    pre.append(res['precision'])\n",
    "    rec.append(res['recall'])\n",
    "    f1s.append(res['f1_score'])\n",
    "\n",
    "print(len(acc_tp))\n",
    "rec_df[f'{id_name}_tp'] = acc_tp\n",
    "rec_df[f'{id_name}_tn'] = acc_tn\n",
    "rec_df[f'{id_name}_fp'] = acc_fp\n",
    "rec_df[f'{id_name}_fn'] = acc_fn\n",
    "rec_df[f'{id_name}_na_b'] = acc_na_b\n",
    "rec_df[f'{id_name}_na_a'] = acc_na_a\n",
    "\n",
    "rec_df[f'{id_name}_accuracy'] = acc\n",
    "rec_df[f'{id_name}_precision'] = pre\n",
    "rec_df[f'{id_name}_recall'] = rec\n",
    "rec_df[f'{id_name}_f1_score'] = f1s\n",
    "\n",
    "rec_df.to_csv('classification_metric.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "258597bb-3ad8-4a07-b7e1-8852852452b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| Experiment | TP | FP | TN | FN | NA_B| NA_A | Accuracy | Precision | Recall | F1 score |\n",
      "|------------|----|----|----|----|----|----|----|----|----|----|\n",
      "| D1_artigenz | 973 | 977 | 23 | 27 | 0 | 0 | 0.498 |0.499 |0.973 |0.66 |\n",
      "| D1_codeQwen | 927 | 929 | 70 | 73 | 0 | 1 | 0.499 |0.499 |0.927 |0.649 |\n",
      "| D1_deepSeekCoder | 915 | 912 | 88 | 85 | 0 | 0 | 0.501 |0.501 |0.915 |0.647 |\n",
      "| D1_llama3 | 988 | 992 | 8 | 12 | 0 | 0 | 0.498 |0.499 |0.988 |0.663 |\n",
      "| D2_artigenz | 860 | 818 | 179 | 138 | 2 | 3 | 0.521 |0.513 |0.862 |0.643 |\n",
      "| D2_codeQwen | 831 | 842 | 154 | 169 | 0 | 4 | 0.493 |0.497 |0.831 |0.622 |\n",
      "| D2_deepSeekCoder | 753 | 704 | 295 | 242 | 5 | 1 | 0.526 |0.517 |0.757 |0.614 |\n",
      "| D2_llama3 | 949 | 891 | 107 | 50 | 1 | 2 | 0.529 |0.516 |0.95 |0.669 |\n",
      "| R1_artigenz | 695 | 702 | 298 | 305 | 0 | 0 | 0.496 |0.497 |0.695 |0.58 |\n",
      "| R1_codeQwen | 605 | 578 | 422 | 395 | 0 | 0 | 0.514 |0.511 |0.605 |0.554 |\n",
      "| R1_deepSeekCoder | 609 | 604 | 396 | 391 | 0 | 0 | 0.502 |0.502 |0.609 |0.55 |\n",
      "| R1_llama3 | 727 | 720 | 279 | 273 | 0 | 1 | 0.503 |0.502 |0.727 |0.594 |\n",
      "| R2_artigenz | 573 | 550 | 447 | 426 | 1 | 3 | 0.511 |0.51 |0.574 |0.54 |\n",
      "| R2_codeQwen | 604 | 573 | 427 | 396 | 0 | 0 | 0.516 |0.513 |0.604 |0.555 |\n",
      "| R2_deepSeekCoder | 504 | 487 | 511 | 495 | 1 | 2 | 0.508 |0.509 |0.505 |0.507 |\n",
      "| R2_llama3 | 629 | 472 | 528 | 371 | 0 | 0 | 0.578 |0.571 |0.629 |0.599 |\n",
      "| S1_artigenz | 751 | 740 | 260 | 249 | 0 | 0 | 0.505 |0.504 |0.751 |0.603 |\n",
      "| S1_codeQwen | 629 | 600 | 397 | 369 | 2 | 3 | 0.514 |0.512 |0.63 |0.565 |\n",
      "| S1_deepSeekCoder | 337 | 328 | 672 | 661 | 2 | 0 | 0.505 |0.507 |0.338 |0.405 |\n",
      "| S1_llama3 | 772 | 787 | 213 | 228 | 0 | 0 | 0.492 |0.495 |0.772 |0.603 |\n",
      "| S2_artigenz | 998 | 995 | 5 | 2 | 0 | 0 | 0.501 |0.501 |0.998 |0.667 |\n",
      "| S2_codeQwen | 60 | 59 | 941 | 940 | 0 | 0 | 0.5 |0.504 |0.06 |0.107 |\n",
      "| S2_deepSeekCoder | 358 | 382 | 618 | 642 | 0 | 0 | 0.488 |0.484 |0.358 |0.411 |\n",
      "| S2_llama3 | 978 | 974 | 26 | 22 | 0 | 0 | 0.502 |0.501 |0.978 |0.663 |\n",
      "| S3_artigenz | 990 | 988 | 12 | 10 | 0 | 0 | 0.501 |0.501 |0.99 |0.665 |\n",
      "| S3_codeQwen | 51 | 62 | 938 | 949 | 0 | 0 | 0.494 |0.451 |0.051 |0.092 |\n",
      "| S3_deepSeekCoder | 463 | 483 | 517 | 535 | 2 | 0 | 0.49 |0.489 |0.464 |0.476 |\n",
      "| S3_llama3 | 947 | 947 | 53 | 53 | 0 | 0 | 0.5 |0.5 |0.947 |0.654 |\n",
      "\n"
     ]
    }
   ],
   "source": [
    "table = \"| Experiment | TP | FP | TN | FN | NA_B| NA_A | Accuracy | Precision | Recall | F1 score |\\n\"\n",
    "table += \"|------------|----|----|----|----|----|----|----|----|----|----|\\n\"\n",
    "\n",
    "rec_df = pd.read_csv('classification_metric.csv')\n",
    "\n",
    "id_name = 'overall'\n",
    "\n",
    "for i in range(rec_df.shape[0]):    \n",
    "    table += f\"| {rec_df['experiment_name'].iloc[i]} | {rec_df[f'{id_name}_tp'].iloc[i]} | {rec_df[f'{id_name}_fp'].iloc[i]} | {rec_df[f'{id_name}_tn'].iloc[i]} | {rec_df[f'{id_name}_fn'].iloc[i]} | {rec_df[f'{id_name}_na_b'].iloc[i]} | {rec_df[f'{id_name}_na_a'].iloc[i]} | {round(rec_df[f'{id_name}_accuracy'].iloc[i], 3)} |{round(rec_df[f'{id_name}_precision'].iloc[i], 3)} |{round(rec_df[f'{id_name}_recall'].iloc[i], 3)} |{round(rec_df[f'{id_name}_f1_score'].iloc[i], 3)} |\\n\"\n",
    "\n",
    "print(table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c98fed1-bc94-442c-b45a-e77fb36b812c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26c89cff-2896-45cd-a14d-e2e3522606a6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dce5cec1-a08a-4649-9007-008a7889899e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/local/QCRI/ejeong/miniconda3/envs/vllm/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "2024-05-28 13:42:35,071\tINFO util.py:154 -- Missing packages: ['ipywidgets']. Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.\n",
      "  2%|▏         | 21/1000 [00:35<39:07,  2.40s/it]"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from utils import *\n",
    "from evaluation import *\n",
    "\n",
    "table = \"| Experiment | TP | FP | TN | FN | NA_B | NA_A\\n\"\n",
    "table += \"|------------|----|----|----|----|----|----|\\n\"\n",
    "\n",
    "csv_files = [file for file in os.listdir('kat_res/') if file.endswith('.csv') and 'new' not in file]\n",
    "\n",
    "csv_files.sort()\n",
    "print(csv_files)\n",
    "\n",
    "for csv_file in csv_files:\n",
    "    experiment_name = csv_file.split('res_')[1].split('.csv')[0]\n",
    "    \n",
    "    df = pd.read_csv(f\"kat_res/{csv_file}\")\n",
    "    tp, fp, tn, fn, na_b, na_a = calculate_metrics(df, 'pred_before', 'pred_after')\n",
    "    table += f\"| {experiment_name} | {tp} | {fp} | {tn} | {fn} |{na_b} |{na_a} |\\n\"\n",
    "\n",
    "print(table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d36e96a8-7591-48df-987d-a1568268c4a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from utils import *\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "import re\n",
    "\n",
    "\n",
    "csv_files = [file for file in os.listdir('kat_res/') if file.endswith('.csv') and 'new' not in file]\n",
    "csv_files.sort()\n",
    "\n",
    "invalid_res_before = []\n",
    "invalid_res_after = []\n",
    "\n",
    "\n",
    "for csv_file in csv_files:\n",
    "    df = pd.read_csv(f\"kat_res/{csv_file}\")\n",
    "\n",
    "    filtered_res_before = df.loc[df['pred_before'] == -1, 'res_before']\n",
    "    invalid_res_before.extend(filtered_res_before.tolist())\n",
    "\n",
    "    filtered_res_after = df.loc[df['pred_after'] == -1, 'res_after']\n",
    "    invalid_res_after.extend(filtered_res_after.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c7903b96-29d7-4de7-8e8f-14d13c10145c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8455, 8250)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(invalid_res_before), len(invalid_res_after)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "45f9c6db-1a4c-4e46-822c-d806b34caba6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(f'prompt_0.csv')\n",
    "prompts = df['prompt'].dropna().tolist()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "3630dc49-da7e-4ec2-ac08-721f79b538d3",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4176"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(prompts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d4e32f1-b55c-4da7-be9f-f125260b7811",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}