main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "b89a2e69-8754-4f93-84d8-3e3cd5fe0104",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import sqlite3 as lite\n",
    "\n",
    "def create_connection(db_file):\n",
    "    conn = None\n",
    "    try:\n",
    "        conn = lite.connect(db_file, timeout=10)\n",
    "    except Error as e:\n",
    "        print(e)\n",
    "    return conn\n",
    "\n",
    "conn = create_connection(\"assets/CVEfixes.db\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "5d1c25c4-69f4-4949-8126-6a68239972e3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_change_id</th>\n",
       "      <th>hash</th>\n",
       "      <th>filename</th>\n",
       "      <th>old_path</th>\n",
       "      <th>new_path</th>\n",
       "      <th>change_type</th>\n",
       "      <th>diff</th>\n",
       "      <th>diff_parsed</th>\n",
       "      <th>num_lines_added</th>\n",
       "      <th>num_lines_deleted</th>\n",
       "      <th>code_after</th>\n",
       "      <th>code_before</th>\n",
       "      <th>nloc</th>\n",
       "      <th>complexity</th>\n",
       "      <th>token_count</th>\n",
       "      <th>programming_language</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100562011220946</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>ChangeLog</td>\n",
       "      <td>ChangeLog</td>\n",
       "      <td>ChangeLog</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>@@ -1,3 +1,55 @@\\n+1999-11-09  Ulrich Drepper ...</td>\n",
       "      <td>{'added': [(1, '1999-11-09  Ulrich Drepper  &lt;d...</td>\n",
       "      <td>52</td>\n",
       "      <td>0</td>\n",
       "      <td>1999-11-09  Ulrich Drepper  &lt;drepper@cygnus.co...</td>\n",
       "      <td>1999-11-08  Ulrich Drepper  &lt;drepper@cygnus.co...</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>Batchfile</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7461293048740</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>dl-load.c</td>\n",
       "      <td>elf/dl-load.c</td>\n",
       "      <td>elf/dl-load.c</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>@@ -149,21 +149,31 @@ local_strdup (const char...</td>\n",
       "      <td>{'added': [(152, '  const char *const start = ...</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>952.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>6592.0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>169254254633061</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>search.texi</td>\n",
       "      <td>manual/search.texi</td>\n",
       "      <td>manual/search.texi</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>@@ -475,9 +475,11 @@ elements.\\n To remove a s...</td>\n",
       "      <td>{'added': [(478, 'and a pointer to the parent ...</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>@node Searching and Sorting, Pattern Matching,...</td>\n",
       "      <td>@node Searching and Sorting, Pattern Matching,...</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>TeX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>253444729592566</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>syslog.c</td>\n",
       "      <td>misc/syslog.c</td>\n",
       "      <td>misc/syslog.c</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>@@ -177,10 +177,14 @@ vsyslog(pri, fmt, ap)\\n ...</td>\n",
       "      <td>{'added': [(180, '\\t\\t/* Append a newline if n...</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>/*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...</td>\n",
       "      <td>/*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...</td>\n",
       "      <td>229.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>1240.0</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>197890948684264</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>Makefile</td>\n",
       "      <td>posix/Makefile</td>\n",
       "      <td>posix/Makefile</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>@@ -57,7 +57,7 @@ include ../Makeconfig\\n \\n a...</td>\n",
       "      <td>{'added': [(60, '\\t\\t   tst-preadwrite test-vf...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td># Copyright (C) 1991,92,93,94,95,96,97,98,99 F...</td>\n",
       "      <td># Copyright (C) 1991,92,93,94,95,96,97,98,99 F...</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>Shell</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    file_change_id                                      hash     filename  \\\n",
       "0  100562011220946  2864e767053317538feafa815046fff89e5a16be    ChangeLog   \n",
       "1    7461293048740  2864e767053317538feafa815046fff89e5a16be    dl-load.c   \n",
       "2  169254254633061  2864e767053317538feafa815046fff89e5a16be  search.texi   \n",
       "3  253444729592566  2864e767053317538feafa815046fff89e5a16be     syslog.c   \n",
       "4  197890948684264  2864e767053317538feafa815046fff89e5a16be     Makefile   \n",
       "\n",
       "             old_path            new_path              change_type  \\\n",
       "0           ChangeLog           ChangeLog  ModificationType.MODIFY   \n",
       "1       elf/dl-load.c       elf/dl-load.c  ModificationType.MODIFY   \n",
       "2  manual/search.texi  manual/search.texi  ModificationType.MODIFY   \n",
       "3       misc/syslog.c       misc/syslog.c  ModificationType.MODIFY   \n",
       "4      posix/Makefile      posix/Makefile  ModificationType.MODIFY   \n",
       "\n",
       "                                                diff  \\\n",
       "0  @@ -1,3 +1,55 @@\\n+1999-11-09  Ulrich Drepper ...   \n",
       "1  @@ -149,21 +149,31 @@ local_strdup (const char...   \n",
       "2  @@ -475,9 +475,11 @@ elements.\\n To remove a s...   \n",
       "3  @@ -177,10 +177,14 @@ vsyslog(pri, fmt, ap)\\n ...   \n",
       "4  @@ -57,7 +57,7 @@ include ../Makeconfig\\n \\n a...   \n",
       "\n",
       "                                         diff_parsed num_lines_added  \\\n",
       "0  {'added': [(1, '1999-11-09  Ulrich Drepper  <d...              52   \n",
       "1  {'added': [(152, '  const char *const start = ...              28   \n",
       "2  {'added': [(478, 'and a pointer to the parent ...               5   \n",
       "3  {'added': [(180, '\\t\\t/* Append a newline if n...               8   \n",
       "4  {'added': [(60, '\\t\\t   tst-preadwrite test-vf...               1   \n",
       "\n",
       "  num_lines_deleted                                         code_after  \\\n",
       "0                 0  1999-11-09  Ulrich Drepper  <drepper@cygnus.co...   \n",
       "1                10  /* Map in a shared object's segments from the ...   \n",
       "2                 3  @node Searching and Sorting, Pattern Matching,...   \n",
       "3                 4  /*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...   \n",
       "4                 1  # Copyright (C) 1991,92,93,94,95,96,97,98,99 F...   \n",
       "\n",
       "                                         code_before   nloc complexity  \\\n",
       "0  1999-11-08  Ulrich Drepper  <drepper@cygnus.co...    nan        nan   \n",
       "1  /* Map in a shared object's segments from the ...  952.0      260.0   \n",
       "2  @node Searching and Sorting, Pattern Matching,...    nan        nan   \n",
       "3  /*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...  229.0       42.0   \n",
       "4  # Copyright (C) 1991,92,93,94,95,96,97,98,99 F...    nan        nan   \n",
       "\n",
       "  token_count programming_language  \n",
       "0         nan            Batchfile  \n",
       "1      6592.0                    C  \n",
       "2         nan                  TeX  \n",
       "3      1240.0                    C  \n",
       "4         nan                Shell  "
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "query = \"\"\"\n",
    "    SELECT *\n",
    "    FROM file_change\n",
    "\"\"\"\n",
    "\n",
    "df = pd.read_sql_query(query, conn)\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "d4c2649a-b297-4e54-91e7-c6177d6791d7",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "programming_language\n",
       "PHP                 5841\n",
       "C                   5214\n",
       "unknown             3792\n",
       "JavaScript          2004\n",
       "C++                 1720\n",
       "Markdown            1495\n",
       "Python              1208\n",
       "Java                1187\n",
       "Ruby                1021\n",
       "HTML                 870\n",
       "TypeScript           811\n",
       "Go                   668\n",
       "Shell                667\n",
       "SQL                  347\n",
       "C#                   283\n",
       "Batchfile            276\n",
       "CoffeeScript         268\n",
       "Objective-C          254\n",
       "Perl                 228\n",
       "CSS                  220\n",
       "Scala                194\n",
       "Lua                  181\n",
       "Rust                 118\n",
       "TeX                   91\n",
       "Haskell               89\n",
       "PowerShell            68\n",
       "Jupyter Notebook      63\n",
       "Swift                 54\n",
       "Matlab                43\n",
       "R                     20\n",
       "Erlang                 9\n",
       "None                   5\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.programming_language.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "f2792b50-a02c-4d93-9c3e-0dc7c82f90ae",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>programming_language</th>\n",
       "      <th>code_before</th>\n",
       "      <th>code_after</th>\n",
       "      <th>diff</th>\n",
       "      <th>num_lines_added</th>\n",
       "      <th>num_lines_deleted</th>\n",
       "      <th>num_lines_in_file</th>\n",
       "      <th>num_tokens_in_file</th>\n",
       "      <th>complexity</th>\n",
       "      <th>file_change_id</th>\n",
       "      <th>hash</th>\n",
       "      <th>change_type</th>\n",
       "      <th>old_file_path</th>\n",
       "      <th>new_file_path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ChangeLog</td>\n",
       "      <td>Batchfile</td>\n",
       "      <td>1999-11-08  Ulrich Drepper  &lt;drepper@cygnus.co...</td>\n",
       "      <td>1999-11-09  Ulrich Drepper  &lt;drepper@cygnus.co...</td>\n",
       "      <td>{'added': [(1, '1999-11-09  Ulrich Drepper  &lt;d...</td>\n",
       "      <td>52</td>\n",
       "      <td>0</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>100562011220946</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>ChangeLog</td>\n",
       "      <td>ChangeLog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dl-load.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>{'added': [(152, '  const char *const start = ...</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>952.0</td>\n",
       "      <td>6592.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>7461293048740</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>elf/dl-load.c</td>\n",
       "      <td>elf/dl-load.c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>search.texi</td>\n",
       "      <td>TeX</td>\n",
       "      <td>@node Searching and Sorting, Pattern Matching,...</td>\n",
       "      <td>@node Searching and Sorting, Pattern Matching,...</td>\n",
       "      <td>{'added': [(478, 'and a pointer to the parent ...</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>169254254633061</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>manual/search.texi</td>\n",
       "      <td>manual/search.texi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>syslog.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...</td>\n",
       "      <td>/*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...</td>\n",
       "      <td>{'added': [(180, '\\t\\t/* Append a newline if n...</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>229.0</td>\n",
       "      <td>1240.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>253444729592566</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>misc/syslog.c</td>\n",
       "      <td>misc/syslog.c</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Makefile</td>\n",
       "      <td>Shell</td>\n",
       "      <td># Copyright (C) 1991,92,93,94,95,96,97,98,99 F...</td>\n",
       "      <td># Copyright (C) 1991,92,93,94,95,96,97,98,99 F...</td>\n",
       "      <td>{'added': [(60, '\\t\\t   tst-preadwrite test-vf...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>197890948684264</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>posix/Makefile</td>\n",
       "      <td>posix/Makefile</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     file_name programming_language  \\\n",
       "0    ChangeLog            Batchfile   \n",
       "1    dl-load.c                    C   \n",
       "2  search.texi                  TeX   \n",
       "3     syslog.c                    C   \n",
       "4     Makefile                Shell   \n",
       "\n",
       "                                         code_before  \\\n",
       "0  1999-11-08  Ulrich Drepper  <drepper@cygnus.co...   \n",
       "1  /* Map in a shared object's segments from the ...   \n",
       "2  @node Searching and Sorting, Pattern Matching,...   \n",
       "3  /*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...   \n",
       "4  # Copyright (C) 1991,92,93,94,95,96,97,98,99 F...   \n",
       "\n",
       "                                          code_after  \\\n",
       "0  1999-11-09  Ulrich Drepper  <drepper@cygnus.co...   \n",
       "1  /* Map in a shared object's segments from the ...   \n",
       "2  @node Searching and Sorting, Pattern Matching,...   \n",
       "3  /*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...   \n",
       "4  # Copyright (C) 1991,92,93,94,95,96,97,98,99 F...   \n",
       "\n",
       "                                                diff num_lines_added  \\\n",
       "0  {'added': [(1, '1999-11-09  Ulrich Drepper  <d...              52   \n",
       "1  {'added': [(152, '  const char *const start = ...              28   \n",
       "2  {'added': [(478, 'and a pointer to the parent ...               5   \n",
       "3  {'added': [(180, '\\t\\t/* Append a newline if n...               8   \n",
       "4  {'added': [(60, '\\t\\t   tst-preadwrite test-vf...               1   \n",
       "\n",
       "  num_lines_deleted num_lines_in_file num_tokens_in_file complexity  \\\n",
       "0                 0               nan                nan        nan   \n",
       "1                10             952.0             6592.0      260.0   \n",
       "2                 3               nan                nan        nan   \n",
       "3                 4             229.0             1240.0       42.0   \n",
       "4                 1               nan                nan        nan   \n",
       "\n",
       "    file_change_id                                      hash  \\\n",
       "0  100562011220946  2864e767053317538feafa815046fff89e5a16be   \n",
       "1    7461293048740  2864e767053317538feafa815046fff89e5a16be   \n",
       "2  169254254633061  2864e767053317538feafa815046fff89e5a16be   \n",
       "3  253444729592566  2864e767053317538feafa815046fff89e5a16be   \n",
       "4  197890948684264  2864e767053317538feafa815046fff89e5a16be   \n",
       "\n",
       "               change_type       old_file_path       new_file_path  \n",
       "0  ModificationType.MODIFY           ChangeLog           ChangeLog  \n",
       "1  ModificationType.MODIFY       elf/dl-load.c       elf/dl-load.c  \n",
       "2  ModificationType.MODIFY  manual/search.texi  manual/search.texi  \n",
       "3  ModificationType.MODIFY       misc/syslog.c       misc/syslog.c  \n",
       "4  ModificationType.MODIFY      posix/Makefile      posix/Makefile  "
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df[['filename', 'programming_language', 'code_before', 'code_after',\n",
    "         'diff_parsed', 'num_lines_added', 'num_lines_deleted', 'nloc', 'token_count', 'complexity',\n",
    "         'file_change_id', 'hash', 'change_type', 'old_path', 'new_path']]\n",
    "\n",
    "df = df.rename(columns={\n",
    "    'filename': 'file_name',\n",
    "    'old_path': 'old_file_path',\n",
    "    'new_path': 'new_file_path',\n",
    "    'diff_parsed': 'diff',\n",
    "    'nloc': 'num_lines_in_file',\n",
    "    'token_count': 'num_tokens_in_file',  \n",
    "})\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "889b9d1e-38c2-4ecf-ba26-d3d8cbc22ec4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>programming_language</th>\n",
       "      <th>code_before</th>\n",
       "      <th>code_after</th>\n",
       "      <th>diff</th>\n",
       "      <th>num_lines_added</th>\n",
       "      <th>num_lines_deleted</th>\n",
       "      <th>num_lines_in_file</th>\n",
       "      <th>num_tokens_in_file</th>\n",
       "      <th>complexity</th>\n",
       "      <th>file_change_id</th>\n",
       "      <th>hash</th>\n",
       "      <th>change_type</th>\n",
       "      <th>old_file_path</th>\n",
       "      <th>new_file_path</th>\n",
       "      <th>repo_url</th>\n",
       "      <th>author</th>\n",
       "      <th>committer</th>\n",
       "      <th>commit_message</th>\n",
       "      <th>merge</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ChangeLog</td>\n",
       "      <td>Batchfile</td>\n",
       "      <td>1999-11-08  Ulrich Drepper  &lt;drepper@cygnus.co...</td>\n",
       "      <td>1999-11-09  Ulrich Drepper  &lt;drepper@cygnus.co...</td>\n",
       "      <td>{'added': [(1, '1999-11-09  Ulrich Drepper  &lt;d...</td>\n",
       "      <td>52</td>\n",
       "      <td>0</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>100562011220946</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>ChangeLog</td>\n",
       "      <td>ChangeLog</td>\n",
       "      <td>https://github.com/bminor/glibc</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Update.\\n\\n1999-11-09  Ulrich Drepper  &lt;dreppe...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dl-load.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>{'added': [(152, '  const char *const start = ...</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>952.0</td>\n",
       "      <td>6592.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>7461293048740</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>elf/dl-load.c</td>\n",
       "      <td>elf/dl-load.c</td>\n",
       "      <td>https://github.com/bminor/glibc</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Update.\\n\\n1999-11-09  Ulrich Drepper  &lt;dreppe...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>search.texi</td>\n",
       "      <td>TeX</td>\n",
       "      <td>@node Searching and Sorting, Pattern Matching,...</td>\n",
       "      <td>@node Searching and Sorting, Pattern Matching,...</td>\n",
       "      <td>{'added': [(478, 'and a pointer to the parent ...</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>169254254633061</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>manual/search.texi</td>\n",
       "      <td>manual/search.texi</td>\n",
       "      <td>https://github.com/bminor/glibc</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Update.\\n\\n1999-11-09  Ulrich Drepper  &lt;dreppe...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>syslog.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...</td>\n",
       "      <td>/*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...</td>\n",
       "      <td>{'added': [(180, '\\t\\t/* Append a newline if n...</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>229.0</td>\n",
       "      <td>1240.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>253444729592566</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>misc/syslog.c</td>\n",
       "      <td>misc/syslog.c</td>\n",
       "      <td>https://github.com/bminor/glibc</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Update.\\n\\n1999-11-09  Ulrich Drepper  &lt;dreppe...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Makefile</td>\n",
       "      <td>Shell</td>\n",
       "      <td># Copyright (C) 1991,92,93,94,95,96,97,98,99 F...</td>\n",
       "      <td># Copyright (C) 1991,92,93,94,95,96,97,98,99 F...</td>\n",
       "      <td>{'added': [(60, '\\t\\t   tst-preadwrite test-vf...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>197890948684264</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>posix/Makefile</td>\n",
       "      <td>posix/Makefile</td>\n",
       "      <td>https://github.com/bminor/glibc</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Update.\\n\\n1999-11-09  Ulrich Drepper  &lt;dreppe...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     file_name programming_language  \\\n",
       "0    ChangeLog            Batchfile   \n",
       "1    dl-load.c                    C   \n",
       "2  search.texi                  TeX   \n",
       "3     syslog.c                    C   \n",
       "4     Makefile                Shell   \n",
       "\n",
       "                                         code_before  \\\n",
       "0  1999-11-08  Ulrich Drepper  <drepper@cygnus.co...   \n",
       "1  /* Map in a shared object's segments from the ...   \n",
       "2  @node Searching and Sorting, Pattern Matching,...   \n",
       "3  /*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...   \n",
       "4  # Copyright (C) 1991,92,93,94,95,96,97,98,99 F...   \n",
       "\n",
       "                                          code_after  \\\n",
       "0  1999-11-09  Ulrich Drepper  <drepper@cygnus.co...   \n",
       "1  /* Map in a shared object's segments from the ...   \n",
       "2  @node Searching and Sorting, Pattern Matching,...   \n",
       "3  /*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...   \n",
       "4  # Copyright (C) 1991,92,93,94,95,96,97,98,99 F...   \n",
       "\n",
       "                                                diff num_lines_added  \\\n",
       "0  {'added': [(1, '1999-11-09  Ulrich Drepper  <d...              52   \n",
       "1  {'added': [(152, '  const char *const start = ...              28   \n",
       "2  {'added': [(478, 'and a pointer to the parent ...               5   \n",
       "3  {'added': [(180, '\\t\\t/* Append a newline if n...               8   \n",
       "4  {'added': [(60, '\\t\\t   tst-preadwrite test-vf...               1   \n",
       "\n",
       "  num_lines_deleted num_lines_in_file num_tokens_in_file complexity  \\\n",
       "0                 0               nan                nan        nan   \n",
       "1                10             952.0             6592.0      260.0   \n",
       "2                 3               nan                nan        nan   \n",
       "3                 4             229.0             1240.0       42.0   \n",
       "4                 1               nan                nan        nan   \n",
       "\n",
       "    file_change_id                                      hash  \\\n",
       "0  100562011220946  2864e767053317538feafa815046fff89e5a16be   \n",
       "1    7461293048740  2864e767053317538feafa815046fff89e5a16be   \n",
       "2  169254254633061  2864e767053317538feafa815046fff89e5a16be   \n",
       "3  253444729592566  2864e767053317538feafa815046fff89e5a16be   \n",
       "4  197890948684264  2864e767053317538feafa815046fff89e5a16be   \n",
       "\n",
       "               change_type       old_file_path       new_file_path  \\\n",
       "0  ModificationType.MODIFY           ChangeLog           ChangeLog   \n",
       "1  ModificationType.MODIFY       elf/dl-load.c       elf/dl-load.c   \n",
       "2  ModificationType.MODIFY  manual/search.texi  manual/search.texi   \n",
       "3  ModificationType.MODIFY       misc/syslog.c       misc/syslog.c   \n",
       "4  ModificationType.MODIFY      posix/Makefile      posix/Makefile   \n",
       "\n",
       "                          repo_url          author       committer  \\\n",
       "0  https://github.com/bminor/glibc  Ulrich Drepper  Ulrich Drepper   \n",
       "1  https://github.com/bminor/glibc  Ulrich Drepper  Ulrich Drepper   \n",
       "2  https://github.com/bminor/glibc  Ulrich Drepper  Ulrich Drepper   \n",
       "3  https://github.com/bminor/glibc  Ulrich Drepper  Ulrich Drepper   \n",
       "4  https://github.com/bminor/glibc  Ulrich Drepper  Ulrich Drepper   \n",
       "\n",
       "                                      commit_message  merge  \n",
       "0  Update.\\n\\n1999-11-09  Ulrich Drepper  <dreppe...  False  \n",
       "1  Update.\\n\\n1999-11-09  Ulrich Drepper  <dreppe...  False  \n",
       "2  Update.\\n\\n1999-11-09  Ulrich Drepper  <dreppe...  False  \n",
       "3  Update.\\n\\n1999-11-09  Ulrich Drepper  <dreppe...  False  \n",
       "4  Update.\\n\\n1999-11-09  Ulrich Drepper  <dreppe...  False  "
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = \"\"\"\n",
    "    SELECT *\n",
    "    FROM commits\n",
    "\"\"\"\n",
    "\n",
    "# Using pandas to read data from the database and store it into a DataFrame\n",
    "df1 = pd.read_sql_query(query, conn)\n",
    "\n",
    "df1 = df1[[\n",
    "    'hash', 'repo_url', 'author', 'committer', 'msg', 'merge'\n",
    "]]\n",
    "df1 = df1.rename(columns={\n",
    "    'msg': 'commit_message',\n",
    "})\n",
    "\n",
    "merged_df = df.merge(df1, on='hash', how='inner')\n",
    "\n",
    "# Display the merged DataFrame\n",
    "merged_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "279cdf00-51ef-4998-acaf-b8a06f88a9f7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>programming_language</th>\n",
       "      <th>code_before</th>\n",
       "      <th>code_after</th>\n",
       "      <th>diff</th>\n",
       "      <th>num_lines_added</th>\n",
       "      <th>num_lines_deleted</th>\n",
       "      <th>num_lines_in_file</th>\n",
       "      <th>num_tokens_in_file</th>\n",
       "      <th>complexity</th>\n",
       "      <th>...</th>\n",
       "      <th>hash</th>\n",
       "      <th>change_type</th>\n",
       "      <th>old_file_path</th>\n",
       "      <th>new_file_path</th>\n",
       "      <th>repo_url</th>\n",
       "      <th>author</th>\n",
       "      <th>committer</th>\n",
       "      <th>commit_message</th>\n",
       "      <th>merge</th>\n",
       "      <th>cve_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ChangeLog</td>\n",
       "      <td>Batchfile</td>\n",
       "      <td>1999-11-08  Ulrich Drepper  &lt;drepper@cygnus.co...</td>\n",
       "      <td>1999-11-09  Ulrich Drepper  &lt;drepper@cygnus.co...</td>\n",
       "      <td>{'added': [(1, '1999-11-09  Ulrich Drepper  &lt;d...</td>\n",
       "      <td>52</td>\n",
       "      <td>0</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>...</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>ChangeLog</td>\n",
       "      <td>ChangeLog</td>\n",
       "      <td>https://github.com/bminor/glibc</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Update.\\n\\n1999-11-09  Ulrich Drepper  &lt;dreppe...</td>\n",
       "      <td>False</td>\n",
       "      <td>CVE-1999-0199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dl-load.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>{'added': [(152, '  const char *const start = ...</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>952.0</td>\n",
       "      <td>6592.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>elf/dl-load.c</td>\n",
       "      <td>elf/dl-load.c</td>\n",
       "      <td>https://github.com/bminor/glibc</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Update.\\n\\n1999-11-09  Ulrich Drepper  &lt;dreppe...</td>\n",
       "      <td>False</td>\n",
       "      <td>CVE-1999-0199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>search.texi</td>\n",
       "      <td>TeX</td>\n",
       "      <td>@node Searching and Sorting, Pattern Matching,...</td>\n",
       "      <td>@node Searching and Sorting, Pattern Matching,...</td>\n",
       "      <td>{'added': [(478, 'and a pointer to the parent ...</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>...</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>manual/search.texi</td>\n",
       "      <td>manual/search.texi</td>\n",
       "      <td>https://github.com/bminor/glibc</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Update.\\n\\n1999-11-09  Ulrich Drepper  &lt;dreppe...</td>\n",
       "      <td>False</td>\n",
       "      <td>CVE-1999-0199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>syslog.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...</td>\n",
       "      <td>/*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...</td>\n",
       "      <td>{'added': [(180, '\\t\\t/* Append a newline if n...</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>229.0</td>\n",
       "      <td>1240.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>misc/syslog.c</td>\n",
       "      <td>misc/syslog.c</td>\n",
       "      <td>https://github.com/bminor/glibc</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Update.\\n\\n1999-11-09  Ulrich Drepper  &lt;dreppe...</td>\n",
       "      <td>False</td>\n",
       "      <td>CVE-1999-0199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Makefile</td>\n",
       "      <td>Shell</td>\n",
       "      <td># Copyright (C) 1991,92,93,94,95,96,97,98,99 F...</td>\n",
       "      <td># Copyright (C) 1991,92,93,94,95,96,97,98,99 F...</td>\n",
       "      <td>{'added': [(60, '\\t\\t   tst-preadwrite test-vf...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>...</td>\n",
       "      <td>2864e767053317538feafa815046fff89e5a16be</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>posix/Makefile</td>\n",
       "      <td>posix/Makefile</td>\n",
       "      <td>https://github.com/bminor/glibc</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Update.\\n\\n1999-11-09  Ulrich Drepper  &lt;dreppe...</td>\n",
       "      <td>False</td>\n",
       "      <td>CVE-1999-0199</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     file_name programming_language  \\\n",
       "0    ChangeLog            Batchfile   \n",
       "1    dl-load.c                    C   \n",
       "2  search.texi                  TeX   \n",
       "3     syslog.c                    C   \n",
       "4     Makefile                Shell   \n",
       "\n",
       "                                         code_before  \\\n",
       "0  1999-11-08  Ulrich Drepper  <drepper@cygnus.co...   \n",
       "1  /* Map in a shared object's segments from the ...   \n",
       "2  @node Searching and Sorting, Pattern Matching,...   \n",
       "3  /*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...   \n",
       "4  # Copyright (C) 1991,92,93,94,95,96,97,98,99 F...   \n",
       "\n",
       "                                          code_after  \\\n",
       "0  1999-11-09  Ulrich Drepper  <drepper@cygnus.co...   \n",
       "1  /* Map in a shared object's segments from the ...   \n",
       "2  @node Searching and Sorting, Pattern Matching,...   \n",
       "3  /*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...   \n",
       "4  # Copyright (C) 1991,92,93,94,95,96,97,98,99 F...   \n",
       "\n",
       "                                                diff num_lines_added  \\\n",
       "0  {'added': [(1, '1999-11-09  Ulrich Drepper  <d...              52   \n",
       "1  {'added': [(152, '  const char *const start = ...              28   \n",
       "2  {'added': [(478, 'and a pointer to the parent ...               5   \n",
       "3  {'added': [(180, '\\t\\t/* Append a newline if n...               8   \n",
       "4  {'added': [(60, '\\t\\t   tst-preadwrite test-vf...               1   \n",
       "\n",
       "  num_lines_deleted num_lines_in_file num_tokens_in_file complexity  ...  \\\n",
       "0                 0               nan                nan        nan  ...   \n",
       "1                10             952.0             6592.0      260.0  ...   \n",
       "2                 3               nan                nan        nan  ...   \n",
       "3                 4             229.0             1240.0       42.0  ...   \n",
       "4                 1               nan                nan        nan  ...   \n",
       "\n",
       "                                       hash              change_type  \\\n",
       "0  2864e767053317538feafa815046fff89e5a16be  ModificationType.MODIFY   \n",
       "1  2864e767053317538feafa815046fff89e5a16be  ModificationType.MODIFY   \n",
       "2  2864e767053317538feafa815046fff89e5a16be  ModificationType.MODIFY   \n",
       "3  2864e767053317538feafa815046fff89e5a16be  ModificationType.MODIFY   \n",
       "4  2864e767053317538feafa815046fff89e5a16be  ModificationType.MODIFY   \n",
       "\n",
       "        old_file_path       new_file_path                         repo_url  \\\n",
       "0           ChangeLog           ChangeLog  https://github.com/bminor/glibc   \n",
       "1       elf/dl-load.c       elf/dl-load.c  https://github.com/bminor/glibc   \n",
       "2  manual/search.texi  manual/search.texi  https://github.com/bminor/glibc   \n",
       "3       misc/syslog.c       misc/syslog.c  https://github.com/bminor/glibc   \n",
       "4      posix/Makefile      posix/Makefile  https://github.com/bminor/glibc   \n",
       "\n",
       "           author       committer  \\\n",
       "0  Ulrich Drepper  Ulrich Drepper   \n",
       "1  Ulrich Drepper  Ulrich Drepper   \n",
       "2  Ulrich Drepper  Ulrich Drepper   \n",
       "3  Ulrich Drepper  Ulrich Drepper   \n",
       "4  Ulrich Drepper  Ulrich Drepper   \n",
       "\n",
       "                                      commit_message  merge         cve_id  \n",
       "0  Update.\\n\\n1999-11-09  Ulrich Drepper  <dreppe...  False  CVE-1999-0199  \n",
       "1  Update.\\n\\n1999-11-09  Ulrich Drepper  <dreppe...  False  CVE-1999-0199  \n",
       "2  Update.\\n\\n1999-11-09  Ulrich Drepper  <dreppe...  False  CVE-1999-0199  \n",
       "3  Update.\\n\\n1999-11-09  Ulrich Drepper  <dreppe...  False  CVE-1999-0199  \n",
       "4  Update.\\n\\n1999-11-09  Ulrich Drepper  <dreppe...  False  CVE-1999-0199  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = \"\"\"\n",
    "    SELECT *\n",
    "    FROM fixes\n",
    "\"\"\"\n",
    "\n",
    "# Using pandas to read data from the database and store it into a DataFrame\n",
    "df3 = pd.read_sql_query(query, conn)\n",
    "\n",
    "df3 = df3[[\n",
    "    'cve_id', 'hash'\n",
    "]]\n",
    "\n",
    "# df2 = df2.rename(columns={\n",
    "#     'stars_count': 'repo_stars_count',\n",
    "# })\n",
    "\n",
    "merged_df = merged_df.merge(df3, on='hash', how='inner')\n",
    "\n",
    "# Display the merged DataFrame\n",
    "merged_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "051286f5-a573-4fb5-a65a-1c7add269d4f",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cve_id</th>\n",
       "      <th>cwe_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CVE-1999-0199</td>\n",
       "      <td>[CWE-252]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CVE-1999-0731</td>\n",
       "      <td>[NVD-CWE-Other]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CVE-2002-2443</td>\n",
       "      <td>[CWE-20]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CVE-2006-3635</td>\n",
       "      <td>[CWE-119]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CVE-2006-5331</td>\n",
       "      <td>[CWE-19]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>CVE-2011-2905</td>\n",
       "      <td>[NVD-CWE-Other]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>CVE-2011-2906</td>\n",
       "      <td>[CWE-400]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>CVE-2011-2909</td>\n",
       "      <td>[CWE-200]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>CVE-2011-2918</td>\n",
       "      <td>[CWE-400]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>CVE-2011-2929</td>\n",
       "      <td>[CWE-20]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           cve_id           cwe_id\n",
       "0   CVE-1999-0199        [CWE-252]\n",
       "1   CVE-1999-0731  [NVD-CWE-Other]\n",
       "2   CVE-2002-2443         [CWE-20]\n",
       "3   CVE-2006-3635        [CWE-119]\n",
       "4   CVE-2006-5331         [CWE-19]\n",
       "..            ...              ...\n",
       "95  CVE-2011-2905  [NVD-CWE-Other]\n",
       "96  CVE-2011-2906        [CWE-400]\n",
       "97  CVE-2011-2909        [CWE-200]\n",
       "98  CVE-2011-2918        [CWE-400]\n",
       "99  CVE-2011-2929         [CWE-20]\n",
       "\n",
       "[100 rows x 2 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = \"\"\"\n",
    "    SELECT *\n",
    "    FROM cwe_classification\n",
    "\"\"\"\n",
    "\n",
    "# Using pandas to read data from the database and store it into a DataFrame\n",
    "df4 = pd.read_sql_query(query, conn)\n",
    "\n",
    "df4 = df4.groupby('cve_id')['cwe_id'].apply(list).reset_index()\n",
    "\n",
    "df4.head(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "e9ae6336-f902-4775-9507-d4839bf5046d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>programming_language</th>\n",
       "      <th>code_before</th>\n",
       "      <th>code_after</th>\n",
       "      <th>diff</th>\n",
       "      <th>num_lines_added</th>\n",
       "      <th>num_lines_deleted</th>\n",
       "      <th>num_lines_in_file</th>\n",
       "      <th>num_tokens_in_file</th>\n",
       "      <th>complexity</th>\n",
       "      <th>...</th>\n",
       "      <th>change_type</th>\n",
       "      <th>old_file_path</th>\n",
       "      <th>new_file_path</th>\n",
       "      <th>repo_url</th>\n",
       "      <th>author</th>\n",
       "      <th>committer</th>\n",
       "      <th>commit_message</th>\n",
       "      <th>merge</th>\n",
       "      <th>cve_id</th>\n",
       "      <th>cwe_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ChangeLog</td>\n",
       "      <td>Batchfile</td>\n",
       "      <td>1999-11-08  Ulrich Drepper  &lt;drepper@cygnus.co...</td>\n",
       "      <td>1999-11-09  Ulrich Drepper  &lt;drepper@cygnus.co...</td>\n",
       "      <td>{'added': [(1, '1999-11-09  Ulrich Drepper  &lt;d...</td>\n",
       "      <td>52</td>\n",
       "      <td>0</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>...</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>ChangeLog</td>\n",
       "      <td>ChangeLog</td>\n",
       "      <td>https://github.com/bminor/glibc</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Update.\\n\\n1999-11-09  Ulrich Drepper  &lt;dreppe...</td>\n",
       "      <td>False</td>\n",
       "      <td>CVE-1999-0199</td>\n",
       "      <td>CWE-252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dl-load.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>{'added': [(152, '  const char *const start = ...</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>952.0</td>\n",
       "      <td>6592.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>...</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>elf/dl-load.c</td>\n",
       "      <td>elf/dl-load.c</td>\n",
       "      <td>https://github.com/bminor/glibc</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Update.\\n\\n1999-11-09  Ulrich Drepper  &lt;dreppe...</td>\n",
       "      <td>False</td>\n",
       "      <td>CVE-1999-0199</td>\n",
       "      <td>CWE-252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>search.texi</td>\n",
       "      <td>TeX</td>\n",
       "      <td>@node Searching and Sorting, Pattern Matching,...</td>\n",
       "      <td>@node Searching and Sorting, Pattern Matching,...</td>\n",
       "      <td>{'added': [(478, 'and a pointer to the parent ...</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>...</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>manual/search.texi</td>\n",
       "      <td>manual/search.texi</td>\n",
       "      <td>https://github.com/bminor/glibc</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Update.\\n\\n1999-11-09  Ulrich Drepper  &lt;dreppe...</td>\n",
       "      <td>False</td>\n",
       "      <td>CVE-1999-0199</td>\n",
       "      <td>CWE-252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>syslog.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...</td>\n",
       "      <td>/*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...</td>\n",
       "      <td>{'added': [(180, '\\t\\t/* Append a newline if n...</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>229.0</td>\n",
       "      <td>1240.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>...</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>misc/syslog.c</td>\n",
       "      <td>misc/syslog.c</td>\n",
       "      <td>https://github.com/bminor/glibc</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Update.\\n\\n1999-11-09  Ulrich Drepper  &lt;dreppe...</td>\n",
       "      <td>False</td>\n",
       "      <td>CVE-1999-0199</td>\n",
       "      <td>CWE-252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Makefile</td>\n",
       "      <td>Shell</td>\n",
       "      <td># Copyright (C) 1991,92,93,94,95,96,97,98,99 F...</td>\n",
       "      <td># Copyright (C) 1991,92,93,94,95,96,97,98,99 F...</td>\n",
       "      <td>{'added': [(60, '\\t\\t   tst-preadwrite test-vf...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>...</td>\n",
       "      <td>ModificationType.MODIFY</td>\n",
       "      <td>posix/Makefile</td>\n",
       "      <td>posix/Makefile</td>\n",
       "      <td>https://github.com/bminor/glibc</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Ulrich Drepper</td>\n",
       "      <td>Update.\\n\\n1999-11-09  Ulrich Drepper  &lt;dreppe...</td>\n",
       "      <td>False</td>\n",
       "      <td>CVE-1999-0199</td>\n",
       "      <td>CWE-252</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     file_name programming_language  \\\n",
       "0    ChangeLog            Batchfile   \n",
       "1    dl-load.c                    C   \n",
       "2  search.texi                  TeX   \n",
       "3     syslog.c                    C   \n",
       "4     Makefile                Shell   \n",
       "\n",
       "                                         code_before  \\\n",
       "0  1999-11-08  Ulrich Drepper  <drepper@cygnus.co...   \n",
       "1  /* Map in a shared object's segments from the ...   \n",
       "2  @node Searching and Sorting, Pattern Matching,...   \n",
       "3  /*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...   \n",
       "4  # Copyright (C) 1991,92,93,94,95,96,97,98,99 F...   \n",
       "\n",
       "                                          code_after  \\\n",
       "0  1999-11-09  Ulrich Drepper  <drepper@cygnus.co...   \n",
       "1  /* Map in a shared object's segments from the ...   \n",
       "2  @node Searching and Sorting, Pattern Matching,...   \n",
       "3  /*\\n * Copyright (c) 1983, 1988, 1993\\n *\\tThe...   \n",
       "4  # Copyright (C) 1991,92,93,94,95,96,97,98,99 F...   \n",
       "\n",
       "                                                diff num_lines_added  \\\n",
       "0  {'added': [(1, '1999-11-09  Ulrich Drepper  <d...              52   \n",
       "1  {'added': [(152, '  const char *const start = ...              28   \n",
       "2  {'added': [(478, 'and a pointer to the parent ...               5   \n",
       "3  {'added': [(180, '\\t\\t/* Append a newline if n...               8   \n",
       "4  {'added': [(60, '\\t\\t   tst-preadwrite test-vf...               1   \n",
       "\n",
       "  num_lines_deleted num_lines_in_file num_tokens_in_file complexity  ...  \\\n",
       "0                 0               nan                nan        nan  ...   \n",
       "1                10             952.0             6592.0      260.0  ...   \n",
       "2                 3               nan                nan        nan  ...   \n",
       "3                 4             229.0             1240.0       42.0  ...   \n",
       "4                 1               nan                nan        nan  ...   \n",
       "\n",
       "               change_type       old_file_path       new_file_path  \\\n",
       "0  ModificationType.MODIFY           ChangeLog           ChangeLog   \n",
       "1  ModificationType.MODIFY       elf/dl-load.c       elf/dl-load.c   \n",
       "2  ModificationType.MODIFY  manual/search.texi  manual/search.texi   \n",
       "3  ModificationType.MODIFY       misc/syslog.c       misc/syslog.c   \n",
       "4  ModificationType.MODIFY      posix/Makefile      posix/Makefile   \n",
       "\n",
       "                          repo_url          author       committer  \\\n",
       "0  https://github.com/bminor/glibc  Ulrich Drepper  Ulrich Drepper   \n",
       "1  https://github.com/bminor/glibc  Ulrich Drepper  Ulrich Drepper   \n",
       "2  https://github.com/bminor/glibc  Ulrich Drepper  Ulrich Drepper   \n",
       "3  https://github.com/bminor/glibc  Ulrich Drepper  Ulrich Drepper   \n",
       "4  https://github.com/bminor/glibc  Ulrich Drepper  Ulrich Drepper   \n",
       "\n",
       "                                      commit_message  merge         cve_id  \\\n",
       "0  Update.\\n\\n1999-11-09  Ulrich Drepper  <dreppe...  False  CVE-1999-0199   \n",
       "1  Update.\\n\\n1999-11-09  Ulrich Drepper  <dreppe...  False  CVE-1999-0199   \n",
       "2  Update.\\n\\n1999-11-09  Ulrich Drepper  <dreppe...  False  CVE-1999-0199   \n",
       "3  Update.\\n\\n1999-11-09  Ulrich Drepper  <dreppe...  False  CVE-1999-0199   \n",
       "4  Update.\\n\\n1999-11-09  Ulrich Drepper  <dreppe...  False  CVE-1999-0199   \n",
       "\n",
       "    cwe_id  \n",
       "0  CWE-252  \n",
       "1  CWE-252  \n",
       "2  CWE-252  \n",
       "3  CWE-252  \n",
       "4  CWE-252  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df = merged_df.merge(df4, on='cve_id', how='inner')\n",
    "merged_df = merged_df.explode('cwe_id').reset_index(drop=True)\n",
    "\n",
    "# Display the merged DataFrame\n",
    "merged_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "id": "8e516a1d-5c54-40d0-b963-1e6d5524199c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "query = \"\"\"\n",
    "    SELECT *\n",
    "    FROM method_change\n",
    "\"\"\"\n",
    "\n",
    "# Using pandas to read data from the database and store it into a DataFrame\n",
    "df6 = pd.read_sql_query(query, conn)\n",
    "\n",
    "df6 = df6[[\n",
    "    'file_change_id', 'method_change_id', 'name', 'signature', 'parameters', 'code', 'nloc', 'complexity', 'token_count', 'before_change'\n",
    "]]\n",
    "\n",
    "df6 = df6.rename(columns={\n",
    "    'name': 'method_name',\n",
    "    'signature': 'method_signature', \n",
    "    'parameters': 'method_parameters',\n",
    "    'code': 'method_code',\n",
    "    'nloc': 'num_lines_in_method',\n",
    "    'complexity': 'method_complexity',\n",
    "    'token_count': 'num_tokens_in_method',\n",
    "    'before_change': 'vulnerable'\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "11297e05-891c-438b-894b-fa65236311f8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>programming_language</th>\n",
       "      <th>code_before</th>\n",
       "      <th>code_after</th>\n",
       "      <th>diff</th>\n",
       "      <th>num_lines_added</th>\n",
       "      <th>num_lines_deleted</th>\n",
       "      <th>num_lines_in_file</th>\n",
       "      <th>num_tokens_in_file</th>\n",
       "      <th>complexity</th>\n",
       "      <th>...</th>\n",
       "      <th>cwe_id</th>\n",
       "      <th>method_change_id</th>\n",
       "      <th>method_name</th>\n",
       "      <th>method_signature</th>\n",
       "      <th>method_parameters</th>\n",
       "      <th>method_code</th>\n",
       "      <th>num_lines_in_method</th>\n",
       "      <th>method_complexity</th>\n",
       "      <th>num_tokens_in_method</th>\n",
       "      <th>vulnerable</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dl-load.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>{'added': [(152, '  const char *const start = ...</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>952.0</td>\n",
       "      <td>6592.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>...</td>\n",
       "      <td>CWE-252</td>\n",
       "      <td>241941308837203</td>\n",
       "      <td>expand_dynamic_string_token</td>\n",
       "      <td>expand_dynamic_string_token( struct link_map *...</td>\n",
       "      <td>['l', 's']</td>\n",
       "      <td>expand_dynamic_string_token (struct link_map *...</td>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "      <td>95</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dl-load.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>{'added': [(152, '  const char *const start = ...</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>952.0</td>\n",
       "      <td>6592.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>...</td>\n",
       "      <td>CWE-252</td>\n",
       "      <td>235368100398254</td>\n",
       "      <td>_dl_dst_substitute</td>\n",
       "      <td>_dl_dst_substitute( struct link_map * l , cons...</td>\n",
       "      <td>['l', 'name', 'result', 'is_path']</td>\n",
       "      <td>_dl_dst_substitute (struct link_map *l, const ...</td>\n",
       "      <td>51</td>\n",
       "      <td>22</td>\n",
       "      <td>349</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dl-load.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>{'added': [(152, '  const char *const start = ...</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>952.0</td>\n",
       "      <td>6592.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>...</td>\n",
       "      <td>CWE-252</td>\n",
       "      <td>161060042440601</td>\n",
       "      <td>_dl_dst_count</td>\n",
       "      <td>_dl_dst_count( const char * name , int is_path)</td>\n",
       "      <td>['name', 'is_path']</td>\n",
       "      <td>_dl_dst_count (const char *name, int is_path)\\...</td>\n",
       "      <td>22</td>\n",
       "      <td>13</td>\n",
       "      <td>199</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dl-load.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>{'added': [(152, '  const char *const start = ...</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>952.0</td>\n",
       "      <td>6592.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>...</td>\n",
       "      <td>CWE-252</td>\n",
       "      <td>205987001054463</td>\n",
       "      <td>expand_dynamic_string_token</td>\n",
       "      <td>expand_dynamic_string_token( struct link_map *...</td>\n",
       "      <td>['l', 's']</td>\n",
       "      <td>expand_dynamic_string_token (struct link_map *...</td>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "      <td>95</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>dl-load.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>{'added': [(152, '  const char *const start = ...</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>952.0</td>\n",
       "      <td>6592.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>...</td>\n",
       "      <td>CWE-252</td>\n",
       "      <td>38197852149195</td>\n",
       "      <td>_dl_dst_substitute</td>\n",
       "      <td>_dl_dst_substitute( struct link_map * l , cons...</td>\n",
       "      <td>['l', 'name', 'result', 'is_path']</td>\n",
       "      <td>_dl_dst_substitute (struct link_map *l, const ...</td>\n",
       "      <td>57</td>\n",
       "      <td>28</td>\n",
       "      <td>399</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   file_name programming_language  \\\n",
       "0  dl-load.c                    C   \n",
       "1  dl-load.c                    C   \n",
       "2  dl-load.c                    C   \n",
       "3  dl-load.c                    C   \n",
       "4  dl-load.c                    C   \n",
       "\n",
       "                                         code_before  \\\n",
       "0  /* Map in a shared object's segments from the ...   \n",
       "1  /* Map in a shared object's segments from the ...   \n",
       "2  /* Map in a shared object's segments from the ...   \n",
       "3  /* Map in a shared object's segments from the ...   \n",
       "4  /* Map in a shared object's segments from the ...   \n",
       "\n",
       "                                          code_after  \\\n",
       "0  /* Map in a shared object's segments from the ...   \n",
       "1  /* Map in a shared object's segments from the ...   \n",
       "2  /* Map in a shared object's segments from the ...   \n",
       "3  /* Map in a shared object's segments from the ...   \n",
       "4  /* Map in a shared object's segments from the ...   \n",
       "\n",
       "                                                diff num_lines_added  \\\n",
       "0  {'added': [(152, '  const char *const start = ...              28   \n",
       "1  {'added': [(152, '  const char *const start = ...              28   \n",
       "2  {'added': [(152, '  const char *const start = ...              28   \n",
       "3  {'added': [(152, '  const char *const start = ...              28   \n",
       "4  {'added': [(152, '  const char *const start = ...              28   \n",
       "\n",
       "  num_lines_deleted num_lines_in_file num_tokens_in_file complexity  ...  \\\n",
       "0                10             952.0             6592.0      260.0  ...   \n",
       "1                10             952.0             6592.0      260.0  ...   \n",
       "2                10             952.0             6592.0      260.0  ...   \n",
       "3                10             952.0             6592.0      260.0  ...   \n",
       "4                10             952.0             6592.0      260.0  ...   \n",
       "\n",
       "    cwe_id method_change_id                  method_name  \\\n",
       "0  CWE-252  241941308837203  expand_dynamic_string_token   \n",
       "1  CWE-252  235368100398254           _dl_dst_substitute   \n",
       "2  CWE-252  161060042440601                _dl_dst_count   \n",
       "3  CWE-252  205987001054463  expand_dynamic_string_token   \n",
       "4  CWE-252   38197852149195           _dl_dst_substitute   \n",
       "\n",
       "                                    method_signature  \\\n",
       "0  expand_dynamic_string_token( struct link_map *...   \n",
       "1  _dl_dst_substitute( struct link_map * l , cons...   \n",
       "2    _dl_dst_count( const char * name , int is_path)   \n",
       "3  expand_dynamic_string_token( struct link_map *...   \n",
       "4  _dl_dst_substitute( struct link_map * l , cons...   \n",
       "\n",
       "                    method_parameters  \\\n",
       "0                          ['l', 's']   \n",
       "1  ['l', 'name', 'result', 'is_path']   \n",
       "2                 ['name', 'is_path']   \n",
       "3                          ['l', 's']   \n",
       "4  ['l', 'name', 'result', 'is_path']   \n",
       "\n",
       "                                         method_code num_lines_in_method  \\\n",
       "0  expand_dynamic_string_token (struct link_map *...                  14   \n",
       "1  _dl_dst_substitute (struct link_map *l, const ...                  51   \n",
       "2  _dl_dst_count (const char *name, int is_path)\\...                  22   \n",
       "3  expand_dynamic_string_token (struct link_map *...                  14   \n",
       "4  _dl_dst_substitute (struct link_map *l, const ...                  57   \n",
       "\n",
       "  method_complexity num_tokens_in_method vulnerable  \n",
       "0                 3                   95       True  \n",
       "1                22                  349       True  \n",
       "2                13                  199       True  \n",
       "3                 3                   95      False  \n",
       "4                28                  399      False  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df = merged_df.merge(df6, on='file_change_id', how='inner')\n",
    "\n",
    "# Display the merged DataFrame\n",
    "merged_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "93b027e6-2175-46c9-894b-e593ab5e5718",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['file_name', 'programming_language', 'code_before', 'code_after',\n",
       "       'diff', 'num_lines_added', 'num_lines_deleted', 'num_lines_in_file',\n",
       "       'num_tokens_in_file', 'complexity', 'file_change_id', 'hash',\n",
       "       'change_type', 'old_file_path', 'new_file_path', 'repo_url', 'author',\n",
       "       'committer', 'commit_message', 'merge', 'cve_id', 'cwe_id',\n",
       "       'method_change_id', 'method_name', 'method_signature',\n",
       "       'method_parameters', 'method_code', 'num_lines_in_method',\n",
       "       'method_complexity', 'num_tokens_in_method', 'vulnerable'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "5fb82186-18e2-4f8e-adcc-7b76d58c3cca",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "programming_language\n",
       "JavaScript      46755\n",
       "PHP             18187\n",
       "C               14474\n",
       "unknown         13635\n",
       "C++              5101\n",
       "Python           4343\n",
       "Java             3955\n",
       "Ruby             3064\n",
       "Go               2274\n",
       "TypeScript       1193\n",
       "C#                942\n",
       "Objective-C       643\n",
       "HTML              298\n",
       "Scala             294\n",
       "Rust              264\n",
       "Swift             118\n",
       "Lua                97\n",
       "CoffeeScript       80\n",
       "Shell              69\n",
       "SQL                67\n",
       "Perl               67\n",
       "PowerShell         47\n",
       "Matlab             39\n",
       "Markdown           32\n",
       "TeX                10\n",
       "Haskell             6\n",
       "Erlang              4\n",
       "Batchfile           2\n",
       "None                1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df.programming_language.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "6ff51ff2-e9f8-487c-8c15-2e2cb18f5f2b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['file_name', 'programming_language', 'code_before', 'code_after',\n",
       "       'diff', 'num_lines_added', 'num_lines_deleted', 'num_lines_in_file',\n",
       "       'num_tokens_in_file', 'complexity', 'file_change_id', 'hash',\n",
       "       'change_type', 'old_file_path', 'new_file_path', 'repo_url', 'author',\n",
       "       'committer', 'commit_message', 'merge', 'cve_id', 'cwe_id',\n",
       "       'method_change_id', 'method_name', 'method_signature',\n",
       "       'method_parameters', 'method_code', 'num_lines_in_method',\n",
       "       'method_complexity', 'num_tokens_in_method', 'vulnerable'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9635ad5c-3093-4c59-8a18-271d7c928e5b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "36d6d957-4cde-4e86-8c5c-8b3612b5deef",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "id": "b18fb878-bf8a-4b9a-beb0-0554d9d3adc4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(19575, 31)"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "me_c_cpp = merged_df[(merged_df['programming_language'] == 'C') | (merged_df['programming_language'] == 'C++')]\n",
    "me_c_cpp_shape = me_c_cpp.shape\n",
    "me_c_cpp_shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "id": "981e59a2-9bc4-41ac-aa88-0d2fe99ac84a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>programming_language</th>\n",
       "      <th>code_before</th>\n",
       "      <th>code_after</th>\n",
       "      <th>diff</th>\n",
       "      <th>num_lines_added</th>\n",
       "      <th>num_lines_deleted</th>\n",
       "      <th>num_lines_in_file</th>\n",
       "      <th>num_tokens_in_file</th>\n",
       "      <th>complexity</th>\n",
       "      <th>...</th>\n",
       "      <th>cwe_id</th>\n",
       "      <th>method_change_id</th>\n",
       "      <th>method_name</th>\n",
       "      <th>method_signature</th>\n",
       "      <th>method_parameters</th>\n",
       "      <th>method_code</th>\n",
       "      <th>num_lines_in_method</th>\n",
       "      <th>method_complexity</th>\n",
       "      <th>num_tokens_in_method</th>\n",
       "      <th>vulnerable</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dl-load.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>{'added': [(152, '  const char *const start = ...</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>952.0</td>\n",
       "      <td>6592.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>...</td>\n",
       "      <td>CWE-252</td>\n",
       "      <td>241941308837203</td>\n",
       "      <td>expand_dynamic_string_token</td>\n",
       "      <td>expand_dynamic_string_token( struct link_map *...</td>\n",
       "      <td>['l', 's']</td>\n",
       "      <td>expand_dynamic_string_token (struct link_map *...</td>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "      <td>95</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dl-load.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>{'added': [(152, '  const char *const start = ...</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>952.0</td>\n",
       "      <td>6592.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>...</td>\n",
       "      <td>CWE-252</td>\n",
       "      <td>235368100398254</td>\n",
       "      <td>_dl_dst_substitute</td>\n",
       "      <td>_dl_dst_substitute( struct link_map * l , cons...</td>\n",
       "      <td>['l', 'name', 'result', 'is_path']</td>\n",
       "      <td>_dl_dst_substitute (struct link_map *l, const ...</td>\n",
       "      <td>51</td>\n",
       "      <td>22</td>\n",
       "      <td>349</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>dl-load.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>{'added': [(152, '  const char *const start = ...</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>952.0</td>\n",
       "      <td>6592.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>...</td>\n",
       "      <td>CWE-252</td>\n",
       "      <td>161060042440601</td>\n",
       "      <td>_dl_dst_count</td>\n",
       "      <td>_dl_dst_count( const char * name , int is_path)</td>\n",
       "      <td>['name', 'is_path']</td>\n",
       "      <td>_dl_dst_count (const char *name, int is_path)\\...</td>\n",
       "      <td>22</td>\n",
       "      <td>13</td>\n",
       "      <td>199</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dl-load.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>{'added': [(152, '  const char *const start = ...</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>952.0</td>\n",
       "      <td>6592.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>...</td>\n",
       "      <td>CWE-252</td>\n",
       "      <td>205987001054463</td>\n",
       "      <td>expand_dynamic_string_token</td>\n",
       "      <td>expand_dynamic_string_token( struct link_map *...</td>\n",
       "      <td>['l', 's']</td>\n",
       "      <td>expand_dynamic_string_token (struct link_map *...</td>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "      <td>95</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>dl-load.c</td>\n",
       "      <td>C</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>/* Map in a shared object's segments from the ...</td>\n",
       "      <td>{'added': [(152, '  const char *const start = ...</td>\n",
       "      <td>28</td>\n",
       "      <td>10</td>\n",
       "      <td>952.0</td>\n",
       "      <td>6592.0</td>\n",
       "      <td>260.0</td>\n",
       "      <td>...</td>\n",
       "      <td>CWE-252</td>\n",
       "      <td>38197852149195</td>\n",
       "      <td>_dl_dst_substitute</td>\n",
       "      <td>_dl_dst_substitute( struct link_map * l , cons...</td>\n",
       "      <td>['l', 'name', 'result', 'is_path']</td>\n",
       "      <td>_dl_dst_substitute (struct link_map *l, const ...</td>\n",
       "      <td>57</td>\n",
       "      <td>28</td>\n",
       "      <td>399</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   file_name programming_language  \\\n",
       "0  dl-load.c                    C   \n",
       "1  dl-load.c                    C   \n",
       "2  dl-load.c                    C   \n",
       "3  dl-load.c                    C   \n",
       "4  dl-load.c                    C   \n",
       "\n",
       "                                         code_before  \\\n",
       "0  /* Map in a shared object's segments from the ...   \n",
       "1  /* Map in a shared object's segments from the ...   \n",
       "2  /* Map in a shared object's segments from the ...   \n",
       "3  /* Map in a shared object's segments from the ...   \n",
       "4  /* Map in a shared object's segments from the ...   \n",
       "\n",
       "                                          code_after  \\\n",
       "0  /* Map in a shared object's segments from the ...   \n",
       "1  /* Map in a shared object's segments from the ...   \n",
       "2  /* Map in a shared object's segments from the ...   \n",
       "3  /* Map in a shared object's segments from the ...   \n",
       "4  /* Map in a shared object's segments from the ...   \n",
       "\n",
       "                                                diff num_lines_added  \\\n",
       "0  {'added': [(152, '  const char *const start = ...              28   \n",
       "1  {'added': [(152, '  const char *const start = ...              28   \n",
       "2  {'added': [(152, '  const char *const start = ...              28   \n",
       "3  {'added': [(152, '  const char *const start = ...              28   \n",
       "4  {'added': [(152, '  const char *const start = ...              28   \n",
       "\n",
       "  num_lines_deleted num_lines_in_file num_tokens_in_file complexity  ...  \\\n",
       "0                10             952.0             6592.0      260.0  ...   \n",
       "1                10             952.0             6592.0      260.0  ...   \n",
       "2                10             952.0             6592.0      260.0  ...   \n",
       "3                10             952.0             6592.0      260.0  ...   \n",
       "4                10             952.0             6592.0      260.0  ...   \n",
       "\n",
       "    cwe_id method_change_id                  method_name  \\\n",
       "0  CWE-252  241941308837203  expand_dynamic_string_token   \n",
       "1  CWE-252  235368100398254           _dl_dst_substitute   \n",
       "2  CWE-252  161060042440601                _dl_dst_count   \n",
       "3  CWE-252  205987001054463  expand_dynamic_string_token   \n",
       "4  CWE-252   38197852149195           _dl_dst_substitute   \n",
       "\n",
       "                                    method_signature  \\\n",
       "0  expand_dynamic_string_token( struct link_map *...   \n",
       "1  _dl_dst_substitute( struct link_map * l , cons...   \n",
       "2    _dl_dst_count( const char * name , int is_path)   \n",
       "3  expand_dynamic_string_token( struct link_map *...   \n",
       "4  _dl_dst_substitute( struct link_map * l , cons...   \n",
       "\n",
       "                    method_parameters  \\\n",
       "0                          ['l', 's']   \n",
       "1  ['l', 'name', 'result', 'is_path']   \n",
       "2                 ['name', 'is_path']   \n",
       "3                          ['l', 's']   \n",
       "4  ['l', 'name', 'result', 'is_path']   \n",
       "\n",
       "                                         method_code num_lines_in_method  \\\n",
       "0  expand_dynamic_string_token (struct link_map *...                  14   \n",
       "1  _dl_dst_substitute (struct link_map *l, const ...                  51   \n",
       "2  _dl_dst_count (const char *name, int is_path)\\...                  22   \n",
       "3  expand_dynamic_string_token (struct link_map *...                  14   \n",
       "4  _dl_dst_substitute (struct link_map *l, const ...                  57   \n",
       "\n",
       "  method_complexity num_tokens_in_method vulnerable  \n",
       "0                 3                   95       True  \n",
       "1                22                  349       True  \n",
       "2                13                  199       True  \n",
       "3                 3                   95      False  \n",
       "4                28                  399      False  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "me_c_cpp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "e7aed737-e992-4d76-ae8a-df7157d724a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['file_name', 'programming_language', 'code_before', 'code_after',\n",
       "       'diff', 'num_lines_added', 'num_lines_deleted', 'num_lines_in_file',\n",
       "       'num_tokens_in_file', 'complexity', 'file_change_id', 'hash',\n",
       "       'change_type', 'old_file_path', 'new_file_path', 'repo_url', 'author',\n",
       "       'committer', 'commit_message', 'merge', 'cve_id', 'cwe_id',\n",
       "       'method_change_id', 'method_name', 'method_signature',\n",
       "       'method_parameters', 'method_code', 'num_lines_in_method',\n",
       "       'method_complexity', 'num_tokens_in_method', 'vulnerable'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "me_c_cpp.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91ade5eb-3e01-42fb-8927-1bf9ba7100c8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "id": "550fe975-69e6-4252-88ab-e76449de4c00",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "vulnerable\n",
       "False    11349\n",
       "True      8226\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "me_c_cpp.vulnerable.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c8814c4-2596-45ff-82be-314eeaea0064",
   "metadata": {},
   "outputs": [],
   "source": [
    "me_c_cpp.to_csv('CVEFixes_c.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf8e4b2c-e439-486d-b050-38cd23149d8c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "768b039c-c152-402d-abfe-c633e1ddcbad",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4201, 31)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "me_py = merged_df[merged_df.programming_language == 'Python']\n",
    "me_py.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "dcda7c30-d448-49ed-aac7-fc344d73f7e2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "me_py.to_csv('CVEFixes_python.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "d8be9a95-b4e3-4829-93d4-e184bc860796",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13840, 31)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "me_c = merged_df[merged_df.programming_language == 'C']\n",
    "me_c.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "cbe12ccc-df2a-42ed-94c5-0631a0c5751c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "me_c.to_csv('CVEFixes_c.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5b1cfb45-33e9-42f7-8a01-7f48bb171661",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3855, 31)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "me_ja = merged_df[merged_df.programming_language == 'Java']\n",
    "me_ja.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "837fb8a9-9a39-4d27-bd7e-4ce4b9563ad3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "me_ja.to_csv('CVEFixes_java.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "fb55f924-df4e-4fed-85f3-8220692e3968",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4393, 31)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "me_cpp = merged_df[merged_df.programming_language == 'C++']\n",
    "me_cpp.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "8a7d2971-7c28-4f3b-944a-22d1110b2599",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "me_cpp.to_csv('CVEFixes_c++.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "id": "f82f558b-7a43-4bba-a29c-535d2a14c857",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3056, 31)"
      ]
     },
     "execution_count": 247,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "me_ru = merged_df[merged_df.programming_language == 'Ruby']\n",
    "me_ru.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "id": "28eefe12-5b8a-4cfc-acca-93868d57456b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "me_ru.to_csv('CVEFixes_ruby.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "id": "d3a50f79-4261-4e64-8d4b-01b782e05b64",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2274, 31)"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "me_go = merged_df[merged_df.programming_language == 'Go']\n",
    "me_go.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "id": "a47930bc-de9d-4e35-9fc9-922f12a69e9c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "me_go.to_csv('CVEFixes_go.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "995b5f96-2e27-49b3-b812-9ed9afed9120",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    7461293048740\n",
       "1    7461293048740\n",
       "2    7461293048740\n",
       "3    7461293048740\n",
       "4    7461293048740\n",
       "5    7461293048740\n",
       "Name: file_change_id, dtype: object"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df[merged_df['file_change_id']=='7461293048740'].file_change_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "bfe941bc-686f-46b7-9f8b-43a4b27cfff8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7461293048740\n"
     ]
    }
   ],
   "source": [
    "print(merged_df['file_change_id'].iloc[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "id": "bf6031c3-d07c-4170-b2e7-b56fecbc9b8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "merged_df = me_c_cpp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "id": "9d56f99c-dc60-4042-8939-5cff8ecef59b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['file_name', 'programming_language', 'code_before', 'code_after',\n",
       "       'diff', 'num_lines_added', 'num_lines_deleted', 'num_lines_in_file',\n",
       "       'num_tokens_in_file', 'complexity', 'change_type', 'old_file_path',\n",
       "       'new_file_path', 'repo_url', 'commit_message', 'merge', 'cve_id',\n",
       "       'cwe_id', 'method_change_id', 'method_name', 'method_signature',\n",
       "       'method_parameters', 'method_code', 'num_lines_in_method',\n",
       "       'method_complexity', 'num_tokens_in_method', 'vulnerable'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "me1 = merged_df.drop(['committer', 'author', 'file_change_id', 'hash'], axis=1)\n",
    "me1.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "id": "149ef7d3-e326-4185-bb49-b51adeba03ec",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "me_py = pd.read_csv('CVEFixes_c++.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "id": "a583efa1-749a-44ca-978a-091314619733",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>programming_language</th>\n",
       "      <th>code_before</th>\n",
       "      <th>code_after</th>\n",
       "      <th>diff</th>\n",
       "      <th>num_lines_added</th>\n",
       "      <th>num_lines_deleted</th>\n",
       "      <th>num_lines_in_file</th>\n",
       "      <th>num_tokens_in_file</th>\n",
       "      <th>complexity</th>\n",
       "      <th>...</th>\n",
       "      <th>cwe_id</th>\n",
       "      <th>method_change_id</th>\n",
       "      <th>method_name</th>\n",
       "      <th>method_signature</th>\n",
       "      <th>method_parameters</th>\n",
       "      <th>method_code</th>\n",
       "      <th>num_lines_in_method</th>\n",
       "      <th>method_complexity</th>\n",
       "      <th>num_tokens_in_method</th>\n",
       "      <th>vulnerable</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>saver.cpp</td>\n",
       "      <td>C++</td>\n",
       "      <td>#include &lt;pwd.h&gt;\\n#include &lt;unistd.h&gt;\\n#includ...</td>\n",
       "      <td>#include &lt;pwd.h&gt;\\n#include &lt;unistd.h&gt;\\n#includ...</td>\n",
       "      <td>{'added': [(154, '            timer.stop();')]...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>197.0</td>\n",
       "      <td>1132.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>...</td>\n",
       "      <td>['NVD-CWE-Other']</td>\n",
       "      <td>2.759389e+13</td>\n",
       "      <td>KPasswordDlg::keyPressed</td>\n",
       "      <td>KPasswordDlg::keyPressed( QKeyEvent * e)</td>\n",
       "      <td>['e']</td>\n",
       "      <td>void KPasswordDlg::keyPressed( QKeyEvent *e )\\...</td>\n",
       "      <td>44.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>perf_event_v6.c</td>\n",
       "      <td>C++</td>\n",
       "      <td>/*\\n * ARMv6 Performance counter handling code...</td>\n",
       "      <td>/*\\n * ARMv6 Performance counter handling code...</td>\n",
       "      <td>{'added': [(482, '\\t\\tif (perf_event_overflow(...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>498.0</td>\n",
       "      <td>2757.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>...</td>\n",
       "      <td>['CWE-400']</td>\n",
       "      <td>1.609609e+14</td>\n",
       "      <td>armv6pmu_handle_irq</td>\n",
       "      <td>armv6pmu_handle_irq( int irq_num , void * dev)</td>\n",
       "      <td>['irq_num', 'dev']</td>\n",
       "      <td>armv6pmu_handle_irq(int irq_num,\\n\\t\\t    void...</td>\n",
       "      <td>32.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>206.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>perf_event_v6.c</td>\n",
       "      <td>C++</td>\n",
       "      <td>/*\\n * ARMv6 Performance counter handling code...</td>\n",
       "      <td>/*\\n * ARMv6 Performance counter handling code...</td>\n",
       "      <td>{'added': [(482, '\\t\\tif (perf_event_overflow(...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>498.0</td>\n",
       "      <td>2757.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>...</td>\n",
       "      <td>['CWE-400']</td>\n",
       "      <td>9.293469e+13</td>\n",
       "      <td>armv6pmu_handle_irq</td>\n",
       "      <td>armv6pmu_handle_irq( int irq_num , void * dev)</td>\n",
       "      <td>['irq_num', 'dev']</td>\n",
       "      <td>armv6pmu_handle_irq(int irq_num,\\n\\t\\t    void...</td>\n",
       "      <td>32.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>204.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>perf_event_intel.c</td>\n",
       "      <td>C++</td>\n",
       "      <td>#ifdef CONFIG_CPU_SUP_INTEL\\n\\n#define MAX_EXT...</td>\n",
       "      <td>#ifdef CONFIG_CPU_SUP_INTEL\\n\\n#define MAX_EXT...</td>\n",
       "      <td>{'added': [(1006, '\\t\\tif (perf_event_overflow...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1132.0</td>\n",
       "      <td>6756.0</td>\n",
       "      <td>121.0</td>\n",
       "      <td>...</td>\n",
       "      <td>['CWE-400']</td>\n",
       "      <td>4.727234e+13</td>\n",
       "      <td>intel_pmu_handle_irq</td>\n",
       "      <td>intel_pmu_handle_irq( struct pt_regs * regs)</td>\n",
       "      <td>['regs']</td>\n",
       "      <td>static int intel_pmu_handle_irq(struct pt_regs...</td>\n",
       "      <td>50.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>266.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>perf_event_intel.c</td>\n",
       "      <td>C++</td>\n",
       "      <td>#ifdef CONFIG_CPU_SUP_INTEL\\n\\n#define MAX_EXT...</td>\n",
       "      <td>#ifdef CONFIG_CPU_SUP_INTEL\\n\\n#define MAX_EXT...</td>\n",
       "      <td>{'added': [(1006, '\\t\\tif (perf_event_overflow...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1132.0</td>\n",
       "      <td>6756.0</td>\n",
       "      <td>121.0</td>\n",
       "      <td>...</td>\n",
       "      <td>['CWE-400']</td>\n",
       "      <td>1.460641e+14</td>\n",
       "      <td>intel_pmu_handle_irq</td>\n",
       "      <td>intel_pmu_handle_irq( struct pt_regs * regs)</td>\n",
       "      <td>['regs']</td>\n",
       "      <td>static int intel_pmu_handle_irq(struct pt_regs...</td>\n",
       "      <td>50.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>264.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            file_name programming_language  \\\n",
       "0           saver.cpp                  C++   \n",
       "1     perf_event_v6.c                  C++   \n",
       "2     perf_event_v6.c                  C++   \n",
       "3  perf_event_intel.c                  C++   \n",
       "4  perf_event_intel.c                  C++   \n",
       "\n",
       "                                         code_before  \\\n",
       "0  #include <pwd.h>\\n#include <unistd.h>\\n#includ...   \n",
       "1  /*\\n * ARMv6 Performance counter handling code...   \n",
       "2  /*\\n * ARMv6 Performance counter handling code...   \n",
       "3  #ifdef CONFIG_CPU_SUP_INTEL\\n\\n#define MAX_EXT...   \n",
       "4  #ifdef CONFIG_CPU_SUP_INTEL\\n\\n#define MAX_EXT...   \n",
       "\n",
       "                                          code_after  \\\n",
       "0  #include <pwd.h>\\n#include <unistd.h>\\n#includ...   \n",
       "1  /*\\n * ARMv6 Performance counter handling code...   \n",
       "2  /*\\n * ARMv6 Performance counter handling code...   \n",
       "3  #ifdef CONFIG_CPU_SUP_INTEL\\n\\n#define MAX_EXT...   \n",
       "4  #ifdef CONFIG_CPU_SUP_INTEL\\n\\n#define MAX_EXT...   \n",
       "\n",
       "                                                diff  num_lines_added  \\\n",
       "0  {'added': [(154, '            timer.stop();')]...              1.0   \n",
       "1  {'added': [(482, '\\t\\tif (perf_event_overflow(...              1.0   \n",
       "2  {'added': [(482, '\\t\\tif (perf_event_overflow(...              1.0   \n",
       "3  {'added': [(1006, '\\t\\tif (perf_event_overflow...              1.0   \n",
       "4  {'added': [(1006, '\\t\\tif (perf_event_overflow...              1.0   \n",
       "\n",
       "   num_lines_deleted  num_lines_in_file  num_tokens_in_file  complexity  ...  \\\n",
       "0                0.0              197.0              1132.0        34.0  ...   \n",
       "1                1.0              498.0              2757.0        45.0  ...   \n",
       "2                1.0              498.0              2757.0        45.0  ...   \n",
       "3                1.0             1132.0              6756.0       121.0  ...   \n",
       "4                1.0             1132.0              6756.0       121.0  ...   \n",
       "\n",
       "              cwe_id method_change_id               method_name  \\\n",
       "0  ['NVD-CWE-Other']     2.759389e+13  KPasswordDlg::keyPressed   \n",
       "1        ['CWE-400']     1.609609e+14       armv6pmu_handle_irq   \n",
       "2        ['CWE-400']     9.293469e+13       armv6pmu_handle_irq   \n",
       "3        ['CWE-400']     4.727234e+13      intel_pmu_handle_irq   \n",
       "4        ['CWE-400']     1.460641e+14      intel_pmu_handle_irq   \n",
       "\n",
       "                                 method_signature   method_parameters  \\\n",
       "0        KPasswordDlg::keyPressed( QKeyEvent * e)               ['e']   \n",
       "1  armv6pmu_handle_irq( int irq_num , void * dev)  ['irq_num', 'dev']   \n",
       "2  armv6pmu_handle_irq( int irq_num , void * dev)  ['irq_num', 'dev']   \n",
       "3    intel_pmu_handle_irq( struct pt_regs * regs)            ['regs']   \n",
       "4    intel_pmu_handle_irq( struct pt_regs * regs)            ['regs']   \n",
       "\n",
       "                                         method_code num_lines_in_method  \\\n",
       "0  void KPasswordDlg::keyPressed( QKeyEvent *e )\\...                44.0   \n",
       "1  armv6pmu_handle_irq(int irq_num,\\n\\t\\t    void...                32.0   \n",
       "2  armv6pmu_handle_irq(int irq_num,\\n\\t\\t    void...                32.0   \n",
       "3  static int intel_pmu_handle_irq(struct pt_regs...                50.0   \n",
       "4  static int intel_pmu_handle_irq(struct pt_regs...                50.0   \n",
       "\n",
       "  method_complexity num_tokens_in_method vulnerable  \n",
       "0              10.0                184.0      False  \n",
       "1               7.0                206.0       True  \n",
       "2               7.0                204.0      False  \n",
       "3               8.0                266.0       True  \n",
       "4               8.0                264.0      False  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 295,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "me_py.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "id": "cf47c775-7c3c-429e-9404-8faf59f755c4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['file_name', 'programming_language', 'code_before', 'code_after',\n",
       "       'diff', 'num_lines_added', 'num_lines_deleted', 'num_lines_in_file',\n",
       "       'num_tokens_in_file', 'complexity', 'file_change_id', 'hash',\n",
       "       'change_type', 'old_file_path', 'new_file_path', 'repo_url', 'author',\n",
       "       'committer', 'commit_message', 'merge', 'cve_id', 'cwe_id',\n",
       "       'method_change_id', 'method_name', 'method_signature',\n",
       "       'method_parameters', 'method_code', 'num_lines_in_method',\n",
       "       'method_complexity', 'num_tokens_in_method', 'vulnerable'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 296,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "me_py.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "id": "2450f91f-9775-4698-a987-5f51803b185d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "vulnerable\n",
       "False    2643\n",
       "True     1746\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 297,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "me_py.vulnerable.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "228ffd83-e0b0-4c86-abdd-eeae0a5981af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(116061, 31)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "id": "c1c4a9a4-1461-442e-b44b-ebf79ec8ee70",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_174155/1159927496.py:31: DeprecationWarning: DataFrameGroupBy.apply operated on the grouping columns. This behavior is deprecated, and in a future version of pandas the grouping columns will be excluded from the operation. Either pass `include_groups=False` to exclude the groupings or explicitly select the grouping columns after groupby to silence this warning.\n",
      "  result_df = filtered_df.groupby(['file_change_id', 'method_name', 'hash']).apply(process_group).dropna().reset_index(drop=True)\n"
     ]
    }
   ],
   "source": [
    "filtered_df = merged_df.groupby('file_change_id').filter(lambda x: len(x) >= 2)\n",
    "filtered_df = filtered_df.groupby(['file_change_id', 'method_name', 'hash']).filter(lambda x: len(x) >= 2)\n",
    "\n",
    "def process_group(group):\n",
    "    if len(group) >= 2 and group['vulnerable'].nunique() == 2:\n",
    "        function_before = group.loc[group['vulnerable'] == \"True\"].iloc[0]\n",
    "        function_after = group.loc[group['vulnerable'] == \"False\"].iloc[0]\n",
    "\n",
    "        return pd.Series({\n",
    "            'file_name': function_before['file_name'],\n",
    "            'programming_language': function_before['programming_language'],\n",
    "            'method_name': function_before['method_name'],\n",
    "            'code_before': function_before['code_before'],\n",
    "            'code_after': function_before['code_after'],\n",
    "            'func_before': function_before['method_code'],\n",
    "            'func_after': function_after['method_code'],\n",
    "            'diff': function_before['diff'],\n",
    "            'num_lines_added': function_before['num_lines_added'],\n",
    "            'num_lines_deleted': function_before['num_lines_deleted'],\n",
    "            'num_lines_in_file': function_before['num_lines_in_file'],\n",
    "            'num_tokens_in_file': function_before['num_tokens_in_file'],\n",
    "            'num_lines_in_method': function_before['num_lines_in_method'],\n",
    "            'num_tokens_in_method': function_before['num_tokens_in_method'],\n",
    "            'method_complexity': function_before['method_complexity'],\n",
    "            'repo': function_before['repo_url'],\n",
    "            'cve_id': function_before['cve_id'],\n",
    "            'cwe_id': function_before['cwe_id'],\n",
    "        })\n",
    "\n",
    "# Apply the function to each group and drop any None results\n",
    "result_df = filtered_df.groupby(['file_change_id', 'method_name', 'hash']).apply(process_group).dropna().reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "id": "a49c176e-63fe-4c62-a2f6-d0cb653f3d8a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5937, 18)"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "id": "8a2c9baf-f732-4aea-aadb-48cfb030b79a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>programming_language</th>\n",
       "      <th>method_name</th>\n",
       "      <th>code_before</th>\n",
       "      <th>code_after</th>\n",
       "      <th>func_before</th>\n",
       "      <th>func_after</th>\n",
       "      <th>diff</th>\n",
       "      <th>num_lines_added</th>\n",
       "      <th>num_lines_deleted</th>\n",
       "      <th>num_lines_in_file</th>\n",
       "      <th>num_tokens_in_file</th>\n",
       "      <th>num_lines_in_method</th>\n",
       "      <th>num_tokens_in_method</th>\n",
       "      <th>method_complexity</th>\n",
       "      <th>repo</th>\n",
       "      <th>cve_id</th>\n",
       "      <th>cwe_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>check_securechannel.c</td>\n",
       "      <td>C</td>\n",
       "      <td>setup_secureChannel</td>\n",
       "      <td>/* This Source Code Form is subject to the ter...</td>\n",
       "      <td>/* This Source Code Form is subject to the ter...</td>\n",
       "      <td>setup_secureChannel(void) {\\n    TestingPolicy...</td>\n",
       "      <td>setup_secureChannel(void) {\\n    TestingPolicy...</td>\n",
       "      <td>{'added': [(48, '    testingConnection ='), (4...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>436</td>\n",
       "      <td>3007</td>\n",
       "      <td>9</td>\n",
       "      <td>73</td>\n",
       "      <td>1</td>\n",
       "      <td>https://github.com/open62541/open62541</td>\n",
       "      <td>CVE-2022-25761</td>\n",
       "      <td>CWE-770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>codegen.c</td>\n",
       "      <td>C</td>\n",
       "      <td>gen_assignment</td>\n",
       "      <td>/*\\n** codegen.c - mruby code generator\\n**\\n*...</td>\n",
       "      <td>/*\\n** codegen.c - mruby code generator\\n**\\n*...</td>\n",
       "      <td>gen_assignment(codegen_scope *s, node *tree, n...</td>\n",
       "      <td>gen_assignment(codegen_scope *s, node *tree, n...</td>\n",
       "      <td>{'added': [(1907, '      if (n &lt; 15) {'), (190...</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>3530</td>\n",
       "      <td>23087</td>\n",
       "      <td>162</td>\n",
       "      <td>918</td>\n",
       "      <td>48</td>\n",
       "      <td>https://github.com/mruby/mruby</td>\n",
       "      <td>CVE-2022-0717</td>\n",
       "      <td>CWE-125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>valid.c</td>\n",
       "      <td>C</td>\n",
       "      <td>xmlAddID</td>\n",
       "      <td>/*\\n * valid.c : part of the code use to do th...</td>\n",
       "      <td>/*\\n * valid.c : part of the code use to do th...</td>\n",
       "      <td>xmlAddID(xmlValidCtxtPtr ctxt, xmlDocPtr doc, ...</td>\n",
       "      <td>xmlAddID(xmlValidCtxtPtr ctxt, xmlDocPtr doc, ...</td>\n",
       "      <td>{'added': [(482, '/**'), (483, ' * xmlValidNor...</td>\n",
       "      <td>55</td>\n",
       "      <td>33</td>\n",
       "      <td>4809.0</td>\n",
       "      <td>30269.0</td>\n",
       "      <td>52</td>\n",
       "      <td>320</td>\n",
       "      <td>14</td>\n",
       "      <td>https://github.com/GNOME/libxml2</td>\n",
       "      <td>CVE-2022-23308</td>\n",
       "      <td>CWE-416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>valid.c</td>\n",
       "      <td>C</td>\n",
       "      <td>xmlAddRef</td>\n",
       "      <td>/*\\n * valid.c : part of the code use to do th...</td>\n",
       "      <td>/*\\n * valid.c : part of the code use to do th...</td>\n",
       "      <td>xmlAddRef(xmlValidCtxtPtr ctxt, xmlDocPtr doc,...</td>\n",
       "      <td>xmlAddRef(xmlValidCtxtPtr ctxt, xmlDocPtr doc,...</td>\n",
       "      <td>{'added': [(482, '/**'), (483, ' * xmlValidNor...</td>\n",
       "      <td>55</td>\n",
       "      <td>33</td>\n",
       "      <td>4809.0</td>\n",
       "      <td>30269.0</td>\n",
       "      <td>69</td>\n",
       "      <td>393</td>\n",
       "      <td>16</td>\n",
       "      <td>https://github.com/GNOME/libxml2</td>\n",
       "      <td>CVE-2022-23308</td>\n",
       "      <td>CWE-416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>valid.c</td>\n",
       "      <td>C</td>\n",
       "      <td>xmlValidCtxtNormalizeAttributeValue</td>\n",
       "      <td>/*\\n * valid.c : part of the code use to do th...</td>\n",
       "      <td>/*\\n * valid.c : part of the code use to do th...</td>\n",
       "      <td>xmlValidCtxtNormalizeAttributeValue(xmlValidCt...</td>\n",
       "      <td>xmlValidCtxtNormalizeAttributeValue(xmlValidCt...</td>\n",
       "      <td>{'added': [(482, '/**'), (483, ' * xmlValidNor...</td>\n",
       "      <td>55</td>\n",
       "      <td>33</td>\n",
       "      <td>4809.0</td>\n",
       "      <td>30269.0</td>\n",
       "      <td>60</td>\n",
       "      <td>473</td>\n",
       "      <td>29</td>\n",
       "      <td>https://github.com/GNOME/libxml2</td>\n",
       "      <td>CVE-2022-23308</td>\n",
       "      <td>CWE-416</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               file_name programming_language  \\\n",
       "0  check_securechannel.c                    C   \n",
       "1              codegen.c                    C   \n",
       "2                valid.c                    C   \n",
       "3                valid.c                    C   \n",
       "4                valid.c                    C   \n",
       "\n",
       "                           method_name  \\\n",
       "0                  setup_secureChannel   \n",
       "1                       gen_assignment   \n",
       "2                             xmlAddID   \n",
       "3                            xmlAddRef   \n",
       "4  xmlValidCtxtNormalizeAttributeValue   \n",
       "\n",
       "                                         code_before  \\\n",
       "0  /* This Source Code Form is subject to the ter...   \n",
       "1  /*\\n** codegen.c - mruby code generator\\n**\\n*...   \n",
       "2  /*\\n * valid.c : part of the code use to do th...   \n",
       "3  /*\\n * valid.c : part of the code use to do th...   \n",
       "4  /*\\n * valid.c : part of the code use to do th...   \n",
       "\n",
       "                                          code_after  \\\n",
       "0  /* This Source Code Form is subject to the ter...   \n",
       "1  /*\\n** codegen.c - mruby code generator\\n**\\n*...   \n",
       "2  /*\\n * valid.c : part of the code use to do th...   \n",
       "3  /*\\n * valid.c : part of the code use to do th...   \n",
       "4  /*\\n * valid.c : part of the code use to do th...   \n",
       "\n",
       "                                         func_before  \\\n",
       "0  setup_secureChannel(void) {\\n    TestingPolicy...   \n",
       "1  gen_assignment(codegen_scope *s, node *tree, n...   \n",
       "2  xmlAddID(xmlValidCtxtPtr ctxt, xmlDocPtr doc, ...   \n",
       "3  xmlAddRef(xmlValidCtxtPtr ctxt, xmlDocPtr doc,...   \n",
       "4  xmlValidCtxtNormalizeAttributeValue(xmlValidCt...   \n",
       "\n",
       "                                          func_after  \\\n",
       "0  setup_secureChannel(void) {\\n    TestingPolicy...   \n",
       "1  gen_assignment(codegen_scope *s, node *tree, n...   \n",
       "2  xmlAddID(xmlValidCtxtPtr ctxt, xmlDocPtr doc, ...   \n",
       "3  xmlAddRef(xmlValidCtxtPtr ctxt, xmlDocPtr doc,...   \n",
       "4  xmlValidCtxtNormalizeAttributeValue(xmlValidCt...   \n",
       "\n",
       "                                                diff num_lines_added  \\\n",
       "0  {'added': [(48, '    testingConnection ='), (4...               2   \n",
       "1  {'added': [(1907, '      if (n < 15) {'), (190...               5   \n",
       "2  {'added': [(482, '/**'), (483, ' * xmlValidNor...              55   \n",
       "3  {'added': [(482, '/**'), (483, ' * xmlValidNor...              55   \n",
       "4  {'added': [(482, '/**'), (483, ' * xmlValidNor...              55   \n",
       "\n",
       "  num_lines_deleted num_lines_in_file num_tokens_in_file num_lines_in_method  \\\n",
       "0                 1               436               3007                   9   \n",
       "1                 1              3530              23087                 162   \n",
       "2                33            4809.0            30269.0                  52   \n",
       "3                33            4809.0            30269.0                  69   \n",
       "4                33            4809.0            30269.0                  60   \n",
       "\n",
       "  num_tokens_in_method method_complexity  \\\n",
       "0                   73                 1   \n",
       "1                  918                48   \n",
       "2                  320                14   \n",
       "3                  393                16   \n",
       "4                  473                29   \n",
       "\n",
       "                                     repo          cve_id   cwe_id  \n",
       "0  https://github.com/open62541/open62541  CVE-2022-25761  CWE-770  \n",
       "1          https://github.com/mruby/mruby   CVE-2022-0717  CWE-125  \n",
       "2        https://github.com/GNOME/libxml2  CVE-2022-23308  CWE-416  \n",
       "3        https://github.com/GNOME/libxml2  CVE-2022-23308  CWE-416  \n",
       "4        https://github.com/GNOME/libxml2  CVE-2022-23308  CWE-416  "
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "id": "28e91d04-c176-4ced-a378-484810639e3d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "cwe_id\n",
       "CWE-125    918\n",
       "CWE-119    545\n",
       "CWE-787    468\n",
       "CWE-20     383\n",
       "CWE-476    324\n",
       "          ... \n",
       "CWE-428      1\n",
       "CWE-943      1\n",
       "CWE-116      1\n",
       "CWE-320      1\n",
       "CWE-94       1\n",
       "Name: count, Length: 115, dtype: int64"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df.cwe_id.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66e109e9-7a67-4b15-bfc7-2c32906f0a7e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "979fa5fa-b10f-4046-a40d-5c4addfdfc2c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3512, 18)"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "att_id = [787, 79, 89, 416, 78, 20, 125, 22, 352, 434, 862, 476, 287, 190, 502, 77, 119, 798, 918, 306, 362, 269, 94, 863, 276]\n",
    "attention_cwe = [f'CWE-{i}' for i in att_id]\n",
    "tmp_df = result_df[result_df['cwe_id'].isin(attention_cwe)]\n",
    "tmp_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "id": "0cb54594-7214-4293-a9b8-dcf802263708",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_174155/389181450.py:10: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  filtered_tmp_df['id'] = range(filtered_tmp_df.shape[0])\n"
     ]
    }
   ],
   "source": [
    "cwe_counts = tmp_df['cwe_id'].value_counts()\n",
    "\n",
    "# Filter 'cwe_id' to keep only those with less than 150 occurrences\n",
    "cwe_to_keep = cwe_counts[cwe_counts > 150].index\n",
    "\n",
    "# Filter the DataFrame based on 'cwe_id' counts\n",
    "filtered_tmp_df = tmp_df[tmp_df['cwe_id'].isin(cwe_to_keep)]\n",
    "\n",
    "# Adding 'id' column as the first column again in case it needs to be re-added\n",
    "filtered_tmp_df['id'] = range(filtered_tmp_df.shape[0])\n",
    "cols = ['id'] + [col for col in filtered_tmp_df.columns if col != 'id']\n",
    "filtered_tmp_df = filtered_tmp_df[cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "id": "ee554ebb-dd39-4687-b6d1-75b9f8ffb40f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3289, 19)"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_tmp_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "id": "cbbe04bf-69d9-44b2-8158-12cd7fc132a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "cwe_id\n",
       "CWE-125    918\n",
       "CWE-119    545\n",
       "CWE-787    468\n",
       "CWE-20     383\n",
       "CWE-476    324\n",
       "CWE-190    250\n",
       "CWE-416    207\n",
       "CWE-362    194\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 199,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_tmp_df.cwe_id.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "id": "e00020bd-68de-4afc-9d5f-6c0adce3cb95",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['id', 'file_name', 'programming_language', 'method_name', 'code_before',\n",
       "       'code_after', 'func_before', 'func_after', 'diff', 'num_lines_added',\n",
       "       'num_lines_deleted', 'num_lines_in_file', 'num_tokens_in_file',\n",
       "       'num_lines_in_method', 'num_tokens_in_method', 'method_complexity',\n",
       "       'repo', 'cve_id', 'cwe_id'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_tmp_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "id": "68f7468a-bc98-4699-bc62-0e7d13084e4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered_tmp_df.to_csv(\"cveFixes_master_c.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7dff9e08-921b-48b1-b9c4-4605de7338e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp_df['id'] = range(tmp_df.shape[0])\n",
    "\n",
    "# Reorder the columns to place 'id' first\n",
    "cols = ['id'] + [col for col in tmp_df.columns if col != 'id']\n",
    "tmp_df = tmp_df[cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "4c9636a2-82f9-4b88-addb-b3f426efc5ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1855, 18)"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwe20 = result_df[result_df['cwe_id']=='CWE-20']\n",
    "cwe20.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "fece6f5e-4928-4e1e-a9bf-f433a57cde17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "programming_language\n",
       "JavaScript     1120\n",
       "C               287\n",
       "PHP             157\n",
       "C++              96\n",
       "Python           66\n",
       "Ruby             54\n",
       "Java             24\n",
       "TypeScript       24\n",
       "Go               10\n",
       "Objective-C       7\n",
       "C#                6\n",
       "Rust              1\n",
       "SQL               1\n",
       "Haskell           1\n",
       "Lua               1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwe20.programming_language.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "id": "6b5fea15-f31b-418b-8c63-c76a45086339",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['file_name', 'programming_language', 'method_name', 'code_before',\n",
       "       'code_after', 'func_before', 'func_after', 'diff', 'num_lines_added',\n",
       "       'num_lines_deleted', 'num_lines_in_file', 'num_tokens_in_file',\n",
       "       'num_lines_in_method', 'num_tokens_in_method', 'method_complexity',\n",
       "       'repo', 'cve_id', 'cwe_id'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwe20.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "bdc85b8a-9fde-42a7-8de5-102d9890e4a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_174155/3530366171.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cwe20.num_lines_in_method = cwe20.num_lines_in_method.apply(ast.literal_eval)\n"
     ]
    }
   ],
   "source": [
    "import ast\n",
    "# cwe20.num_tokens_in_file = cwe20.num_tokens_in_file.apply(ast.literal_eval)\n",
    "# cwe20.num_lines_in_file = cwe20.num_lines_in_file.apply(str)\n",
    "# cwe20.num_lines_in_file = cwe20.num_lines_in_file.apply(ast.literal_eval)\n",
    "cwe20.num_lines_in_method = cwe20.num_lines_in_method.apply(ast.literal_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "17d21f59-237a-4e7f-bf76-7151ae721b30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    1.855000e+03\n",
       "mean     9.398365e+05\n",
       "std      9.331303e+05\n",
       "min      5.700000e+01\n",
       "25%      3.157000e+03\n",
       "50%      6.038870e+05\n",
       "75%      1.471078e+06\n",
       "max      2.382875e+06\n",
       "Name: num_tokens_in_file, dtype: float64"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwe20.num_tokens_in_file.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "181b4663-1c1d-46ec-ac17-4893b69a627e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count     1855.000000\n",
       "mean      7708.246900\n",
       "std       7086.544517\n",
       "min          1.000000\n",
       "25%        511.000000\n",
       "50%       7043.000000\n",
       "75%      13252.000000\n",
       "max      17975.000000\n",
       "Name: num_lines_in_file, dtype: float64"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwe20.num_lines_in_file.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "40ba8274-9780-4d5a-b68c-7900f1585307",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1597, 18)"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwe21 = cwe20[(cwe20['num_lines_in_method'] < 40)]\n",
    "cwe21.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "id": "41bd7f7e-758c-4603-aaaa-e0b118c13f30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "programming_language\n",
       "JavaScript     1113\n",
       "PHP             144\n",
       "C               135\n",
       "Python           54\n",
       "Ruby             53\n",
       "C++              52\n",
       "Java             20\n",
       "TypeScript       17\n",
       "Go                6\n",
       "Objective-C       2\n",
       "C#                1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwe21.programming_language.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "4b1bf113-9111-4d1e-b713-e9a4865a6815",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(493, 18)"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwe_sub = cwe21[(cwe21['programming_language'].str.contains('|'.join(['Ruby', 'PHP', 'Python', 'C'])))]\n",
    "cwe_sub.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "39ef761d-f9a9-49c3-beaf-50ba6d356d4c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "programming_language\n",
       "C              167\n",
       "PHP            151\n",
       "C++             59\n",
       "Python          57\n",
       "Ruby            54\n",
       "C#               3\n",
       "Objective-C      2\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwe_sub.programming_language.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "id": "78e2871f-c5b9-4b40-805b-f5d76bbd1c2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "cwe_sub = cwe_sub[cwe_sub['programming_language'] != 'Objective-C']\n",
    "cwe_sub = cwe_sub[cwe_sub['programming_language'] != 'JavaScript']\n",
    "cwe_sub = cwe_sub[cwe_sub['programming_language'] != 'C#']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "622420d4-bfd4-491d-9bec-cd653f15053c",
   "metadata": {},
   "outputs": [],
   "source": [
    "min_lines_per_language = cwe_sub.groupby('programming_language')['num_lines_in_method'].min()\n",
    "\n",
    "cwe_smallest = pd.concat([cwe_sub[cwe_sub['programming_language'] == lang].nsmallest(50, 'num_lines_in_method') for lang in min_lines_per_language.index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "f4ae1f53-5f56-4827-b7ac-cf4c31d5a9cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "programming_language\n",
       "C         50\n",
       "C++       50\n",
       "PHP       50\n",
       "Python    50\n",
       "Ruby      50\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwe_smallest.programming_language.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "8e6c35bd-e2ba-4031-a5c8-6d4fca39592a",
   "metadata": {},
   "outputs": [],
   "source": [
    "cwe_smallest.to_csv('q3_test.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "04131f5c-1238-4778-8cc1-0387f4db1db9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>programming_language</th>\n",
       "      <th>method_name</th>\n",
       "      <th>code_before</th>\n",
       "      <th>code_after</th>\n",
       "      <th>func_before</th>\n",
       "      <th>func_after</th>\n",
       "      <th>diff</th>\n",
       "      <th>num_lines_added</th>\n",
       "      <th>num_lines_deleted</th>\n",
       "      <th>num_lines_in_file</th>\n",
       "      <th>num_tokens_in_file</th>\n",
       "      <th>num_lines_in_method</th>\n",
       "      <th>num_tokens_in_method</th>\n",
       "      <th>method_complexity</th>\n",
       "      <th>repo</th>\n",
       "      <th>cve_id</th>\n",
       "      <th>cwe_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16189</th>\n",
       "      <td>nf_conntrack_proto_generic.c</td>\n",
       "      <td>C</td>\n",
       "      <td>generic_new</td>\n",
       "      <td>/* (C) 1999-2001 Paul `Rusty' Russell\\n * (C) ...</td>\n",
       "      <td>/* (C) 1999-2001 Paul `Rusty' Russell\\n * (C) ...</td>\n",
       "      <td>static bool generic_new(struct nf_conn *ct, co...</td>\n",
       "      <td>static bool generic_new(struct nf_conn *ct, co...</td>\n",
       "      <td>{'added': [(17, 'static bool nf_generic_should...</td>\n",
       "      <td>25</td>\n",
       "      <td>1</td>\n",
       "      <td>173.0</td>\n",
       "      <td>865.0</td>\n",
       "      <td>5</td>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2014-8160</td>\n",
       "      <td>CWE-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>534</th>\n",
       "      <td>jas_icc.c</td>\n",
       "      <td>C</td>\n",
       "      <td>jas_iccputsint</td>\n",
       "      <td>/*\\n * Copyright (c) 2002-2003 Michael David A...</td>\n",
       "      <td>/*\\n * Copyright (c) 2002-2003 Michael David A...</td>\n",
       "      <td>static int jas_iccputsint(jas_stream_t *out, i...</td>\n",
       "      <td>static int jas_iccputsint(jas_stream_t *out, i...</td>\n",
       "      <td>{'added': [(85, 'static int jas_iccgetuint(jas...</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>1502.0</td>\n",
       "      <td>10066.0</td>\n",
       "      <td>6</td>\n",
       "      <td>45</td>\n",
       "      <td>2</td>\n",
       "      <td>https://github.com/mdadams/jasper</td>\n",
       "      <td>CVE-2016-9395</td>\n",
       "      <td>CWE-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5502</th>\n",
       "      <td>segment.c</td>\n",
       "      <td>C</td>\n",
       "      <td>f2fs_wait_discard_bios</td>\n",
       "      <td>/*\\n * fs/f2fs/segment.c\\n *\\n * Copyright (c)...</td>\n",
       "      <td>/*\\n * fs/f2fs/segment.c\\n *\\n * Copyright (c)...</td>\n",
       "      <td>void f2fs_wait_discard_bios(struct f2fs_sb_inf...</td>\n",
       "      <td>void f2fs_wait_discard_bios(struct f2fs_sb_inf...</td>\n",
       "      <td>{'added': [(1213, 'void f2fs_wait_discard_bios...</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2836.0</td>\n",
       "      <td>18721.0</td>\n",
       "      <td>6</td>\n",
       "      <td>28</td>\n",
       "      <td>1</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2017-18200</td>\n",
       "      <td>CWE-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25464</th>\n",
       "      <td>user_defined.c</td>\n",
       "      <td>C</td>\n",
       "      <td>user_describe</td>\n",
       "      <td>/* user_defined.c: user defined key type\\n *\\n...</td>\n",
       "      <td>/* user_defined.c: user defined key type\\n *\\n...</td>\n",
       "      <td>void user_describe(const struct key *key, stru...</td>\n",
       "      <td>void user_describe(const struct key *key, stru...</td>\n",
       "      <td>{'added': [(117, '\\tif (key_is_positive(key))'...</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>125.0</td>\n",
       "      <td>699.0</td>\n",
       "      <td>6</td>\n",
       "      <td>42</td>\n",
       "      <td>2</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2017-15951</td>\n",
       "      <td>CWE-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24250</th>\n",
       "      <td>pgalloc.h</td>\n",
       "      <td>C</td>\n",
       "      <td>pgd_alloc</td>\n",
       "      <td>/*\\n *  S390 version\\n *    Copyright IBM Corp...</td>\n",
       "      <td>/*\\n *  S390 version\\n *    Copyright IBM Corp...</td>\n",
       "      <td>static inline pgd_t *pgd_alloc(struct mm_struc...</td>\n",
       "      <td>static inline pgd_t *pgd_alloc(struct mm_struc...</td>\n",
       "      <td>{'added': [(103, '\\tunsigned long *table = crs...</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "      <td>96.0</td>\n",
       "      <td>682.0</td>\n",
       "      <td>7</td>\n",
       "      <td>49</td>\n",
       "      <td>1</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2016-2143</td>\n",
       "      <td>CWE-20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          file_name programming_language  \\\n",
       "16189  nf_conntrack_proto_generic.c                    C   \n",
       "534                       jas_icc.c                    C   \n",
       "5502                      segment.c                    C   \n",
       "25464                user_defined.c                    C   \n",
       "24250                     pgalloc.h                    C   \n",
       "\n",
       "                  method_name  \\\n",
       "16189             generic_new   \n",
       "534            jas_iccputsint   \n",
       "5502   f2fs_wait_discard_bios   \n",
       "25464           user_describe   \n",
       "24250               pgd_alloc   \n",
       "\n",
       "                                             code_before  \\\n",
       "16189  /* (C) 1999-2001 Paul `Rusty' Russell\\n * (C) ...   \n",
       "534    /*\\n * Copyright (c) 2002-2003 Michael David A...   \n",
       "5502   /*\\n * fs/f2fs/segment.c\\n *\\n * Copyright (c)...   \n",
       "25464  /* user_defined.c: user defined key type\\n *\\n...   \n",
       "24250  /*\\n *  S390 version\\n *    Copyright IBM Corp...   \n",
       "\n",
       "                                              code_after  \\\n",
       "16189  /* (C) 1999-2001 Paul `Rusty' Russell\\n * (C) ...   \n",
       "534    /*\\n * Copyright (c) 2002-2003 Michael David A...   \n",
       "5502   /*\\n * fs/f2fs/segment.c\\n *\\n * Copyright (c)...   \n",
       "25464  /* user_defined.c: user defined key type\\n *\\n...   \n",
       "24250  /*\\n *  S390 version\\n *    Copyright IBM Corp...   \n",
       "\n",
       "                                             func_before  \\\n",
       "16189  static bool generic_new(struct nf_conn *ct, co...   \n",
       "534    static int jas_iccputsint(jas_stream_t *out, i...   \n",
       "5502   void f2fs_wait_discard_bios(struct f2fs_sb_inf...   \n",
       "25464  void user_describe(const struct key *key, stru...   \n",
       "24250  static inline pgd_t *pgd_alloc(struct mm_struc...   \n",
       "\n",
       "                                              func_after  \\\n",
       "16189  static bool generic_new(struct nf_conn *ct, co...   \n",
       "534    static int jas_iccputsint(jas_stream_t *out, i...   \n",
       "5502   void f2fs_wait_discard_bios(struct f2fs_sb_inf...   \n",
       "25464  void user_describe(const struct key *key, stru...   \n",
       "24250  static inline pgd_t *pgd_alloc(struct mm_struc...   \n",
       "\n",
       "                                                    diff num_lines_added  \\\n",
       "16189  {'added': [(17, 'static bool nf_generic_should...              25   \n",
       "534    {'added': [(85, 'static int jas_iccgetuint(jas...              15   \n",
       "5502   {'added': [(1213, 'void f2fs_wait_discard_bios...               3   \n",
       "25464  {'added': [(117, '\\tif (key_is_positive(key))'...               2   \n",
       "24250  {'added': [(103, '\\tunsigned long *table = crs...              19   \n",
       "\n",
       "      num_lines_deleted num_lines_in_file num_tokens_in_file  \\\n",
       "16189                 1             173.0              865.0   \n",
       "534                  15            1502.0            10066.0   \n",
       "5502                  3            2836.0            18721.0   \n",
       "25464                 2             125.0              699.0   \n",
       "24250                 5              96.0              682.0   \n",
       "\n",
       "       num_lines_in_method num_tokens_in_method method_complexity  \\\n",
       "16189                    5                   27                 1   \n",
       "534                      6                   45                 2   \n",
       "5502                     6                   28                 1   \n",
       "25464                    6                   42                 2   \n",
       "24250                    7                   49                 1   \n",
       "\n",
       "                                    repo          cve_id  cwe_id  \n",
       "16189  https://github.com/torvalds/linux   CVE-2014-8160  CWE-20  \n",
       "534    https://github.com/mdadams/jasper   CVE-2016-9395  CWE-20  \n",
       "5502   https://github.com/torvalds/linux  CVE-2017-18200  CWE-20  \n",
       "25464  https://github.com/torvalds/linux  CVE-2017-15951  CWE-20  \n",
       "24250  https://github.com/torvalds/linux   CVE-2016-2143  CWE-20  "
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwe_smallest.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "id": "dc28c99c-3999-4e2f-945d-7bf3f0a25a6d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import ast\n",
    "result_df.cwe_id = result_df.cwe_id.apply(ast.literal_eval)\n",
    "\n",
    "def remove_duplicates(lst):\n",
    "    return list(set(lst))\n",
    "\n",
    "result_df.cwe_id = result_df.cwe_id.apply(remove_duplicates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "id": "5db984a3-fdac-4ada-9301-2db5ed11b68e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(145, 14)"
      ]
     },
     "execution_count": 289,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cwe_top_25 = [787, 79, 89, 416, 78, 20, 125, 22, 352, 434, 862, 476, 287, 190, 502, 77, 119, 798, 918, 306, 362, 269, 94, 863, 276]\n",
    "\n",
    "result_df = result_df[result_df['cwe_id'].apply(lambda x: any(f'CWE-{i}' in x for i in cwe_top_25))]\n",
    "result_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "id": "f4c27cee-ad05-48db-85af-ae39f216a6ec",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>method_name</th>\n",
       "      <th>code_before</th>\n",
       "      <th>code_after</th>\n",
       "      <th>func_before</th>\n",
       "      <th>func_after</th>\n",
       "      <th>diff</th>\n",
       "      <th>num_lines_added</th>\n",
       "      <th>num_lines_deleted</th>\n",
       "      <th>num_lines_in_file</th>\n",
       "      <th>num_tokens_in_file</th>\n",
       "      <th>repo</th>\n",
       "      <th>cve_id</th>\n",
       "      <th>cwe_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>signature-v4-utils_test.go</td>\n",
       "      <td>TestCheckValid</td>\n",
       "      <td>// Copyright (c) 2015-2021 MinIO, Inc.\\n//\\n//...</td>\n",
       "      <td>// Copyright (c) 2015-2021 MinIO, Inc.\\n//\\n//...</td>\n",
       "      <td>func TestCheckValid(t *testing.T) {\\n\\tctx, ca...</td>\n",
       "      <td>func TestCheckValid(t *testing.T) {\\n\\tctx, ca...</td>\n",
       "      <td>{'added': [(79, '\\tglobalIAMSys.CreateUser(ctx...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>235.0</td>\n",
       "      <td>1475.0</td>\n",
       "      <td>https://github.com/minio/minio</td>\n",
       "      <td>CVE-2021-43858</td>\n",
       "      <td>[CWE-863]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>extract.go</td>\n",
       "      <td>Extract</td>\n",
       "      <td>/*\\n * Copyright (c) SAS Institute, Inc.\\n *\\n...</td>\n",
       "      <td>/*\\n * Copyright (c) SAS Institute, Inc.\\n *\\n...</td>\n",
       "      <td>func Extract(rs io.Reader, dest string) error ...</td>\n",
       "      <td>func Extract(rs io.Reader, dest string) error ...</td>\n",
       "      <td>{'added': [(24, '\\t\"path/filepath\"'), (25, '\\t...</td>\n",
       "      <td>17.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>118.0</td>\n",
       "      <td>681.0</td>\n",
       "      <td>https://github.com/sassoftware/go-rpmutils</td>\n",
       "      <td>CVE-2020-7667</td>\n",
       "      <td>[CWE-22]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>criteria.go</td>\n",
       "      <td>OrderBy</td>\n",
       "      <td>// Package criteria implements a Criteria API ...</td>\n",
       "      <td>// Package criteria implements a Criteria API ...</td>\n",
       "      <td>func (c Criteria) OrderBy() string {\\n\\tif c.S...</td>\n",
       "      <td>func (c Criteria) OrderBy() string {\\n\\tif c.S...</td>\n",
       "      <td>{'added': [(31, '\\t\\tlog.Error(\"Invalid field ...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>515.0</td>\n",
       "      <td>https://github.com/navidrome/navidrome</td>\n",
       "      <td>CVE-2022-23857</td>\n",
       "      <td>[CWE-89]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>casa.go</td>\n",
       "      <td>GetServerAppInfo</td>\n",
       "      <td>package service\\n\\nimport (\\n\\tjson2 \"encoding...</td>\n",
       "      <td>package service\\n\\nimport (\\n\\tjson2 \"encoding...</td>\n",
       "      <td>func (o *casaService) GetServerAppInfo(id, t s...</td>\n",
       "      <td>func (o *casaService) GetServerAppInfo(id, t s...</td>\n",
       "      <td>{'added': [(16, '\\tGetServerList(index, size, ...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>850.0</td>\n",
       "      <td>https://github.com/IceWhaleTech/CasaOS</td>\n",
       "      <td>CVE-2022-24193</td>\n",
       "      <td>[CWE-78]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>casa.go</td>\n",
       "      <td>GetServerList</td>\n",
       "      <td>package service\\n\\nimport (\\n\\tjson2 \"encoding...</td>\n",
       "      <td>package service\\n\\nimport (\\n\\tjson2 \"encoding...</td>\n",
       "      <td>func (o *casaService) GetServerList(index, siz...</td>\n",
       "      <td>func (o *casaService) GetServerList(index, siz...</td>\n",
       "      <td>{'added': [(16, '\\tGetServerList(index, size, ...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>850.0</td>\n",
       "      <td>https://github.com/IceWhaleTech/CasaOS</td>\n",
       "      <td>CVE-2022-24193</td>\n",
       "      <td>[CWE-78]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     file_name       method_name  \\\n",
       "1   signature-v4-utils_test.go    TestCheckValid   \n",
       "2                   extract.go           Extract   \n",
       "6                  criteria.go           OrderBy   \n",
       "13                     casa.go  GetServerAppInfo   \n",
       "14                     casa.go     GetServerList   \n",
       "\n",
       "                                          code_before  \\\n",
       "1   // Copyright (c) 2015-2021 MinIO, Inc.\\n//\\n//...   \n",
       "2   /*\\n * Copyright (c) SAS Institute, Inc.\\n *\\n...   \n",
       "6   // Package criteria implements a Criteria API ...   \n",
       "13  package service\\n\\nimport (\\n\\tjson2 \"encoding...   \n",
       "14  package service\\n\\nimport (\\n\\tjson2 \"encoding...   \n",
       "\n",
       "                                           code_after  \\\n",
       "1   // Copyright (c) 2015-2021 MinIO, Inc.\\n//\\n//...   \n",
       "2   /*\\n * Copyright (c) SAS Institute, Inc.\\n *\\n...   \n",
       "6   // Package criteria implements a Criteria API ...   \n",
       "13  package service\\n\\nimport (\\n\\tjson2 \"encoding...   \n",
       "14  package service\\n\\nimport (\\n\\tjson2 \"encoding...   \n",
       "\n",
       "                                          func_before  \\\n",
       "1   func TestCheckValid(t *testing.T) {\\n\\tctx, ca...   \n",
       "2   func Extract(rs io.Reader, dest string) error ...   \n",
       "6   func (c Criteria) OrderBy() string {\\n\\tif c.S...   \n",
       "13  func (o *casaService) GetServerAppInfo(id, t s...   \n",
       "14  func (o *casaService) GetServerList(index, siz...   \n",
       "\n",
       "                                           func_after  \\\n",
       "1   func TestCheckValid(t *testing.T) {\\n\\tctx, ca...   \n",
       "2   func Extract(rs io.Reader, dest string) error ...   \n",
       "6   func (c Criteria) OrderBy() string {\\n\\tif c.S...   \n",
       "13  func (o *casaService) GetServerAppInfo(id, t s...   \n",
       "14  func (o *casaService) GetServerList(index, siz...   \n",
       "\n",
       "                                                 diff  num_lines_added  \\\n",
       "1   {'added': [(79, '\\tglobalIAMSys.CreateUser(ctx...              1.0   \n",
       "2   {'added': [(24, '\\t\"path/filepath\"'), (25, '\\t...             17.0   \n",
       "6   {'added': [(31, '\\t\\tlog.Error(\"Invalid field ...              7.0   \n",
       "13  {'added': [(16, '\\tGetServerList(index, size, ...              7.0   \n",
       "14  {'added': [(16, '\\tGetServerList(index, size, ...              7.0   \n",
       "\n",
       "    num_lines_deleted  num_lines_in_file  num_tokens_in_file  \\\n",
       "1                 1.0              235.0              1475.0   \n",
       "2                 8.0              118.0               681.0   \n",
       "6                 3.0               93.0               515.0   \n",
       "13                7.0               93.0               850.0   \n",
       "14                7.0               93.0               850.0   \n",
       "\n",
       "                                          repo          cve_id     cwe_id  \n",
       "1               https://github.com/minio/minio  CVE-2021-43858  [CWE-863]  \n",
       "2   https://github.com/sassoftware/go-rpmutils   CVE-2020-7667   [CWE-22]  \n",
       "6       https://github.com/navidrome/navidrome  CVE-2022-23857   [CWE-89]  \n",
       "13      https://github.com/IceWhaleTech/CasaOS  CVE-2022-24193   [CWE-78]  \n",
       "14      https://github.com/IceWhaleTech/CasaOS  CVE-2022-24193   [CWE-78]  "
      ]
     },
     "execution_count": 290,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "id": "80f5cee2-ac5f-472e-a833-c44a0d2e3425",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "result_df = result_df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3a8ae9a-a2b3-44de-a9d9-e74bbae0b791",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Only for Python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "c7b76271-0c28-415b-b3bf-e6c9b765f85a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def remove_indent(code):\n",
    "    lines = code.split('\\n')\n",
    "    min_indent = float('inf')\n",
    "    for line in lines:\n",
    "        stripped_line = line.lstrip()\n",
    "        if stripped_line:  # Non-empty line\n",
    "            indent = len(line) - len(stripped_line)\n",
    "            min_indent = min(min_indent, indent)\n",
    "\n",
    "    cleaned_lines = [line[min_indent:] for line in lines]\n",
    "    cleaned_code = '\\n'.join(cleaned_lines)\n",
    "    return cleaned_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "id": "cc8c48c7-7a96-4f80-991d-32b5597a7d9f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "result_df.func_before = result_df.func_before.apply(remove_indent)\n",
    "result_df.func_after = result_df.func_after.apply(remove_indent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "id": "568baaa9-63dc-4886-9a1a-11384e40c950",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def remove_print_statements(code):\n",
    "    lines = code.split('\\n')\n",
    "    modified_lines = [line for line in lines if not line.strip().startswith('print')]\n",
    "    modified_code = '\\n'.join(modified_lines)\n",
    "    return modified_code\n",
    "\n",
    "def is_compilable(code):\n",
    "    try:\n",
    "        compile(code, \"<string>\", \"exec\")\n",
    "        return True\n",
    "    except SyntaxError as e:\n",
    "        if \"Missing parentheses in call to 'print'\" in str(e):\n",
    "            modified_code = remove_print_statements(code)\n",
    "            try:\n",
    "                compile(modified_code, \"<string>\", \"exec\")\n",
    "                return True\n",
    "            except SyntaxError:\n",
    "                return False\n",
    "        else:\n",
    "            return False\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "id": "dbc4e7a2-65af-40f6-9560-05f81c4b5d5b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<string>:591: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:591: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:591: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:591: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:591: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:591: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:591: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:591: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:334: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:352: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:390: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:438: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:496: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:547: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:616: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:624: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:343: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:361: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:399: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:447: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:505: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:556: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:625: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n",
      "<string>:633: SyntaxWarning: \"is\" with a literal. Did you mean \"==\"?\n"
     ]
    }
   ],
   "source": [
    "not_compilable = []\n",
    "for i in range(result_df.shape[0]):\n",
    "    row = result_df.iloc[i]\n",
    "    c1, c2, c3, c4 = row['code_before'], row['code_after'], row['func_before'], row['func_after']\n",
    "    if not all(is_compilable(code) for code in [c1, c2, c3, c4]):\n",
    "        not_compilable.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "id": "fb787976-0425-476d-a5da-2971b6b34047",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "51"
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(not_compilable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "id": "28ec3b48-b78f-4a6b-8913-4b9eda1fedea",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "result_df.drop(not_compilable, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "id": "981e65c4-dce6-4369-9d9b-12e118b6324a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(587, 14)"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "id": "9b5facc3-4533-46e8-9f1d-d8f7de457818",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "result_df = result_df.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "id": "ac8b0c46-8f41-4c82-a2d6-13907d7fb16b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>method_name</th>\n",
       "      <th>code_before</th>\n",
       "      <th>code_after</th>\n",
       "      <th>func_before</th>\n",
       "      <th>func_after</th>\n",
       "      <th>diff</th>\n",
       "      <th>num_lines_added</th>\n",
       "      <th>num_lines_deleted</th>\n",
       "      <th>num_lines_in_file</th>\n",
       "      <th>num_tokens_in_file</th>\n",
       "      <th>repo</th>\n",
       "      <th>cve_id</th>\n",
       "      <th>cwe_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>open_utils.c</td>\n",
       "      <td>read_new_config_info</td>\n",
       "      <td>//////////////////////////////////////////////...</td>\n",
       "      <td>//////////////////////////////////////////////...</td>\n",
       "      <td>static int read_new_config_info (WavpackContex...</td>\n",
       "      <td>static int read_new_config_info (WavpackContex...</td>\n",
       "      <td>{'added': [(563, '    if (bytecnt &gt;= 2) {'), (...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>825.0</td>\n",
       "      <td>6305.0</td>\n",
       "      <td>https://github.com/dbry/WavPack</td>\n",
       "      <td>CVE-2016-10169</td>\n",
       "      <td>[CWE-125]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>route.c</td>\n",
       "      <td>inet_rtm_getroute</td>\n",
       "      <td>/*\\n * INET\\t\\tAn implementation of the TCP/IP...</td>\n",
       "      <td>/*\\n * INET\\t\\tAn implementation of the TCP/IP...</td>\n",
       "      <td>static int inet_rtm_getroute(struct sk_buff *i...</td>\n",
       "      <td>static int inet_rtm_getroute(struct sk_buff *i...</td>\n",
       "      <td>{'added': [(2766, '\\tif (rtm-&gt;rtm_flags &amp; RTM_...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2327.0</td>\n",
       "      <td>14667.0</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2017-13686</td>\n",
       "      <td>[CWE-476]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>main.c</td>\n",
       "      <td>mp4boxMain</td>\n",
       "      <td>/*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...</td>\n",
       "      <td>/*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...</td>\n",
       "      <td>int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...</td>\n",
       "      <td>int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...</td>\n",
       "      <td>{'added': [(5598, '\\t\\t\\tfprintf(stderr, \"File...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5787.0</td>\n",
       "      <td>43960.0</td>\n",
       "      <td>https://github.com/gpac/gpac</td>\n",
       "      <td>CVE-2020-23930</td>\n",
       "      <td>[CWE-476]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>udp.c</td>\n",
       "      <td>udpv6_sendmsg</td>\n",
       "      <td>/*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...</td>\n",
       "      <td>/*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...</td>\n",
       "      <td>int udpv6_sendmsg(struct sock *sk, struct msgh...</td>\n",
       "      <td>int udpv6_sendmsg(struct sock *sk, struct msgh...</td>\n",
       "      <td>{'added': [(1113, '\\tstruct ipv6_txoptions *op...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1221.0</td>\n",
       "      <td>7876.0</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2016-3841</td>\n",
       "      <td>[CWE-264, CWE-416]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>api.c</td>\n",
       "      <td>crypto_larval_lookup</td>\n",
       "      <td>/*\\n * Scatterlist Cryptographic API.\\n *\\n * ...</td>\n",
       "      <td>/*\\n * Scatterlist Cryptographic API.\\n *\\n * ...</td>\n",
       "      <td>struct crypto_alg *crypto_larval_lookup(const ...</td>\n",
       "      <td>struct crypto_alg *crypto_larval_lookup(const ...</td>\n",
       "      <td>{'added': [(219, '\\t\\trequest_module(\"crypto-%...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>424.0</td>\n",
       "      <td>2423.0</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2013-7421</td>\n",
       "      <td>[CWE-269]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      file_name           method_name  \\\n",
       "0  open_utils.c  read_new_config_info   \n",
       "1       route.c     inet_rtm_getroute   \n",
       "2        main.c            mp4boxMain   \n",
       "3         udp.c         udpv6_sendmsg   \n",
       "4         api.c  crypto_larval_lookup   \n",
       "\n",
       "                                         code_before  \\\n",
       "0  //////////////////////////////////////////////...   \n",
       "1  /*\\n * INET\\t\\tAn implementation of the TCP/IP...   \n",
       "2  /*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...   \n",
       "3  /*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...   \n",
       "4  /*\\n * Scatterlist Cryptographic API.\\n *\\n * ...   \n",
       "\n",
       "                                          code_after  \\\n",
       "0  //////////////////////////////////////////////...   \n",
       "1  /*\\n * INET\\t\\tAn implementation of the TCP/IP...   \n",
       "2  /*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...   \n",
       "3  /*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...   \n",
       "4  /*\\n * Scatterlist Cryptographic API.\\n *\\n * ...   \n",
       "\n",
       "                                         func_before  \\\n",
       "0  static int read_new_config_info (WavpackContex...   \n",
       "1  static int inet_rtm_getroute(struct sk_buff *i...   \n",
       "2  int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...   \n",
       "3  int udpv6_sendmsg(struct sock *sk, struct msgh...   \n",
       "4  struct crypto_alg *crypto_larval_lookup(const ...   \n",
       "\n",
       "                                          func_after  \\\n",
       "0  static int read_new_config_info (WavpackContex...   \n",
       "1  static int inet_rtm_getroute(struct sk_buff *i...   \n",
       "2  int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...   \n",
       "3  int udpv6_sendmsg(struct sock *sk, struct msgh...   \n",
       "4  struct crypto_alg *crypto_larval_lookup(const ...   \n",
       "\n",
       "                                                diff  num_lines_added  \\\n",
       "0  {'added': [(563, '    if (bytecnt >= 2) {'), (...              5.0   \n",
       "1  {'added': [(2766, '\\tif (rtm->rtm_flags & RTM_...              9.0   \n",
       "2  {'added': [(5598, '\\t\\t\\tfprintf(stderr, \"File...              1.0   \n",
       "3  {'added': [(1113, '\\tstruct ipv6_txoptions *op...              6.0   \n",
       "4  {'added': [(219, '\\t\\trequest_module(\"crypto-%...              2.0   \n",
       "\n",
       "   num_lines_deleted  num_lines_in_file  num_tokens_in_file  \\\n",
       "0                1.0              825.0              6305.0   \n",
       "1                2.0             2327.0             14667.0   \n",
       "2                1.0             5787.0             43960.0   \n",
       "3                2.0             1221.0              7876.0   \n",
       "4                2.0              424.0              2423.0   \n",
       "\n",
       "                                repo          cve_id              cwe_id  \n",
       "0    https://github.com/dbry/WavPack  CVE-2016-10169           [CWE-125]  \n",
       "1  https://github.com/torvalds/linux  CVE-2017-13686           [CWE-476]  \n",
       "2       https://github.com/gpac/gpac  CVE-2020-23930           [CWE-476]  \n",
       "3  https://github.com/torvalds/linux   CVE-2016-3841  [CWE-264, CWE-416]  \n",
       "4  https://github.com/torvalds/linux   CVE-2013-7421           [CWE-269]  "
      ]
     },
     "execution_count": 244,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c98d77ae-f2b2-47b9-baab-e87c1d001567",
   "metadata": {},
   "source": [
    "## For the rest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "id": "f1054164-377c-481f-b7d8-2a8e0c3b1423",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "cwe_id\n",
       "[CWE-125]             547\n",
       "[CWE-119]             470\n",
       "[CWE-20]              273\n",
       "[CWE-476]             257\n",
       "[CWE-787]             242\n",
       "                     ... \n",
       "[CWE-787, CWE-681]      1\n",
       "[CWE-476, CWE-362]      1\n",
       "[CWE-125, CWE-362]      1\n",
       "[CWE-125, CWE-200]      1\n",
       "[CWE-125, CWE-190]      1\n",
       "Name: count, Length: 73, dtype: int64"
      ]
     },
     "execution_count": 239,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df.cwe_id.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "id": "b10bfa6a-734c-4d77-94da-124a08f670f5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "result_df.to_csv('CVEFixes_go_25.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 293,
   "id": "b7937418-c3ad-4a7d-8d83-52f3a180ef92",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'CWE-863': 36,\n",
       "         'CWE-190': 19,\n",
       "         'CWE-78': 13,\n",
       "         'CWE-918': 12,\n",
       "         'CWE-22': 11,\n",
       "         'CWE-20': 10,\n",
       "         'CWE-89': 9,\n",
       "         'CWE-79': 9,\n",
       "         'CWE-287': 9,\n",
       "         'CWE-276': 4,\n",
       "         'CWE-269': 4,\n",
       "         'CWE-362': 3,\n",
       "         'CWE-798': 2,\n",
       "         'CWE-787': 1,\n",
       "         'CWE-125': 1,\n",
       "         'CWE-862': 1,\n",
       "         'CWE-476': 1})"
      ]
     },
     "execution_count": 293,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "\n",
    "counters = Counter()\n",
    "\n",
    "for sublist in result_df['cwe_id']:\n",
    "    counters.update(sublist)\n",
    "\n",
    "# Display the counts for each element\n",
    "counters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "id": "12cb778b-1b70-47ef-a128-db8d97c9d683",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count      587.000000\n",
       "mean      8708.102215\n",
       "std      10512.587620\n",
       "min         48.000000\n",
       "25%       1116.000000\n",
       "50%       3938.000000\n",
       "75%      13180.000000\n",
       "max      39494.000000\n",
       "Name: num_tokens_in_file, dtype: float64"
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_df.num_tokens_in_file.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "id": "c9d0de2e-79ee-4ed1-b7ed-3064ff10eb9f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import datasets\n",
    "from datasets import Dataset, DatasetDict\n",
    "\n",
    "hfC = Dataset.from_pandas(pd.read_csv('assets/CVEFixes_c_25.csv'))\n",
    "hfG = Dataset.from_pandas(pd.read_csv('assets/CVEFixes_go_25.csv'))\n",
    "hfJ = Dataset.from_pandas(pd.read_csv('assets/CVEFixes_java_25.csv'))\n",
    "hfP = Dataset.from_pandas(pd.read_csv('assets/CVEFixes_python_25.csv'))\n",
    "hfR = Dataset.from_pandas(pd.read_csv('assets/CVEFixes_ruby_25.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "id": "3776910f-afc0-48f0-91e7-777e5cfe76a3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4123d9c240314b01a147f48ce6de18a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b21b1ebeb1c44d9accefa743647123b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/3 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "KeyboardInterrupt\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from datasets import DatasetDict\n",
    "\n",
    "ddict = DatasetDict({\n",
    "    \"c\": hfC,\n",
    "    \"go\": hfG,\n",
    "    \"java\": hfJ,\n",
    "    \"python\": hfP,\n",
    "    \"ruby\": hfR,\n",
    "})\n",
    "\n",
    "ddict.push_to_hub(\"euisuh15/cveFixes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "id": "a258bc4e-1326-4506-b1e6-88f7fda585c6",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd42c599c3c54011ad3ef777a7ac82f8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8bb580b8fe584fdb84e875636192a57d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/3 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21ca1a11dd104da78724d4931cc580ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "779433bde7044971a5744493b5301c3d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf48a4e8b7fe465ab3a00a14a48f06d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cae15cd38f224063a10eefd9e5d451b1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2cd991781cf446b7a95ce5b319caca73",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8cd35aa1dd7a4ce2aa4e08a56c9385c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3552e4813c9494c93e96e74e930bc1f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49108597d0e040c2a5a835593c3c3594",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/datasets/euisuh15/cveFixes1/commit/f5b2212b9c568cb087362e62a0ea05a09cce842d', commit_message='Upload dataset', commit_description='', oid='f5b2212b9c568cb087362e62a0ea05a09cce842d', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 310,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ddict.push_to_hub('euisuh15/cveFixes1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7bb7a282-4bff-44eb-bd11-b01d54a63473",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "cc4bbf11-6107-437b-a009-4d3236924c75",
   "metadata": {},
   "source": [
    "# CVEFixes by language"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3564c35-e971-41e3-9876-a0c7b97f9cc1",
   "metadata": {},
   "source": [
    "## Retrieve dataset from HF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1f8d0a46-e791-4ee7-bbc2-40e197f5e42b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/local/QCRI/ejeong/anaconda2/envs/vllm/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2621, 14)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from datasets import load_dataset\n",
    "import os\n",
    "\n",
    "os.environ[\"HF_HOME\"] = \"/tmp/huggingface\"\n",
    "# dataset = load_dataset(\"euisuh15/cveFixes1\")\n",
    "dataset = load_dataset(\"euisuh15/cveFixes1\", split='c')\n",
    "\n",
    "# lang option:  c | go | java | python | ruby\n",
    "# lang = \"c\"\n",
    "# df = pd.DataFrame(dataset[lang])\n",
    "df = pd.DataFrame(dataset)\n",
    "\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "975a8129-bae3-402b-87af-cfda50df1ab8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file_name</th>\n",
       "      <th>method_name</th>\n",
       "      <th>code_before</th>\n",
       "      <th>code_after</th>\n",
       "      <th>func_before</th>\n",
       "      <th>func_after</th>\n",
       "      <th>diff</th>\n",
       "      <th>num_lines_added</th>\n",
       "      <th>num_lines_deleted</th>\n",
       "      <th>num_lines_in_file</th>\n",
       "      <th>num_tokens_in_file</th>\n",
       "      <th>repo</th>\n",
       "      <th>cve_id</th>\n",
       "      <th>cwe_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>open_utils.c</td>\n",
       "      <td>read_new_config_info</td>\n",
       "      <td>//////////////////////////////////////////////...</td>\n",
       "      <td>//////////////////////////////////////////////...</td>\n",
       "      <td>static int read_new_config_info (WavpackContex...</td>\n",
       "      <td>static int read_new_config_info (WavpackContex...</td>\n",
       "      <td>{'added': [(563, '    if (bytecnt &gt;= 2) {'), (...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>825.0</td>\n",
       "      <td>6305.0</td>\n",
       "      <td>https://github.com/dbry/WavPack</td>\n",
       "      <td>CVE-2016-10169</td>\n",
       "      <td>['CWE-125']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>route.c</td>\n",
       "      <td>inet_rtm_getroute</td>\n",
       "      <td>/*\\n * INET\\t\\tAn implementation of the TCP/IP...</td>\n",
       "      <td>/*\\n * INET\\t\\tAn implementation of the TCP/IP...</td>\n",
       "      <td>static int inet_rtm_getroute(struct sk_buff *i...</td>\n",
       "      <td>static int inet_rtm_getroute(struct sk_buff *i...</td>\n",
       "      <td>{'added': [(2766, '\\tif (rtm-&gt;rtm_flags &amp; RTM_...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2327.0</td>\n",
       "      <td>14667.0</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2017-13686</td>\n",
       "      <td>['CWE-476']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>main.c</td>\n",
       "      <td>mp4boxMain</td>\n",
       "      <td>/*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...</td>\n",
       "      <td>/*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...</td>\n",
       "      <td>int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...</td>\n",
       "      <td>int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...</td>\n",
       "      <td>{'added': [(5598, '\\t\\t\\tfprintf(stderr, \"File...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5787.0</td>\n",
       "      <td>43960.0</td>\n",
       "      <td>https://github.com/gpac/gpac</td>\n",
       "      <td>CVE-2020-23930</td>\n",
       "      <td>['CWE-476']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>udp.c</td>\n",
       "      <td>udpv6_sendmsg</td>\n",
       "      <td>/*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...</td>\n",
       "      <td>/*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...</td>\n",
       "      <td>int udpv6_sendmsg(struct sock *sk, struct msgh...</td>\n",
       "      <td>int udpv6_sendmsg(struct sock *sk, struct msgh...</td>\n",
       "      <td>{'added': [(1113, '\\tstruct ipv6_txoptions *op...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1221.0</td>\n",
       "      <td>7876.0</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2016-3841</td>\n",
       "      <td>['CWE-264', 'CWE-416']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>api.c</td>\n",
       "      <td>crypto_larval_lookup</td>\n",
       "      <td>/*\\n * Scatterlist Cryptographic API.\\n *\\n * ...</td>\n",
       "      <td>/*\\n * Scatterlist Cryptographic API.\\n *\\n * ...</td>\n",
       "      <td>struct crypto_alg *crypto_larval_lookup(const ...</td>\n",
       "      <td>struct crypto_alg *crypto_larval_lookup(const ...</td>\n",
       "      <td>{'added': [(219, '\\t\\trequest_module(\"crypto-%...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>424.0</td>\n",
       "      <td>2423.0</td>\n",
       "      <td>https://github.com/torvalds/linux</td>\n",
       "      <td>CVE-2013-7421</td>\n",
       "      <td>['CWE-269']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      file_name           method_name  \\\n",
       "0  open_utils.c  read_new_config_info   \n",
       "1       route.c     inet_rtm_getroute   \n",
       "2        main.c            mp4boxMain   \n",
       "3         udp.c         udpv6_sendmsg   \n",
       "4         api.c  crypto_larval_lookup   \n",
       "\n",
       "                                         code_before  \\\n",
       "0  //////////////////////////////////////////////...   \n",
       "1  /*\\n * INET\\t\\tAn implementation of the TCP/IP...   \n",
       "2  /*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...   \n",
       "3  /*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...   \n",
       "4  /*\\n * Scatterlist Cryptographic API.\\n *\\n * ...   \n",
       "\n",
       "                                          code_after  \\\n",
       "0  //////////////////////////////////////////////...   \n",
       "1  /*\\n * INET\\t\\tAn implementation of the TCP/IP...   \n",
       "2  /*\\n *\\t\\t\\tGPAC - Multimedia Framework C SDK\\...   \n",
       "3  /*\\n *\\tUDP over IPv6\\n *\\tLinux INET6 impleme...   \n",
       "4  /*\\n * Scatterlist Cryptographic API.\\n *\\n * ...   \n",
       "\n",
       "                                         func_before  \\\n",
       "0  static int read_new_config_info (WavpackContex...   \n",
       "1  static int inet_rtm_getroute(struct sk_buff *i...   \n",
       "2  int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...   \n",
       "3  int udpv6_sendmsg(struct sock *sk, struct msgh...   \n",
       "4  struct crypto_alg *crypto_larval_lookup(const ...   \n",
       "\n",
       "                                          func_after  \\\n",
       "0  static int read_new_config_info (WavpackContex...   \n",
       "1  static int inet_rtm_getroute(struct sk_buff *i...   \n",
       "2  int mp4boxMain(int argc, char **argv)\\n{\\n\\tu3...   \n",
       "3  int udpv6_sendmsg(struct sock *sk, struct msgh...   \n",
       "4  struct crypto_alg *crypto_larval_lookup(const ...   \n",
       "\n",
       "                                                diff  num_lines_added  \\\n",
       "0  {'added': [(563, '    if (bytecnt >= 2) {'), (...              5.0   \n",
       "1  {'added': [(2766, '\\tif (rtm->rtm_flags & RTM_...              9.0   \n",
       "2  {'added': [(5598, '\\t\\t\\tfprintf(stderr, \"File...              1.0   \n",
       "3  {'added': [(1113, '\\tstruct ipv6_txoptions *op...              6.0   \n",
       "4  {'added': [(219, '\\t\\trequest_module(\"crypto-%...              2.0   \n",
       "\n",
       "   num_lines_deleted  num_lines_in_file  num_tokens_in_file  \\\n",
       "0                1.0              825.0              6305.0   \n",
       "1                2.0             2327.0             14667.0   \n",
       "2                1.0             5787.0             43960.0   \n",
       "3                2.0             1221.0              7876.0   \n",
       "4                2.0              424.0              2423.0   \n",
       "\n",
       "                                repo          cve_id                  cwe_id  \n",
       "0    https://github.com/dbry/WavPack  CVE-2016-10169             ['CWE-125']  \n",
       "1  https://github.com/torvalds/linux  CVE-2017-13686             ['CWE-476']  \n",
       "2       https://github.com/gpac/gpac  CVE-2020-23930             ['CWE-476']  \n",
       "3  https://github.com/torvalds/linux   CVE-2016-3841  ['CWE-264', 'CWE-416']  \n",
       "4  https://github.com/torvalds/linux   CVE-2013-7421             ['CWE-269']  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e74e7f5c-fc0d-4b4a-863a-380059836c10",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('vul7.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}