12-create-figure-5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `12-create-figure-5`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas, numpy, pathlib\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.patches import Rectangle\n",
    "\n",
    "from skops.io import load\n",
    "\n",
    "from misc import construct_line\n",
    "\n",
    "pathlib.Path('pdf').mkdir(exist_ok=True)\n",
    "pathlib.Path('pdf/figure-5').mkdir(exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "chosen_model = 'XB'\n",
    "\n",
    "best_model = {}\n",
    "\n",
    "for model in ['LR', 'NN', 'XB']:\n",
    "    best_model[model] = load('models/'+model.lower()+'.skops', trusted=True)\n",
    "\n",
    "X={}\n",
    "Y={}\n",
    "Z={}\n",
    "\n",
    "for i in ['validation-samples', 'validation-mutations', 'mic', 'train', 'test']:\n",
    "    X[i]={}\n",
    "    Y[i]={}\n",
    "    Z[i]={}\n",
    "    with open('data/ds-'+i+'.npy', 'rb') as f:\n",
    "        Y[i]['input'] = numpy.load(f)\n",
    "        X[i]['input'] = numpy.load(f)\n",
    "        Z[i]['input'] = numpy.load(f, allow_pickle=True)\n",
    "\n",
    "# load in the results for SuspectPZA\n",
    "suspectpza={}\n",
    "for i in ['validation-samples', 'validation-mutations','mic']:\n",
    "    suspectpza[i]={}\n",
    "    with open('data/suspectpza-'+i+'.npy', 'rb') as f:\n",
    "        suspectpza[i]['input'] = numpy.load(f)\n",
    "        suspectpza[i]['predicted'] = numpy.load(f)\n",
    "        suspectpza[i]['muts'] = numpy.load(f, allow_pickle=True)\n",
    "line = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate_model(line, best_model, model_name, dataset, X, Y):\n",
    "    \n",
    "    Y[dataset]['predicted'] = best_model.predict(X[dataset]['input'])\n",
    "    Y[dataset]['scores'] = best_model.predict_proba(X[dataset]['input'])[:,1]\n",
    "\n",
    "    row = construct_line(model_name , dataset, None, Y[dataset], None)\n",
    "\n",
    "    n_muts_unique = (numpy.unique(Z[dataset]['input'])).shape[0]\n",
    "\n",
    "    row.append(X[dataset]['input'].shape[0])\n",
    "    row.append(n_muts_unique)\n",
    "    line.append(row)\n",
    "\n",
    "    return(line)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's work out which of the mutations in the Validation dataset have inconsistent mutations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_mutations = pandas.read_csv('data/ds-validation-mutations-full.csv')\n",
    "\n",
    "mutations_with_consistent_phenotypes= set(validation_mutations[validation_mutations.CONSISTENT_PHENOTYPE.isin(['R', 'S'])].MUTATION.unique())\n",
    "\n",
    "mutations_with_inconsistent_phenotypes = set(validation_mutations[validation_mutations.CONSISTENT_PHENOTYPE=='U'].MUTATION.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(168, 199)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(mutations_with_inconsistent_phenotypes), len(mutations_with_consistent_phenotypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "mask = numpy.isin(Z['validation-samples']['input'], numpy.array(list(mutations_with_inconsistent_phenotypes)))\n",
    "\n",
    "X['vs-incons']={}\n",
    "Y['vs-incons']={}\n",
    "Z['vs-incons']={}\n",
    "\n",
    "X['vs-incons']['input']=X['validation-samples']['input'][mask]\n",
    "Y['vs-incons']['input']=Y['validation-samples']['input'][mask]\n",
    "Z['vs-incons']['input']=Z['validation-samples']['input'][mask]\n",
    "\n",
    "mask = numpy.isin(Z['validation-samples']['input'], numpy.array(list(mutations_with_consistent_phenotypes)))\n",
    "\n",
    "X['vs-consis']={}\n",
    "Y['vs-consis']={}\n",
    "Z['vs-consis']={}\n",
    "\n",
    "X['vs-consis']['input']=X['validation-samples']['input'][mask]\n",
    "Y['vs-consis']['input']=Y['validation-samples']['input'][mask]\n",
    "Z['vs-consis']['input']=Z['validation-samples']['input'][mask]\n",
    "\n",
    "mask = numpy.isin(Z['validation-mutations']['input'], numpy.array(list(mutations_with_consistent_phenotypes)))\n",
    "\n",
    "X['vm-consis']={}\n",
    "Y['vm-consis']={}\n",
    "Z['vm-consis']={}\n",
    "\n",
    "X['vm-consis']['input']=X['validation-mutations']['input'][mask]\n",
    "Y['vm-consis']['input']=Y['validation-mutations']['input'][mask]\n",
    "Z['vm-consis']['input']=Z['validation-mutations']['input'][mask]\n",
    "\n",
    "# mask = numpy.isin(Z['validation-mutations']['input'], numpy.array(list(mutations_with_consistent_phenotypes)))\n",
    "\n",
    "# X['vm-incons']={}\n",
    "# Y['vm-incons']={}\n",
    "# Z['vm-incons']={}\n",
    "\n",
    "# X['vm-incons']['input']=X['validation-mutations']['input'][mask]\n",
    "# Y['vm-incons']['input']=Y['validation-mutations']['input'][mask]\n",
    "# Z['vm-incons']['input']=Z['validation-mutations']['input'][mask]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset in ['validation-samples', 'validation-mutations',  'mic', 'vs-incons', 'vs-consis', 'vm-consis']: #, 'vm-incons']: \n",
    "    line = validate_model(line, best_model[chosen_model], chosen_model, dataset, X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 4027 samples in the validation dataset\n",
      "Of these, 908 have a mutation which has an inconsistent phenotype\n",
      "And 160 of these are samples were there are fewer than 4 samples and 748 are samples with five or more samples\n"
     ]
    }
   ],
   "source": [
    "total = validation_mutations.TOTAL.sum()\n",
    "\n",
    "print(\"There are %i samples in the validation dataset\" % (total))\n",
    "\n",
    "total = validation_mutations[validation_mutations.CONSISTENT_PHENOTYPE=='U'].TOTAL.sum()\n",
    "\n",
    "print(\"Of these, %i have a mutation which has an inconsistent phenotype\" % (total))\n",
    "\n",
    "total_lt4 = validation_mutations[(validation_mutations.CONSISTENT_PHENOTYPE=='U') & (validation_mutations.TOTAL<4)].TOTAL.sum()\n",
    "total_ge4 = validation_mutations[(validation_mutations.CONSISTENT_PHENOTYPE=='U') & (validation_mutations.TOTAL>=4)].TOTAL.sum()\n",
    "print(\"And %i of these are samples were there are fewer than 4 samples and %i are samples with five or more samples\" % (total_lt4, total_ge4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(116, 83, 71, 97)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mutations_in_train  = set(Z['train']['input'])  #.union(set(Z['test']['input']))\n",
    "\n",
    "mutations_with_consistent_phenotypes_in_train = mutations_with_consistent_phenotypes.intersection(mutations_in_train)\n",
    "mutations_with_consistent_phenotypes_notin_train = mutations_with_consistent_phenotypes - mutations_in_train\n",
    "\n",
    "mutations_with_inconsistent_phenotypes_in_train = mutations_with_inconsistent_phenotypes.intersection(mutations_in_train)\n",
    "mutations_with_inconsistent_phenotypes_notin_train = mutations_with_inconsistent_phenotypes - mutations_in_train \n",
    "\n",
    "len(mutations_with_consistent_phenotypes_in_train), len(mutations_with_consistent_phenotypes_notin_train), len(mutations_with_inconsistent_phenotypes_in_train), len(mutations_with_inconsistent_phenotypes_notin_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vs-consis-train\n",
      "vs-consis-notrain\n",
      "vs-incons-train\n",
      "vs-incons-notrain\n",
      "vm-consis-train\n",
      "vm-consis-notrain\n"
     ]
    }
   ],
   "source": [
    "for i in ['vs-consis-train', 'vs-consis-notrain', 'vs-incons-train', 'vs-incons-notrain', 'vm-consis-train', 'vm-consis-notrain', 'vm-incons-train', 'vm-incons-notrain']:\n",
    "    X[i] = {}\n",
    "    Y[i] = {}\n",
    "    Z[i] = {}\n",
    "\n",
    "mask = numpy.isin(Z['validation-samples']['input'],  numpy.array(list(mutations_with_consistent_phenotypes_in_train)))\n",
    "X['vs-consis-train']['input']=X['validation-samples']['input'][mask]\n",
    "Y['vs-consis-train']['input']=Y['validation-samples']['input'][mask]\n",
    "Z['vs-consis-train']['input']=Z['validation-samples']['input'][mask]\n",
    "\n",
    "mask = numpy.isin(Z['validation-samples']['input'],  numpy.array(list(mutations_with_consistent_phenotypes_notin_train)))\n",
    "X['vs-consis-notrain']['input']=X['validation-samples']['input'][mask]\n",
    "Y['vs-consis-notrain']['input']=Y['validation-samples']['input'][mask]\n",
    "Z['vs-consis-notrain']['input']=Z['validation-samples']['input'][mask]\n",
    "\n",
    "mask = numpy.isin(Z['validation-samples']['input'],  numpy.array(list(mutations_with_inconsistent_phenotypes_in_train)))\n",
    "X['vs-incons-train']['input']=X['validation-samples']['input'][mask]\n",
    "Z['vs-incons-train']['input']=Z['validation-samples']['input'][mask]\n",
    "Y['vs-incons-train']['input']=Y['validation-samples']['input'][mask]\n",
    "\n",
    "mask = numpy.isin(Z['validation-samples']['input'],  numpy.array(list(mutations_with_inconsistent_phenotypes_notin_train)))\n",
    "X['vs-incons-notrain']['input']=X['validation-samples']['input'][mask]\n",
    "Z['vs-incons-notrain']['input']=Z['validation-samples']['input'][mask]\n",
    "Y['vs-incons-notrain']['input']=Y['validation-samples']['input'][mask]\n",
    "\n",
    "mask = numpy.isin(Z['validation-mutations']['input'],  numpy.array(list(mutations_with_consistent_phenotypes_in_train)))\n",
    "X['vm-consis-train']['input']=X['validation-mutations']['input'][mask]\n",
    "Y['vm-consis-train']['input']=Y['validation-mutations']['input'][mask]\n",
    "Z['vm-consis-train']['input']=Z['validation-mutations']['input'][mask]\n",
    "\n",
    "\n",
    "mask = numpy.isin(Z['validation-mutations']['input'],  numpy.array(list(mutations_with_consistent_phenotypes_in_train)))\n",
    "X['vm-consis-train']['input']=X['validation-mutations']['input'][mask]\n",
    "Y['vm-consis-train']['input']=Y['validation-mutations']['input'][mask]\n",
    "Z['vm-consis-train']['input']=Z['validation-mutations']['input'][mask]\n",
    "\n",
    "mask = numpy.isin(Z['validation-mutations']['input'],  numpy.array(list(mutations_with_consistent_phenotypes_notin_train)))\n",
    "X['vm-consis-notrain']['input']=X['validation-mutations']['input'][mask]\n",
    "Y['vm-consis-notrain']['input']=Y['validation-mutations']['input'][mask]\n",
    "Z['vm-consis-notrain']['input']=Z['validation-mutations']['input'][mask]\n",
    "\n",
    "\n",
    "# mask = numpy.isin(Z['validation-mutations']['input'],  numpy.array(list(mutations_with_inconsistent_phenotypes_in_train)))\n",
    "# X['vm-incons-train']['input']=X['validation-mutations']['input'][mask]\n",
    "# Z['vm-incons-train']['input']=Z['validation-mutations']['input'][mask]\n",
    "# Y['vm-incons-train']['input']=Y['validation-mutations']['input'][mask]\n",
    "\n",
    "# mask = numpy.isin(Z['validation-mutations']['input'],  numpy.array(list(mutations_with_inconsistent_phenotypes_notin_train)))\n",
    "# X['vm-incons-notrain']['input']=X['validation-mutations']['input'][mask]\n",
    "# Z['vm-incons-notrain']['input']=Z['validation-mutations']['input'][mask]\n",
    "# Y['vm-incons-notrain']['input']=Y['validation-mutations']['input'][mask]\n",
    "\n",
    "\n",
    "for dataset in ['vs-consis-train', 'vs-consis-notrain', 'vs-incons-train', 'vs-incons-notrain', 'vm-consis-train', 'vm-consis-notrain']: # 'vm-incons-train', 'vm-incons-notrain']:\n",
    "    print(dataset)\n",
    "    line = validate_model(line, best_model[chosen_model], chosen_model, dataset, X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vs-consis-notrain-large\n",
      "vs-consis-notrain-small\n"
     ]
    }
   ],
   "source": [
    "mutations_with_consistent_phenotypes_notin_train_large = set(list(validation_mutations[(validation_mutations.MUTATION.isin(mutations_with_consistent_phenotypes_notin_train)) & (validation_mutations.TOTAL>=4)].MUTATION.unique()))\n",
    "mutations_with_consistent_phenotypes_notin_train_small = set(list(validation_mutations[(validation_mutations.MUTATION.isin(mutations_with_consistent_phenotypes_notin_train)) & (validation_mutations.TOTAL<4)].MUTATION.unique()))\n",
    "\n",
    "for i in ['vs-consis-notrain-large', 'vs-consis-notrain-small']:\n",
    "    X[i] = {}\n",
    "    Y[i] = {}\n",
    "    Z[i] = {}\n",
    "\n",
    "mask = numpy.isin(Z['validation-samples']['input'],  numpy.array(list(mutations_with_consistent_phenotypes_notin_train_large)))\n",
    "X['vs-consis-notrain-large']['input']=X['validation-samples']['input'][mask]\n",
    "Z['vs-consis-notrain-large']['input']=Z['validation-samples']['input'][mask]\n",
    "Y['vs-consis-notrain-large']['input']=Y['validation-samples']['input'][mask]\n",
    "\n",
    "mask = numpy.isin(Z['validation-samples']['input'],  numpy.array(list(mutations_with_consistent_phenotypes_notin_train_small)))\n",
    "X['vs-consis-notrain-small']['input']=X['validation-samples']['input'][mask]\n",
    "Z['vs-consis-notrain-small']['input']=Z['validation-samples']['input'][mask]\n",
    "Y['vs-consis-notrain-small']['input']=Y['validation-samples']['input'][mask]\n",
    "\n",
    "for dataset in ['vs-consis-notrain-large', 'vs-consis-notrain-small']:\n",
    "    print(dataset)\n",
    "    line = validate_model(line, best_model[chosen_model], chosen_model, dataset, X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'resid 45 82 21 175 135 119 157 24 10 130 146 8 68 182 97 79 12 125 46 120 103 57 97 100 156 180 128 108 35 94 47 172 69 7 146 143 8 49 62 81 21 10 159 143 1 65 71 116 63 67 96'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'resid '+ ' '.join([i[1:-1] for i in mutations_with_consistent_phenotypes_notin_train_large])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'resid 172 102 68 57 68 13 10 10 6 46 43 57 146 137 119 104 102 81 58 168 167 119 140 90 6 15 43 124 157 17 48 13'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'resid '+ ' '.join([i[1:-1] for i in mutations_with_consistent_phenotypes_notin_train_small])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>dataset</th>\n",
       "      <th>sensitivity_mean</th>\n",
       "      <th>sensitivity_std</th>\n",
       "      <th>specificity_mean</th>\n",
       "      <th>specificity_std</th>\n",
       "      <th>roc_auc_mean</th>\n",
       "      <th>roc_auc_std</th>\n",
       "      <th>TN</th>\n",
       "      <th>FP</th>\n",
       "      <th>FN</th>\n",
       "      <th>TP</th>\n",
       "      <th>model_parameters</th>\n",
       "      <th>n_samples</th>\n",
       "      <th>n_mutations</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>XB</td>\n",
       "      <td>validation-samples</td>\n",
       "      <td>97.198276</td>\n",
       "      <td>None</td>\n",
       "      <td>46.017699</td>\n",
       "      <td>None</td>\n",
       "      <td>80.722592</td>\n",
       "      <td>None</td>\n",
       "      <td>572</td>\n",
       "      <td>671</td>\n",
       "      <td>78</td>\n",
       "      <td>2706</td>\n",
       "      <td>null</td>\n",
       "      <td>4027</td>\n",
       "      <td>367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>XB</td>\n",
       "      <td>validation-mutations</td>\n",
       "      <td>97.419355</td>\n",
       "      <td>None</td>\n",
       "      <td>59.090909</td>\n",
       "      <td>None</td>\n",
       "      <td>88.284457</td>\n",
       "      <td>None</td>\n",
       "      <td>26</td>\n",
       "      <td>18</td>\n",
       "      <td>4</td>\n",
       "      <td>151</td>\n",
       "      <td>null</td>\n",
       "      <td>199</td>\n",
       "      <td>199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>XB</td>\n",
       "      <td>mic</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>None</td>\n",
       "      <td>14.285714</td>\n",
       "      <td>None</td>\n",
       "      <td>67.714286</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "      <td>null</td>\n",
       "      <td>57</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>XB</td>\n",
       "      <td>vs-incons</td>\n",
       "      <td>95.127119</td>\n",
       "      <td>None</td>\n",
       "      <td>14.449541</td>\n",
       "      <td>None</td>\n",
       "      <td>57.618615</td>\n",
       "      <td>None</td>\n",
       "      <td>63</td>\n",
       "      <td>373</td>\n",
       "      <td>23</td>\n",
       "      <td>449</td>\n",
       "      <td>null</td>\n",
       "      <td>908</td>\n",
       "      <td>168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>XB</td>\n",
       "      <td>vs-consis</td>\n",
       "      <td>97.621107</td>\n",
       "      <td>None</td>\n",
       "      <td>63.073110</td>\n",
       "      <td>None</td>\n",
       "      <td>86.880823</td>\n",
       "      <td>None</td>\n",
       "      <td>509</td>\n",
       "      <td>298</td>\n",
       "      <td>55</td>\n",
       "      <td>2257</td>\n",
       "      <td>null</td>\n",
       "      <td>3119</td>\n",
       "      <td>199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>XB</td>\n",
       "      <td>vm-consis</td>\n",
       "      <td>97.419355</td>\n",
       "      <td>None</td>\n",
       "      <td>59.090909</td>\n",
       "      <td>None</td>\n",
       "      <td>88.284457</td>\n",
       "      <td>None</td>\n",
       "      <td>26</td>\n",
       "      <td>18</td>\n",
       "      <td>4</td>\n",
       "      <td>151</td>\n",
       "      <td>null</td>\n",
       "      <td>199</td>\n",
       "      <td>199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>XB</td>\n",
       "      <td>vs-consis-train</td>\n",
       "      <td>98.250525</td>\n",
       "      <td>None</td>\n",
       "      <td>75.609756</td>\n",
       "      <td>None</td>\n",
       "      <td>89.577224</td>\n",
       "      <td>None</td>\n",
       "      <td>465</td>\n",
       "      <td>150</td>\n",
       "      <td>25</td>\n",
       "      <td>1404</td>\n",
       "      <td>null</td>\n",
       "      <td>2044</td>\n",
       "      <td>116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>XB</td>\n",
       "      <td>vs-consis-notrain</td>\n",
       "      <td>96.602492</td>\n",
       "      <td>None</td>\n",
       "      <td>22.916667</td>\n",
       "      <td>None</td>\n",
       "      <td>75.711058</td>\n",
       "      <td>None</td>\n",
       "      <td>44</td>\n",
       "      <td>148</td>\n",
       "      <td>30</td>\n",
       "      <td>853</td>\n",
       "      <td>null</td>\n",
       "      <td>1075</td>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>XB</td>\n",
       "      <td>vs-incons-train</td>\n",
       "      <td>96.885813</td>\n",
       "      <td>None</td>\n",
       "      <td>9.016393</td>\n",
       "      <td>None</td>\n",
       "      <td>56.030546</td>\n",
       "      <td>None</td>\n",
       "      <td>22</td>\n",
       "      <td>222</td>\n",
       "      <td>9</td>\n",
       "      <td>280</td>\n",
       "      <td>null</td>\n",
       "      <td>533</td>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>XB</td>\n",
       "      <td>vs-incons-notrain</td>\n",
       "      <td>92.349727</td>\n",
       "      <td>None</td>\n",
       "      <td>21.354167</td>\n",
       "      <td>None</td>\n",
       "      <td>61.111111</td>\n",
       "      <td>None</td>\n",
       "      <td>41</td>\n",
       "      <td>151</td>\n",
       "      <td>14</td>\n",
       "      <td>169</td>\n",
       "      <td>null</td>\n",
       "      <td>375</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>XB</td>\n",
       "      <td>vm-consis-train</td>\n",
       "      <td>98.969072</td>\n",
       "      <td>None</td>\n",
       "      <td>73.684211</td>\n",
       "      <td>None</td>\n",
       "      <td>92.783505</td>\n",
       "      <td>None</td>\n",
       "      <td>14</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>96</td>\n",
       "      <td>null</td>\n",
       "      <td>116</td>\n",
       "      <td>116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>XB</td>\n",
       "      <td>vm-consis-notrain</td>\n",
       "      <td>94.827586</td>\n",
       "      <td>None</td>\n",
       "      <td>48.000000</td>\n",
       "      <td>None</td>\n",
       "      <td>81.931034</td>\n",
       "      <td>None</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>3</td>\n",
       "      <td>55</td>\n",
       "      <td>null</td>\n",
       "      <td>83</td>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>XB</td>\n",
       "      <td>vs-consis-notrain-large</td>\n",
       "      <td>96.998800</td>\n",
       "      <td>None</td>\n",
       "      <td>17.283951</td>\n",
       "      <td>None</td>\n",
       "      <td>75.152654</td>\n",
       "      <td>None</td>\n",
       "      <td>28</td>\n",
       "      <td>134</td>\n",
       "      <td>25</td>\n",
       "      <td>808</td>\n",
       "      <td>null</td>\n",
       "      <td>995</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>XB</td>\n",
       "      <td>vs-consis-notrain-small</td>\n",
       "      <td>90.000000</td>\n",
       "      <td>None</td>\n",
       "      <td>53.333333</td>\n",
       "      <td>None</td>\n",
       "      <td>72.600000</td>\n",
       "      <td>None</td>\n",
       "      <td>16</td>\n",
       "      <td>14</td>\n",
       "      <td>5</td>\n",
       "      <td>45</td>\n",
       "      <td>null</td>\n",
       "      <td>80</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   model                  dataset  sensitivity_mean sensitivity_std  \\\n",
       "0     XB       validation-samples         97.198276            None   \n",
       "1     XB     validation-mutations         97.419355            None   \n",
       "2     XB                      mic        100.000000            None   \n",
       "3     XB                vs-incons         95.127119            None   \n",
       "4     XB                vs-consis         97.621107            None   \n",
       "5     XB                vm-consis         97.419355            None   \n",
       "6     XB          vs-consis-train         98.250525            None   \n",
       "7     XB        vs-consis-notrain         96.602492            None   \n",
       "8     XB          vs-incons-train         96.885813            None   \n",
       "9     XB        vs-incons-notrain         92.349727            None   \n",
       "10    XB          vm-consis-train         98.969072            None   \n",
       "11    XB        vm-consis-notrain         94.827586            None   \n",
       "12    XB  vs-consis-notrain-large         96.998800            None   \n",
       "13    XB  vs-consis-notrain-small         90.000000            None   \n",
       "\n",
       "    specificity_mean specificity_std  roc_auc_mean roc_auc_std   TN   FP  FN  \\\n",
       "0          46.017699            None     80.722592        None  572  671  78   \n",
       "1          59.090909            None     88.284457        None   26   18   4   \n",
       "2          14.285714            None     67.714286        None    1    6   0   \n",
       "3          14.449541            None     57.618615        None   63  373  23   \n",
       "4          63.073110            None     86.880823        None  509  298  55   \n",
       "5          59.090909            None     88.284457        None   26   18   4   \n",
       "6          75.609756            None     89.577224        None  465  150  25   \n",
       "7          22.916667            None     75.711058        None   44  148  30   \n",
       "8           9.016393            None     56.030546        None   22  222   9   \n",
       "9          21.354167            None     61.111111        None   41  151  14   \n",
       "10         73.684211            None     92.783505        None   14    5   1   \n",
       "11         48.000000            None     81.931034        None   12   13   3   \n",
       "12         17.283951            None     75.152654        None   28  134  25   \n",
       "13         53.333333            None     72.600000        None   16   14   5   \n",
       "\n",
       "      TP model_parameters  n_samples  n_mutations  \n",
       "0   2706             null       4027          367  \n",
       "1    151             null        199          199  \n",
       "2     50             null         57           57  \n",
       "3    449             null        908          168  \n",
       "4   2257             null       3119          199  \n",
       "5    151             null        199          199  \n",
       "6   1404             null       2044          116  \n",
       "7    853             null       1075           83  \n",
       "8    280             null        533           71  \n",
       "9    169             null        375           97  \n",
       "10    96             null        116          116  \n",
       "11    55             null         83           83  \n",
       "12   808             null        995           51  \n",
       "13    45             null         80           32  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_results = pandas.DataFrame(line, columns=['model', 'dataset', 'sensitivity_mean', 'sensitivity_std', 'specificity_mean', 'specificity_std' ,'roc_auc_mean', 'roc_auc_std','TN','FP','FN','TP', 'model_parameters', 'n_samples', 'n_mutations'])\n",
    "test_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation-samples\n",
      "validation-mutations"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "mic\n",
      "vs-incons\n",
      "vs-consis\n",
      "vm-consis\n",
      "vs-consis-train\n",
      "vs-consis-notrain\n",
      "vs-incons-train\n",
      "vs-incons-notrain\n",
      "vm-consis-train\n",
      "vm-consis-notrain\n",
      "vs-consis-notrain-large\n",
      "vs-consis-notrain-small\n"
     ]
    }
   ],
   "source": [
    "for dataset in list(test_results.dataset):\n",
    "    print(dataset)\n",
    "    colour='#888888'\n",
    "    fig = plt.figure(figsize=(1.8, 3.5))\n",
    "    axes = plt.gca()\n",
    "    axes.spines['top'].set_visible(False)\n",
    "    axes.spines['right'].set_visible(False)\n",
    "    axes.spines['left'].set_visible(False)\n",
    "    axes.get_yaxis().set_visible(False)\n",
    "    x=[0,1]\n",
    "    y=test_results[(test_results.dataset==dataset)][['sensitivity_mean','specificity_mean']].values[0]\n",
    "    axes.set_ylim([0,100])\n",
    "    axes.bar(x,y, label=y, edgecolor=colour, color='None',linewidth=2)\n",
    "    axes.set_xticks([0,1])\n",
    "    axes.set_xticklabels(['sensitivity', 'specificity'])\n",
    "    for (i,j) in zip(x,y):\n",
    "        axes.text(i,j+2,'%.1f' % j,ha='center',color=colour)\n",
    "    fig.savefig('pdf/figure-5/fig-5-'+str(dataset)+'.pdf',bbox_inches='tight',transparent=True)\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx,row in test_results.iterrows():\n",
    "\n",
    "    fig = plt.figure(figsize=(1.5, 1.5))\n",
    "    axes = plt.gca()\n",
    "\n",
    "    axes.add_patch(Rectangle((0,0),1,1,fc='#e41a1c',alpha=0.7))\n",
    "    axes.add_patch(Rectangle((0,1),1,1,fc='#4daf4a',alpha=0.7))\n",
    "    axes.add_patch(Rectangle((1,1),1,1,fc='#fc9272',alpha=0.7))\n",
    "    axes.add_patch(Rectangle((1,0),1,1,fc='#4daf4a',alpha=0.7))\n",
    "\n",
    "    axes.set_xlim([0,2])\n",
    "    axes.set_ylim([0,2])\n",
    "\n",
    "    axes.set_xticks([0.5,1.5],labels=['R','S'])\n",
    "    axes.set_yticks([0.5,1.5],labels=['S','R'])\n",
    "\n",
    "    axes.text(0.5,0.5,row['FN'],ha='center',va='center')\n",
    "    axes.text(1.5,0.5,row['TN'],ha='center',va='center')\n",
    "    axes.text(0.5,1.5,row['TP'],ha='center',va='center')\n",
    "    axes.text(1.5,1.5,row['FP'],ha='center',va='center')\n",
    "\n",
    "    fig.savefig('pdf/figure-5/truthtable-'+row['dataset']+'-'+row['model']+'.pdf',bbox_inches='tight',transparent=True)\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(995, 51)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_lt4 = validation_mutations[(validation_mutations.MUTATION.isin(mutations_with_consistent_phenotypes_notin_train)) & (validation_mutations.TOTAL>=4)].TOTAL.sum()\n",
    "len_lt4 = len(validation_mutations[(validation_mutations.MUTATION.isin(mutations_with_consistent_phenotypes_notin_train)) & (validation_mutations.TOTAL>=4)])\n",
    "total_lt4, len_lt4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MUTATION</th>\n",
       "      <th>IS_SNP</th>\n",
       "      <th>IN_CDS</th>\n",
       "      <th>R</th>\n",
       "      <th>S</th>\n",
       "      <th>TOTAL</th>\n",
       "      <th>RELIABLE_PHENOTYPE</th>\n",
       "      <th>CONSISTENT_PHENOTYPE</th>\n",
       "      <th>IS_NONSYN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A102P</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>False</td>\n",
       "      <td>U</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A102R</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>True</td>\n",
       "      <td>S</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A102T</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>True</td>\n",
       "      <td>R</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A102V</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>25</td>\n",
       "      <td>False</td>\n",
       "      <td>U</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A134D</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>U</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>362</th>\n",
       "      <td>Y41C</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>U</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>Y64D</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>U</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>Y64S</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>False</td>\n",
       "      <td>U</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365</th>\n",
       "      <td>Y95D</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>U</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>366</th>\n",
       "      <td>Y99C</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>U</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>367 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    MUTATION  IS_SNP  IN_CDS   R   S  TOTAL  RELIABLE_PHENOTYPE  \\\n",
       "0      A102P    True    True   8   4     12               False   \n",
       "1      A102R    True    True   0   3      3                True   \n",
       "2      A102T    True    True   3   0      3                True   \n",
       "3      A102V    True    True  12  13     25               False   \n",
       "4      A134D    True    True   0   1      1               False   \n",
       "..       ...     ...     ...  ..  ..    ...                 ...   \n",
       "362     Y41C    True    True   0   1      1               False   \n",
       "363     Y64D    True    True   2   4      6               False   \n",
       "364     Y64S    True    True   1   2      3               False   \n",
       "365     Y95D    True    True   1   0      1               False   \n",
       "366     Y99C    True    True   0   1      1               False   \n",
       "\n",
       "    CONSISTENT_PHENOTYPE  IS_NONSYN  \n",
       "0                      U       True  \n",
       "1                      S       True  \n",
       "2                      R       True  \n",
       "3                      U       True  \n",
       "4                      U       True  \n",
       "..                   ...        ...  \n",
       "362                    U       True  \n",
       "363                    U       True  \n",
       "364                    U       True  \n",
       "365                    U       True  \n",
       "366                    U       True  \n",
       "\n",
       "[367 rows x 9 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation_mutations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}