Benchmark.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install pandas numpy scikit-learn ucimlrepo lime bayesian-optimization shap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.preprocessing import OneHotEncoder, Normalizer, LabelEncoder\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.mixture import GaussianMixture, BayesianGaussianMixture\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "from ucimlrepo import fetch_ucirepo \n",
    "import lime\n",
    "from lime import lime_tabular\n",
    "\n",
    "import pickle as pkl\n",
    "\n",
    "from bayes_opt import BayesianOptimization\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from scipy.stats import uniform, norm\n",
    "from scipy.special import softmax\n",
    "import shap\n",
    "\n",
    "from local2global import Local2GlobalExplainer\n",
    "\n",
    "MODEL_NAME = 'Wine'\n",
    "\n",
    "MODEL_FUNCTION = DecisionTreeClassifier\n",
    "model_params = {'random_state': 1340304}\n",
    "\n",
    "coding = {\"Wine\": 109, 'German Credit': 144}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fetch dataset \n",
    "split_size = 0.3\n",
    "seed = 1234567\n",
    "\n",
    "dataset_id = coding[MODEL_NAME]\n",
    "\n",
    "wine = fetch_ucirepo(id=109) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# data (as pandas dataframes) \n",
    "X = wine.data.features\n",
    "y = wine.data.targets.to_numpy()\n",
    "y = y.reshape((len(y), ))\n",
    "    \n",
    "\n",
    "n_classes = len(np.unique(y))\n",
    "n_feats = X.shape[1]\n",
    "x_train, x_test, y_train, y_test = train_test_split(X, y, stratify = y, test_size = split_size, random_state = seed)\n",
    "\n",
    "normalizer = Normalizer().fit(x_train)\n",
    "encoder = LabelEncoder().fit(y_train)\n",
    "\n",
    "x_train = normalizer.transform(x_train)\n",
    "x_test = normalizer.transform(x_test)\n",
    "y_train = encoder.transform(y_train)\n",
    "y_test  = encoder.transform(y_test)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_model(model, samples, targets):\n",
    "    pred = model.predict(samples)\n",
    "    return accuracy_score(targets, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MODEL_FUNCTION(**model_params)\n",
    "model.fit(x_train, y_train)\n",
    "\n",
    "acc_train = evaluate_model(model, x_train, y_train)\n",
    "acc_test  = evaluate_model(model, x_test, y_test)\n",
    "\n",
    "print(acc_train, acc_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Interpretability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_shap_feature_importances(model, x_train):\n",
    "    '''Get scores with shap'''\n",
    "    samples = np.random.choice(list(range(len(x_train))), min(5, len(x_train)), replace = False)\n",
    "    se = shap.KernelExplainer(model.predict, x_train[samples])\n",
    "    shap_values = se.shap_values(x_train)\n",
    "    importance_order = np.argsort(-abs(np.abs(np.array(shap_values)).mean(axis = 0)))\n",
    "    importances = np.abs(np.array(shap_values)).mean(axis = 0)[importance_order]\n",
    "    return importance_order, importances\n",
    "\n",
    "order, importances = get_shap_feature_importances(model, x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_model_feats_removed(x_train, y_train, x_test, y_test, n_feats, feat_indices, k = 4, trials = 15):\n",
    "    selected_feats = list(set(np.arange(n_feats)).difference(feat_indices[:k]))      # exclude top k features\n",
    "    x_t = x_train[:, selected_feats]\n",
    "    x_tst = x_test[:, selected_feats]\n",
    "    \n",
    "    train_accs = []\n",
    "    test_accs = []\n",
    "    \n",
    "    for i in range(trials):\n",
    "\n",
    "        m = MODEL_FUNCTION(**model_params).fit(x_t, y_train)\n",
    "        acc_train = evaluate_model(m, x_t, y_train)\n",
    "        acc_test  = evaluate_model(m, x_tst, y_test)\n",
    "        \n",
    "        train_accs.append(acc_train)\n",
    "        test_accs.append(acc_test)\n",
    "\n",
    "        \n",
    "    train_accs, test_accs = np.array(train_accs), np.array(test_accs)\n",
    "    \n",
    "    return {'train_acc': train_accs.mean(),\n",
    "            'train_var': train_accs.var(),\n",
    "            'test_acc': test_accs.mean(),\n",
    "            'test_var': test_accs.var()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Local2Global"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#setup\n",
    "\n",
    "num_trials = 500\n",
    "\n",
    "#shap\n",
    "order_shap, _ = get_shap_feature_importances(model, x_train)\n",
    "\n",
    "#fit\n",
    "l2g_exp = Local2GlobalExplainer(x_train, model, n_classes)\n",
    "mc_exp, _ = l2g_exp.mcmc_estimate(num_trials)\n",
    "is_exp, _ = l2g_exp.importance_sampling(num_trials)\n",
    "\n",
    "#order calculation\n",
    "order_mc = l2g_exp.get_only_feature_importance(mc_exp)\n",
    "order_is = l2g_exp.get_only_feature_importance(is_exp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_feats_removed_limit = 6\n",
    "trials = 100\n",
    "\n",
    "scores = pd.DataFrame()\n",
    "\n",
    "for rem_feats in range(1, num_feats_removed_limit+1):\n",
    "    shap_scores =  evaluate_model_feats_removed(x_train, y_train, x_test, y_test, n_feats, order_shap, k = rem_feats, trials = trials)\n",
    "    s_df = pd.DataFrame(shap_scores, index = [0])\n",
    "    s_df = s_df.drop(columns = ['train_acc', 'train_var']).rename(columns = {'test_acc': '(SHAP) Test acc', 'test_var': '(SHAP) Test var'})\n",
    "    \n",
    "    mc_scores =  evaluate_model_feats_removed(x_train, y_train, x_test, y_test, n_feats, order_mc, k = rem_feats, trials = trials)\n",
    "    mc_df = pd.DataFrame(mc_scores, index = [0])\n",
    "    mc_df =mc_df.drop(columns = ['train_acc', 'train_var']).rename(columns = {'test_acc': '(MCMC) Test acc', 'test_var': '(MCMC) Test var'})\n",
    "    \n",
    "    \n",
    "    is_scores =  evaluate_model_feats_removed(x_train, y_train, x_test, y_test, n_feats, order_is, k = rem_feats, trials = trials)\n",
    "    is_df = pd.DataFrame(is_scores, index = [0])\n",
    "    is_df = is_df.drop(columns = ['train_acc', 'train_var']).rename(columns = {'test_acc': '(Imp. Samp.) Test acc', 'test_var': '(Imp. Samp.) Test var'})\n",
    "    \n",
    "    \n",
    "    test = pd.concat([s_df, mc_df, is_df], axis = 1)\n",
    "    test['Features Removed'] = rem_feats\n",
    "    \n",
    "    scores = pd.concat([scores, test], axis = 0)\n",
    "\n",
    "scores = scores.reset_index(drop = True)\n",
    "scores['Dataset'] = MODEL_NAME\n",
    "\n",
    "order = list(scores.columns[-2:]) + list(scores.columns[:-2])\n",
    "scores = scores[order].set_index(\"Features Removed\")\n",
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores.to_csv(f\"Benchmarks/benchmarks_{MODEL_NAME}.csv\", index = True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.11.5 ('torchnew')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "837d38702d68a55c15eaac5c5e02ad740c2103af87bc8f81f0c8506afc645a88"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}