MalML.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import sklearn.feature_extraction\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import os, warnings \n",
    "warnings.filterwarnings('ignore', category=DeprecationWarning)\n",
    "%matplotlib inline "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_json(filelist):\n",
    "    flist = []\n",
    "    for filename in filelist:\n",
    "        with open(filename) as f:\n",
    "            flist.append(eval(f.read()))         \n",
    "    return flist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load bad \n",
    "filelist = [os.path.join('data/bad-json', child) for child in os.listdir('data/bad-json')]\n",
    "badfeatures = load_json(filelist)\n",
    "# load good\n",
    "filelist = [os.path.join('data/good-json', child) for child in os.listdir('data/good-json')]\n",
    "goodfeatures = load_json(filelist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>check_sum</th>\n",
       "      <th>compile_date</th>\n",
       "      <th>datadir_IMAGE_DIRECTORY_ENTRY_BASERELOC_size</th>\n",
       "      <th>datadir_IMAGE_DIRECTORY_ENTRY_EXPORT_size</th>\n",
       "      <th>datadir_IMAGE_DIRECTORY_ENTRY_IAT_size</th>\n",
       "      <th>datadir_IMAGE_DIRECTORY_ENTRY_IMPORT_size</th>\n",
       "      <th>datadir_IMAGE_DIRECTORY_ENTRY_RESOURCE_size</th>\n",
       "      <th>debug_size</th>\n",
       "      <th>export_size</th>\n",
       "      <th>generated_check_sum</th>\n",
       "      <th>...</th>\n",
       "      <th>size_code</th>\n",
       "      <th>size_image</th>\n",
       "      <th>size_initdata</th>\n",
       "      <th>size_uninit</th>\n",
       "      <th>std_section_names</th>\n",
       "      <th>total_size_pe</th>\n",
       "      <th>virtual_address</th>\n",
       "      <th>virtual_size</th>\n",
       "      <th>virtual_size_2</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13123</td>\n",
       "      <td>1247295473</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9136</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13123</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>16384</td>\n",
       "      <td>9216</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>9728</td>\n",
       "      <td>4096</td>\n",
       "      <td>12288</td>\n",
       "      <td>0</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1005787136</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1336</td>\n",
       "      <td>260</td>\n",
       "      <td>9208</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>263396</td>\n",
       "      <td>...</td>\n",
       "      <td>151552</td>\n",
       "      <td>241664</td>\n",
       "      <td>86016</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>225280</td>\n",
       "      <td>4096</td>\n",
       "      <td>148815</td>\n",
       "      <td>36246</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>556650</td>\n",
       "      <td>1256845306</td>\n",
       "      <td>27892</td>\n",
       "      <td>193</td>\n",
       "      <td>956</td>\n",
       "      <td>260</td>\n",
       "      <td>37272</td>\n",
       "      <td>28</td>\n",
       "      <td>193</td>\n",
       "      <td>556650</td>\n",
       "      <td>...</td>\n",
       "      <td>459264</td>\n",
       "      <td>552960</td>\n",
       "      <td>77312</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>537600</td>\n",
       "      <td>4096</td>\n",
       "      <td>458305</td>\n",
       "      <td>184</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>44616</td>\n",
       "      <td>961014673</td>\n",
       "      <td>40</td>\n",
       "      <td>7517</td>\n",
       "      <td>28</td>\n",
       "      <td>86</td>\n",
       "      <td>10960</td>\n",
       "      <td>28</td>\n",
       "      <td>7517</td>\n",
       "      <td>44616</td>\n",
       "      <td>...</td>\n",
       "      <td>8704</td>\n",
       "      <td>32768</td>\n",
       "      <td>11776</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>21776</td>\n",
       "      <td>4096</td>\n",
       "      <td>8444</td>\n",
       "      <td>10960</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>449571</td>\n",
       "      <td>1285732079</td>\n",
       "      <td>8</td>\n",
       "      <td>52</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>398132</td>\n",
       "      <td>28</td>\n",
       "      <td>52</td>\n",
       "      <td>449571</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>413696</td>\n",
       "      <td>399360</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>406352</td>\n",
       "      <td>4096</td>\n",
       "      <td>120</td>\n",
       "      <td>401408</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 492 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   check_sum  compile_date  datadir_IMAGE_DIRECTORY_ENTRY_BASERELOC_size  \\\n",
       "0      13123    1247295473                                             0   \n",
       "1          0    1005787136                                             0   \n",
       "2     556650    1256845306                                         27892   \n",
       "3      44616     961014673                                            40   \n",
       "4     449571    1285732079                                             8   \n",
       "\n",
       "   datadir_IMAGE_DIRECTORY_ENTRY_EXPORT_size  \\\n",
       "0                                          0   \n",
       "1                                          0   \n",
       "2                                        193   \n",
       "3                                       7517   \n",
       "4                                         52   \n",
       "\n",
       "   datadir_IMAGE_DIRECTORY_ENTRY_IAT_size  \\\n",
       "0                                       0   \n",
       "1                                    1336   \n",
       "2                                     956   \n",
       "3                                      28   \n",
       "4                                       0   \n",
       "\n",
       "   datadir_IMAGE_DIRECTORY_ENTRY_IMPORT_size  \\\n",
       "0                                          0   \n",
       "1                                        260   \n",
       "2                                        260   \n",
       "3                                         86   \n",
       "4                                          0   \n",
       "\n",
       "   datadir_IMAGE_DIRECTORY_ENTRY_RESOURCE_size  debug_size  export_size  \\\n",
       "0                                         9136           0            0   \n",
       "1                                         9208           0            0   \n",
       "2                                        37272          28          193   \n",
       "3                                        10960          28         7517   \n",
       "4                                       398132          28           52   \n",
       "\n",
       "   generated_check_sum  ...    size_code  size_image  size_initdata  \\\n",
       "0                13123  ...            0       16384           9216   \n",
       "1               263396  ...       151552      241664          86016   \n",
       "2               556650  ...       459264      552960          77312   \n",
       "3                44616  ...         8704       32768          11776   \n",
       "4               449571  ...            0      413696         399360   \n",
       "\n",
       "   size_uninit  std_section_names  total_size_pe  virtual_address  \\\n",
       "0            0                  1           9728             4096   \n",
       "1            0                  1         225280             4096   \n",
       "2            0                  0         537600             4096   \n",
       "3            0                  1          21776             4096   \n",
       "4            0                  1         406352             4096   \n",
       "\n",
       "   virtual_size  virtual_size_2  label  \n",
       "0         12288               0   good  \n",
       "1        148815           36246   good  \n",
       "2        458305             184   good  \n",
       "3          8444           10960   good  \n",
       "4           120          401408   good  \n",
       "\n",
       "[5 rows x 492 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bad = pd.DataFrame.from_records(badfeatures)\n",
    "df_bad['label'] = 'bad'\n",
    "df_good = pd.DataFrame.from_records(goodfeatures)\n",
    "df_good['label'] = 'good'\n",
    "df_good.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>check_sum</th>\n",
       "      <th>compile_date</th>\n",
       "      <th>datadir_IMAGE_DIRECTORY_ENTRY_BASERELOC_size</th>\n",
       "      <th>datadir_IMAGE_DIRECTORY_ENTRY_EXPORT_size</th>\n",
       "      <th>datadir_IMAGE_DIRECTORY_ENTRY_IAT_size</th>\n",
       "      <th>datadir_IMAGE_DIRECTORY_ENTRY_IMPORT_size</th>\n",
       "      <th>datadir_IMAGE_DIRECTORY_ENTRY_RESOURCE_size</th>\n",
       "      <th>debug_size</th>\n",
       "      <th>export_size</th>\n",
       "      <th>generated_check_sum</th>\n",
       "      <th>...</th>\n",
       "      <th>sec_vasize_zwt\\u0007</th>\n",
       "      <th>size_code</th>\n",
       "      <th>size_image</th>\n",
       "      <th>size_initdata</th>\n",
       "      <th>size_uninit</th>\n",
       "      <th>std_section_names</th>\n",
       "      <th>total_size_pe</th>\n",
       "      <th>virtual_address</th>\n",
       "      <th>virtual_size</th>\n",
       "      <th>virtual_size_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>175316</td>\n",
       "      <td>1329360236</td>\n",
       "      <td>2152</td>\n",
       "      <td>0</td>\n",
       "      <td>312</td>\n",
       "      <td>60</td>\n",
       "      <td>58056</td>\n",
       "      <td>28</td>\n",
       "      <td>0</td>\n",
       "      <td>310358</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>36352</td>\n",
       "      <td>126976</td>\n",
       "      <td>77824</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>294366</td>\n",
       "      <td>4096</td>\n",
       "      <td>36182</td>\n",
       "      <td>6752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1134194116</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>692</td>\n",
       "      <td>180</td>\n",
       "      <td>36864</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373180</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>25600</td>\n",
       "      <td>618496</td>\n",
       "      <td>278528</td>\n",
       "      <td>8192</td>\n",
       "      <td>0</td>\n",
       "      <td>308152</td>\n",
       "      <td>4096</td>\n",
       "      <td>25270</td>\n",
       "      <td>4554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>339801</td>\n",
       "      <td>1512937434</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>276</td>\n",
       "      <td>40</td>\n",
       "      <td>2560</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>339801</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>262144</td>\n",
       "      <td>274432</td>\n",
       "      <td>8192</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>274432</td>\n",
       "      <td>4096</td>\n",
       "      <td>260396</td>\n",
       "      <td>2588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1279145053</td>\n",
       "      <td>0</td>\n",
       "      <td>654311424</td>\n",
       "      <td>328</td>\n",
       "      <td>120</td>\n",
       "      <td>1944</td>\n",
       "      <td>0</td>\n",
       "      <td>654311424</td>\n",
       "      <td>233979</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24576</td>\n",
       "      <td>206848</td>\n",
       "      <td>176128</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>201216</td>\n",
       "      <td>4096</td>\n",
       "      <td>19176</td>\n",
       "      <td>170388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>145303</td>\n",
       "      <td>1314010393</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>564</td>\n",
       "      <td>40</td>\n",
       "      <td>10312</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>195686</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>102400</td>\n",
       "      <td>135168</td>\n",
       "      <td>28672</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>135168</td>\n",
       "      <td>4096</td>\n",
       "      <td>100888</td>\n",
       "      <td>1540</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1284 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   check_sum  compile_date  datadir_IMAGE_DIRECTORY_ENTRY_BASERELOC_size  \\\n",
       "0     175316    1329360236                                          2152   \n",
       "1          0    1134194116                                             0   \n",
       "2     339801    1512937434                                             0   \n",
       "3          0    1279145053                                             0   \n",
       "4     145303    1314010393                                             0   \n",
       "\n",
       "   datadir_IMAGE_DIRECTORY_ENTRY_EXPORT_size  \\\n",
       "0                                          0   \n",
       "1                                          0   \n",
       "2                                          0   \n",
       "3                                  654311424   \n",
       "4                                          0   \n",
       "\n",
       "   datadir_IMAGE_DIRECTORY_ENTRY_IAT_size  \\\n",
       "0                                     312   \n",
       "1                                     692   \n",
       "2                                     276   \n",
       "3                                     328   \n",
       "4                                     564   \n",
       "\n",
       "   datadir_IMAGE_DIRECTORY_ENTRY_IMPORT_size  \\\n",
       "0                                         60   \n",
       "1                                        180   \n",
       "2                                         40   \n",
       "3                                        120   \n",
       "4                                         40   \n",
       "\n",
       "   datadir_IMAGE_DIRECTORY_ENTRY_RESOURCE_size  debug_size  export_size  \\\n",
       "0                                        58056          28            0   \n",
       "1                                        36864           0            0   \n",
       "2                                         2560           0            0   \n",
       "3                                         1944           0    654311424   \n",
       "4                                        10312           0            0   \n",
       "\n",
       "   generated_check_sum       ...        sec_vasize_zwt\\u0007 size_code  \\\n",
       "0               310358       ...                         0.0     36352   \n",
       "1               373180       ...                         0.0     25600   \n",
       "2               339801       ...                         0.0    262144   \n",
       "3               233979       ...                         0.0     24576   \n",
       "4               195686       ...                         0.0    102400   \n",
       "\n",
       "   size_image  size_initdata  size_uninit  std_section_names  total_size_pe  \\\n",
       "0      126976          77824            0                  0         294366   \n",
       "1      618496         278528         8192                  0         308152   \n",
       "2      274432           8192            0                  1         274432   \n",
       "3      206848         176128            0                  0         201216   \n",
       "4      135168          28672            0                  1         135168   \n",
       "\n",
       "   virtual_address  virtual_size  virtual_size_2  \n",
       "0             4096         36182            6752  \n",
       "1             4096         25270            4554  \n",
       "2             4096        260396            2588  \n",
       "3             4096         19176          170388  \n",
       "4             4096        100888            1540  \n",
       "\n",
       "[5 rows x 1284 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.concat([df_bad, df_good], ignore_index=True)\n",
    "df.replace(np.nan, 0, inplace=True)        \n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = df.as_matrix(['number_of_import_symbols', 'number_of_sections'])\n",
    "y = np.array(df['label'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "my_seed = 100\n",
    "my_tsize = .2\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=my_tsize, random_state=my_seed)  \n",
    "\n",
    "import sklearn.ensemble\n",
    "clf = sklearn.ensemble.RandomForestClassifier(n_estimators=30)\n",
    "clf.fit(X_train, y_train)\n",
    "y_pred = clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_cm(cm, labels):              \n",
    "    # Compute percentanges\n",
    "    percent = (cm*100.0)/np.array(np.matrix(cm.sum(axis=1)).T)  # Derp, I'm sure there's a better way   \n",
    "    print 'Confusion Matrix Stats'\n",
    "    for i, label_i in enumerate(labels):\n",
    "        for j, label_j in enumerate(labels):\n",
    "            print \"%s/%s: %.2f%% (%d/%d)\" % (label_i, label_j, (percent[i][j]), cm[i][j], cm[i].sum())\n",
    "\n",
    "    # Show confusion matrix\n",
    "    # Thanks kermit666 from stackoverflow :)\n",
    "    fig = plt.figure()\n",
    "    ax = fig.add_subplot(111)\n",
    "    ax.grid(b=False)\n",
    "    cax = ax.matshow(percent, cmap='coolwarm',vmin=0,vmax=100)\n",
    "    plt.title('Confusion matrix of the classifier')\n",
    "    fig.colorbar(cax)\n",
    "    ax.set_xticklabels([''] + labels)\n",
    "    ax.set_yticklabels([''] + labels)\n",
    "    plt.xlabel('Predicted')\n",
    "    plt.ylabel('True')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix Stats\n",
      "good/good: 74.51% (152/204)\n",
      "good/bad: 25.49% (52/204)\n",
      "bad/good: 25.51% (50/196)\n",
      "bad/bad: 74.49% (146/196)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUsAAAEQCAYAAAA5/laqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG2RJREFUeJzt3XucXWV97/HPNwmYkAABgikgMRxBFDkFMXJQ0KKoBQtC\nTy0iFyNiab1w9KAiWlvR6jnUWqtVUeMRG4GqyNFDvFTB9AhCAUMQ5BIEvMQAIRckXKIgyXz7x3pG\nN+PM7DUze8+aPfN9v17rNXut9ey1fvv2m2c9z7PWkm0iImJ405oOICKiFyRZRkTUkGQZEVFDkmVE\nRA1JlhERNSRZRkTUkGTZJZJmSfq6pAclfWUM2zlJ0mWdjK0pkl4g6cdd2O6I32tJ35P0+k7HMmAf\nr5V0VRe3/2+SFrfMf0DSRkn3SVog6RFJ07u1/6lmRtMBNE3SicCZwDOAh4EbgQ/aHuuX/JXAfGAX\n21tGuxHbFwEXjTGWrpNkYB/bdw1Vxvb3gX27sPth32tJ5wB72z65C/tujO2j+h9LWgC8DXiq7fVl\n8ZxGApukpnTNUtKZwEeB/0X1Y1sAfBJ4RQc2/1TgjrEkyslEUjf/Mee9rr6797ckylHr8mfVu2xP\nyQnYEXgE+PNhyjyJKpneW6aPAk8q6w4H7qb6b74eWAucWta9D/gN8HjZx2nAOcCFLdteCBiYUeZf\nC/yUqnb7M+CkluVXtTzv+cAK4MHy9/kt674H/B1wddnOZcC8IV5bf/xntcR/HPBy4A7gl8C7W8of\nDFwDbCplPwFsW9ZdWV7L5vJ6X9Wy/XcC9wEX9C8rz3la2cdBZX53YANw+BDxPrO8vk3ArcArhnqv\nBzzvyAHrb6rzXgGHAP9R9nfTUHGVsnsCXy3x3w98YojP7mPAGuAhYCXwggHv7/Vl3TrgI2X5TODC\nst1N5TOf3/IaXg+8BPg10Fde47/w+9+vHYHPlc/uHuADwPSWOK8G/qns5wNN/z4n4tR4AI298OpH\ntKX/yzREmfcD1wJPBnYtP56/K+sOL89/P7ANVZL5FbBTWX8OT0yOA+d/+2UGZpcfyb5l3W7As8rj\n3/7ggJ2BB4BTyvNeXeZ3Keu/B/wEeDowq8yfO8Rr64//b0v8f1F+7P8KbA88q/wA9yrln0OVQGaU\n2FcBb23ZnqkOdQdu/++p/unMoiVZljJ/AdwGbAd8B/jwELFuA9wFvBvYFngxVYLbd7D3dpDn/976\n4d4rYA+qpPFyqqOvl5b5XQfZ9nSqZPpP5XOcCRw28LMr8ycDu5T38G1U/0RmlnXXAKeUx3OAQ8rj\nvwS+Xt6j6eVz2KHlNby+5f1ufW8X8sRk+TXgMyXGJwM/AP6yJc4twBkltllN/z4n4jSVD8N3ATZ6\n+EO3k4D3215vewNVLeaUlvWPl/WP2/4W1X/10bbJ9QH7S5ple63tWwcp8yfAnbYvsL3F9heB24Fj\nWsp83vYdtn8NXAwcOMw+H6dqn30c+BIwD/iY7YfL/m8DDgCwvdL2tWW/P6f64f1Rjdf0XtuPlXie\nwPZnqZLgdVT/IP56iO0cQpVAzrX9G9v/DnyD6p/FWAz1Xp0MfMv2t2z32b6cqtb38kG2cTBVrfgd\ntjfbftRDtHfbvtD2/eU9/EeqfyL935fHgb0lzbP9iO1rW5bvQvWPaGv5HB4ayYuUNL/E/tYS43qq\n5H5CS7F7bX+8xPZ7n1VM7TbL+4F5bdpndgdWt8yvLst+u40ByfZXjKJR3fZmqkPXvwLWSvqmpGfU\niKc/pj1a5u8bQTz3295aHvf/QNa1rP91//MlPV3SN0pP60NU7bzzhtk2wAbbj7Yp81lgf+Djth8b\noszuwBrbfS3LBr7u0RjqvXoq8OeSNvVPwGFUCX2gPYHVbf7pAiDp7ZJWlV77TVSHxv3v4WlUtdzb\nJa2QdHRZfgFVrftLku6V9CFJ24zwdT6Vqna+tuX1fIaqhtlvzQi3OeVM5WR5DfAYVTvdUO6l+qL1\nW1CWjcZmqkOpfn/QutL2d2y/lOoHeTtVEmkXT39M94wyppH4FFVc+9jegeqQWG2eM+wlrSTNoWoH\n/hxwjqSdhyh6L7CnpNbv60he90gvrbUGuMD23JZptu1zhyi7oF2niKQXULUPH0/VVDOXqt1ZALbv\ntP1qqgT298AlkmaXo5b32d6Pqr36aOA1o3g9j1G1yfa/nh1sP6ulTC4/1saUTZa2H6Rqr/ukpOMk\nbSdpG0lHSfpQKfZF4D2SdpU0r5S/cJS7vBF4YRn/tiPwrv4VkuZLOlbSbKov9SNUh7ADfQt4uqQT\nJc2Q9CpgP6pD0m7bnqpd9ZFS633DgPXrgP8ywm1+DLje9uuBbwKfHqLcdVQ1v7PKZ3Q4VdPDl2ru\nZx2wcECyHc6FwDGS/ljSdEkzJR0u6SmDlP0BVafJuZJml7KHDlJue6p2wQ3ADEl/C+zQv1LSyZJ2\nLbXnTWVxn6QXSfqvZbzkQ1SH5YN9N4Zkey1VB9Y/StpB0jRJT5PUrhklWkzZZAlQ2o3OBN5D9SVe\nA7wZ+H+lyAeo2qp+BNwM3FCWjWZflwNfLttayRMT3LQSx71UPcR/xO8nI2zfT1WzeBtVM8JZwNG2\nN44mphF6O3AiVcfKZ6leS6tzgKXlMO/4dhuTdCxVJ1v/6zwTOEjSSQPL2v4NVXI8CtgInAe8xvbt\nNWPvH6h+v6Qb2hW2vQY4lqr23P+9eAeD/F5KM8YxwN7AL6hGALxqkM1+B/g21UiD1cCjPPHQ90jg\nVkmPUP0TOaG0Hf4BcAlVolwFXEF1aD5Sr6HqHLuNqlPwEgZvVoghyE7tezKRtBD4hu39Gw5lShjL\n+53PqrdM6ZplRERdGanfMEl/QzVUpf9wbyXwXar2u+2oxgK+zvYDkg4cYvlzgPPLJifFeeQ9Zoak\ni4CDqAbMv4aq2eIYqjGc/0E1ptH5rHpXapYNkvRc4M+oxjIeBSwqq74AvNP2H1K1lb63zfLPA2fY\nPmC8Yo8n2Bc4z/YzqdoW30h1Fs9zyyH2LKq2Zshn1bOSLJt1KHBpGcj8MNWZGrOBubavKGWWUvWi\n7zjE8rll+ZVl+Wga/2Ns1ti+ujy+kGpM5oskXSfpZqozjp6Vz6q35TA8YuwG9pKaqsd+ke015apH\nM8c9quio1CybdTXVeL6ZZYD20VSD1x8og5ihOr3yijIudLDlm4BNkg4ry39v6E103QJJzyuPTwT6\nT3fcWD7XVwLks+ptqVk2yPYKScuoxl6uo2qHfBBYDHxa0nZUVyI6tTxlqOWnAueXa0qm02D8/Rh4\nk6TzqcYxfgrYCbiF6pTKFS1l81n1qIyzbJikObYfKQnwSuB0220HTkfE+ErNsnlLJO1H1aa1NIky\nYmJKzTIiooZ08ERE1JBkGRFRQ5JlD5F0etMxxMjkMxsfks6XtF7SLS3LdpZ0uaQ7y9+dWta9S9Jd\nkn4s6Y/r7CPJsrfkh9d78pmNj3+husxdq7OB5bb3AZaXeUqH6glU95k6EjhPNe6vnmQZET2vnEL6\nywGLj6U6LZjy97iW5V8q94b6GdV9oA5ut49JOXRo3vbbecEuc5sOo+P23HkHDlq4+6QcvrBhxu7t\nC/WgnXZdwJ57L5p0n9kv1/+czQ9tbHdbkWE9Z9psP/TbW0AN7y4eu5Xqgsn9lthe0uZp88tV4qE6\nOWB+ebwH1V1b+91Njfs5TcpkuWCXuXz/Pac1HUaMwJInv6/pEGIEPnJm24pYWw95Kx+dMfCWUoM7\nessdj9pe1L7k4Mrl8cb0T2tSJsuI6AECbVOzctr23pmDWidpN9trJe0GrC/L76G6K2e/p1Dj5ndp\ns4yIRmiamD5req1plJZRXU+B8vfSluUnSHqSpL2AfahuPDes1CwjohmCaTPG1Oz5u01JXwQOB+ZJ\nupvqwtjnAhdLOo3qJnHHA9i+VdLFVBc92QK8qdx4blhJlhHRjJEchrdR7rk+mCOGKP9B4IMj2UeS\nZUQ0QlLHapbjIckyIprRwZrleEiyjIhmdLDNcjwkWUZEIySYvm3vDMhJsoyIhghNS80yImJ4Ak1P\nzTIiYlgCpk1PzTIiYngih+EREe1ISgdPREQdmpZkGRExvByGR0TUoXTwRES0o9QsIyLqSZtlREQ7\nEtO3SbKMiBhWDsMjImrKYXhERDupWUZE1JGrDkVEtCXBtBmjvnPjuEuyjIjGZFB6REQ7ymF4REQt\n6Q2PiGgj4ywjImpKsoyIaEdKb3hERHtKm2VERC3KYXhExLDSwRMRUVMOwyMi2smg9IiIetIbHhHR\nRtosIyJqEaTNMiKiPfXQ0KHeSesRMbmo6g2vM9XanPQ/Jd0q6RZJX5Q0U9LOki6XdGf5u9Now02y\njIhmSGjG9FpT+01pD+B/AIts7w9MB04AzgaW294HWF7mRyXJMiIao2mqNdU0A5glaQawHXAvcCyw\ntKxfChw32liTLCOiEUJI02pNwDxJ17dMp7duy/Y9wIeBXwBrgQdtXwbMt722FLsPmD/aeNPBExHN\nEFC/1rjR9qIhN1W1RR4L7AVsAr4i6eTWMrYtyaOMNskyIprTwdMdXwL8zPYGAElfBZ4PrJO0m+21\nknYD1o92BzkMj4jGdLDN8hfAIZK2UzUe6QhgFbAMWFzKLAYuHW2sqVlGRDMkNL0zpzvavk7SJcAN\nwBbgh8ASYA5wsaTTgNXA8aPdR5JlRDSng2fw2H4v8N4Bix+jqmWOWZJlRDRCUs7g6RZJCyXd0nQc\nEdEh06bVmyaA1CwjojG56lAh6W+Ak4ENwBpgJfBd4NNUI+x/ArzO9gOSDhxi+XOA88smL+tmvBEx\njqprtDUdRW1di1TSc4E/Aw4AjgL6B5R+AXin7T8EbuZ3DbJDLf88cIbtA7oVa0Q0Q9On15omgm6m\n9UOBS20/avth4OvAbGCu7StKmaXACyXtOMTyuWX5lWX5BUPtTNLp/adCbXx4c1deUER0UP8ZPHWm\nCaB36sBt2F5ie5HtRfO2n910OBHRljp6ibZu62YUVwPHlGvKzQGOBjYDD0h6QSlzCnCF7QeHWL4J\n2CTpsLL8pC7GGxHjTao3TQBd6+CxvULSMuBHwDqqdsgHqU45+rSk7YCfAqeWpwy1/FTg/HICfDp4\nIiYLMWGGBdXR7aFDH7Z9TkmAVwIrbd8IHDKw4DDLV1J1EvU7q1vBRsR46tzpjuOh28lyiaT9gJnA\nUts3dHl/EdErRE8NHepqsrR9Yje3HxG9bOL0dNeRM3giohGC/qug94Qky4hoxsiulN64JMuIaEhv\nne6YZBkRzUlveEREGz12IY0ky4hoTtosIyJqSM0yIqKGCXLedx1JlhHRDCnnhkdE1DItveEREcNL\nzTIioqa0WUZE1JDe8IiIdibOVdDrSLKMiEZY4JzuGBHRTk53jIioJ8kyIqI9p80yIqKNXHUoIqKm\n1CwjItpResMjItrqsVvh9k6kETHpWNNqTXVImivpEkm3S1ol6XmSdpZ0uaQ7y9+dRhtrkmVENKSc\nwVNnqudjwLdtPwM4AFgFnA0st70PsLzMj0qSZUQ0plM1S0k7Ai8EPgdg+ze2NwHHAktLsaXAcaON\nNW2WEdEMaSTXs5wn6fqW+SW2l7TM7wVsAD4v6QBgJfAWYL7ttaXMfcD80YabZBkRjTAjGpS+0fai\nYdbPAA4CzrB9naSPMeCQ27YleVTBksPwiGiSptWb2rsbuNv2dWX+EqrkuU7SbgDl7/rRhppkGRGN\nMao1td2OfR+wRtK+ZdERwG3AMmBxWbYYuHS0seYwPCIaotrDgmo6A7hI0rbAT4FTqSqEF0s6DVgN\nHD/ajSdZRkRzOpgsbd8IDNaueUQntp9kGRGNsERf7u4YEVFDLqQREdFeh9ssuyrJMiIaUq+ne6JI\nsoyIxqRmGRHRjpicbZaSnmT7sW4GExFThxF96p3e8LZ1YEkHS7oZuLPMHyDp412PLCImvU5ez7Lb\n6kTxz8DRwP0Atm8CXtTNoCJiaujU6Y7joc5h+DTbq/XEtoWtXYonIqaMjp/u2FV1kuUaSQcDljSd\n6vzLO7obVkRMBZPtvuFvoDoUXwCsA75blkVEjJrVWx08bZOl7fXACeMQS0RMMROlPbKOtslS0mep\nLmr8BLZP70pEETFlTLY2y++2PJ4J/CmwpjvhRMRUMqlqlra/3Dov6QLgqq5FFBFTgidhb/hAezGG\nO6SNhw0zdmfJk9/XdBgxAqevf2/TIcQIXLDl3o5sZ1LVLCU9wO/aLKcBv2QMNyqPiOjX10O3ARs2\nWaoaiX4AcE9Z1Gd71LeSjIj4HeEeSpbDRloS47dsby1TEmVEdITprdMd66T1GyU9u+uRRMSU00vJ\ncsjDcEkzbG8Bng2skPQTYDPVVehs+6BxijEiJqmJkgjrGK7N8gfAQcArximWiJhSJk6tsY7hkqUA\nbP9knGKJiCnEQJ97p4NnuGS5q6Qzh1pp+yNdiCcippDJUrOcDsyBHno1EdFTJkuyXGv7/eMWSURM\nMcKeHMmyd15FRPQcA309lGaGS5ZHjFsUETH1eJJ08Nj+5XgGEhFTz2Rps4yI6KLJ02YZEdE1/eeG\n94oky4hoTGqWERE19DUdwAgkWUZEI4x6qje8dyKNiEnHVq2pDknTJf1Q0jfK/M6SLpd0Z/m701hi\nTbKMiMZ0+HqWbwFWtcyfDSy3vQ+wnDHeDifJMiKaYeirObUj6SnAnwD/p2XxscDS8ngpcNxYwk2b\nZUQ0YoRDh+ZJur5lfontJS3zHwXOArZvWTbf9try+D7GeFfaJMuIaMwIhg5ttL1osBWSjgbW214p\n6fDB92NLGtM9xJIsI6IhYmtnxlkeCrxC0suBmcAOki4E1knazfZaSbsB68eyk7RZRkQjTGd6w22/\ny/ZTbC8ETgD+3fbJwDJgcSm2GLh0LPGmZhkRjenyzbXPBS6WdBqwGjh+LBtLsoyIxnT63HDb3wO+\nVx7fTwcvNZlkGRHNqDksaKJIsoyIRhjo68uFNCIi2post5WIiOiqLnfwdFSSZUQ0wrlSekREDeng\niYioJ4fhERFtGDp1uuO4SLKMiMakZhkRUUOSZUREGzb05TA8IqK91CwjImrY2kP3wk2yjIhG9F/P\nslckWUZEM5zD8IiIWnIGT0REG9VheNNR1JdkGRGNSbKMiGjH6Q2PiGirulJ601HUl2QZEY3JYXhE\nRA1JlhERbbjHLv47bbx3KGmhpFvG+7kRMfHYrjVNBKlZRkRjtm5tOoL6xr1mWcyQdJGkVZIukbSd\npL+VtELSLZKWSBKApOdIuknSTcCbGoo3IjrMrj9NBE0ly32B82w/E3gIeCPwCdvPtb0/MAs4upT9\nPHCG7QOG26Ck0yVdL+n6zQ9t6GbsEdEhfa43TQRNJcs1tq8ujy8EDgNeJOk6STcDLwaeJWkuMNf2\nlaXsBUNt0PYS24tsL5q9w65dDT4iOqOXapZNtVkOfPkGzgMW2V4j6Rxg5rhHFRHjyhOl2lhDUzXL\nBZKeVx6fCFxVHm+UNAd4JYDtTcAmSYeV9SeNb5gR0S0upzvWmSaCpmqWPwbeJOl84DbgU8BOwC3A\nfcCKlrKnAudLMnDZeAcaEd3T10M1y3FPlrZ/DjxjkFXvKdPA8iuB1s6ds7oTWUSMp1yiLSKijgnU\neVNHkmVENMT09VC2bKqDJyIC99Wb2pG0p6T/L+k2SbdKektZvrOkyyXdWf7uNNpYkywjohE2bN3q\nWlMNW4C32d4POISqA3k/4Gxgue19gOVlflSSLCOiMZ26kIbttbZvKI8fBlYBewDHAktLsaXAcaON\nNW2WEdEIM6JTGedJur5lfontJYMVlLQQeDZwHTDf9tqy6j5g/mhihSTLiGiKR3QGz0bbi9oVKie1\n/F/grbYfKtfjqXZnu4zXHpUchkdEYzp5brikbagS5UW2v1oWr5O0W1m/G7B+tLEmWUZEY/r6XGtq\np1zS8XPAKtsfaVm1DFhcHi8GLh1trDkMj4hG2KavXk93HYcCpwA3S7qxLHs3cC5wsaTTgNXA8aPd\nQZJlRDSmU4PSbV8FaIjVR3RiH0mWEdGYiXJ/nTqSLCOiEXauOhQRUUsPVSyTLCOiGbbZOlGu7FtD\nkmVENKaXbiuRZBkRjUmyjIhoZwLd5raOJMuIaIRJzTIiooZ6l1+bKJIsI6IZJr3hERHt5DA8IqKO\nkV3PsnFJlhHRkN66u2OSZUQ0JjXLiIg2TK46FBHRnmHrlvSGR0S0kXGWERFt2eC+1CwjItrKxX8j\nImrIYXhERBu26UsHT0REe31OsoyIGF5Od4yIaM84yTIioo508EREtGPoyzjLiIjhGdO3dWvTYdSW\nZBkRzUgHT0REPUmWERFtOeMsIyLacQ7DIyLqyVWHIiLacXrDIyLaMrlEW0REez128d9pTQcQEVNV\ndW54nakOSUdK+rGkuySd3eloU7OMiMa4Q0OHJE0HPgm8FLgbWCFpme3bOrIDkiwjoiHVxX871sFz\nMHCX7Z8CSPoScCzQsWSpXrrqR12SNgCrm46jC+YBG5sOIkZksn5mT7W961g2IOnbVO9PHTOBR1vm\nl9he0rKtVwJH2n59mT8F+G+23zyWGFtNyprlWD/EiUrS9bYXNR1H1JfPbGi2j2w6hpFIB09ETAb3\nAHu2zD+lLOuYJMuImAxWAPtI2kvStsAJwLJO7mBSHoZPYkvaF4kJJp/ZOLC9RdKbge8A04Hzbd/a\nyX1Myg6e6A5JW4Gbqf7JrgIW2/7VKLd1OPB220dLegWwn+1zhyg7FzjR9nkj3Mc5wCO2PzyaGCNa\n5TA8RuLXtg+0vT/wG+CvWleqMuLvlO1lQyXKYi7wxpFuN6KTkixjtL4P7C1pYTlr4gvALcCekl4m\n6RpJN0j6iqQ58NszLG6XdAPw3/s3JOm1kj5RHs+X9DVJN5Xp+cC5wNMk3SjpH0q5d0haIelHkt7X\nsq2/lnSHpKuAfcft3YhJL22WMWKSZgBHAd8ui/ahOiS/VtI84D3AS2xvlvRO4ExJHwI+C7wYuAv4\n8hCb/2fgCtt/Ws7KmAOcDexv+8Cy/5eVfR4MCFgm6YXAZqqG/QOpvts3ACs7++pjqkqyjJGYJenG\n8vj7wOeA3YHVtq8tyw8B9gOulgSwLXAN8AzgZ7bvBJB0IXD6IPt4MfAaANtbgQcl7TSgzMvK9MMy\nP4cqeW4PfK2/HVVSR3tDY2pLsoyR+HV/7a5fSYibWxcBl9t+9YByT3jeGAn437Y/M2Afb+3gPiKe\nIG2W0WnXAodK2htA0mxJTwduBxZKelop9+ohnr8ceEN57nRJOwIPU9Ua+30HeF1LW+gekp4MXAkc\nJ2mWpO2BYzr82mIKS7KMjrK9AXgt8EVJP6Icgtt+lOqw+5ulg2f9EJt4C/AiSTdTtTfuZ/t+qsP6\nWyT9g+3LgH8FrinlLgG2t30DVVvoTcC/UQ1UjuiIjLOMiKghNcuIiBqSLCMiakiyjIioIckyIqKG\nJMuIiBqSLCMiakiyjIio4T8BjIqwaiQwzN8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10deb7a90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "labels = ['good', 'bad']\n",
    "cm = confusion_matrix(y_test, y_pred, labels)\n",
    "plot_cm(cm, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix Stats\n",
      "good/good: 91.18% (186/204)\n",
      "good/bad: 8.82% (18/204)\n",
      "bad/good: 2.04% (4/196)\n",
      "bad/bad: 97.96% (192/196)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUsAAAEQCAYAAAA5/laqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG2BJREFUeJzt3XucXWV97/HPNwmQhAABg2m4GY4iCrQgRA4KWhS1YLn1\n1CJyMSKW1gvVg4p4acXbOdRaq1VR4xEagYrI0UO8VMH0CEIBQxDkEgTUxgAhIUi4yS2Zb/9Yz+hm\nzMxeM7P3rNkz3/frtV6z11rPXuu3b7951vM8ay3ZJiIihjal6QAiInpBkmVERA1JlhERNSRZRkTU\nkGQZEVFDkmVERA1Jll0iaYakb0l6UNLXR7Gd4yVd2snYmiLpJZJ+1oXtDvu9lvRDSW/qdCwD9vEG\nSVd2cfv/Jmlhy/xHJa2TdK+kXSQ9Imlqt/Y/2UxrOoCmSToOOA14HvAwcAPwMduj/ZK/BpgLPMP2\nhpFuxPYFwAWjjKXrJBnYzfadg5Wx/SNg9y7sfsj3WtKZwHNsn9CFfTfG9mH9jyXtArwTeJbttWXx\nrEYCm6Amdc1S0mnAp4D/RfVj2wX4HHBkBzb/LOD20STKiURSN/8x572uvrv3tyTKEevyZ9W7bE/K\nCdgGeAT4iyHKbEGVTO8p06eALcq6g4G7qP6brwVWAyeVdR8CngSeKvs4GTgTOL9l2/MBA9PK/BuA\nX1DVbn8JHN+y/MqW570YWAY8WP6+uGXdD4GPAFeV7VwKzBnktfXHf3pL/EcDrwZuB34NvK+l/P7A\n1cD6UvazwOZl3RXltTxaXu9rW7b/HuBe4Lz+ZeU5zy772LfM7wDcBxw8SLzPL69vPXALcORg7/WA\n5x06YP2Ndd4r4ADgP8r+bhwsrlJ2Z+AbJf77gc8O8tl9GlgFPAQsB14y4P29rqxbA3yyLJ8OnF+2\nu7585nNbXsObgFcAjwF95TX+C7///doG+HL57O4GPgpMbYnzKuCfyn4+2vTvczxOjQfQ2AuvfkQb\n+r9Mg5T5MHAN8Exg+/Lj+UhZd3B5/oeBzaiSzG+Abcv6M3l6chw4/9svM7Bl+ZHsXtbNA/Ysj3/7\ngwO2Ax4ATizPe12Zf0ZZ/0Pg58BzgRll/qxBXlt//H9X4v/L8mP/V2ArYM/yA9y1lN+PKoFMK7Gv\nAN7Rsj1THeoO3P7fU/3TmUFLsixl/hK4FZgJfB/4xCCxbgbcCbwP2Bx4OVWC231T7+0mnv9764d6\nr4AdqZLGq6mOvl5Z5rffxLanUiXTfyqf43TgoIGfXZk/AXhGeQ/fSfVPZHpZdzVwYnk8CzigPP4r\n4FvlPZpaPoetW17Dm1re79b3dj5PT5bfBL5YYnwm8GPgr1ri3ACcWmKb0fTvczxOk/kw/BnAOg99\n6HY88GHba23fR1WLObFl/VNl/VO2v0v1X32kbXJ9wF6SZthebfuWTZT5U+AO2+fZ3mD7q8BtwBEt\nZc61fbvtx4CLgH2G2OdTVO2zTwEXAnOAT9t+uOz/VmBvANvLbV9T9vufVD+8P67xmj5o+4kSz9PY\n/hJVEryW6h/E+wfZzgFUCeQs20/a/nfg21T/LEZjsPfqBOC7tr9ru8/2ZVS1vldvYhv7U9WK3237\nUduPe5D2btvn276/vIf/SPVPpP/78hTwHElzbD9i+5qW5c+g+ke0sXwODw3nRUqaW2J/R4lxLVVy\nP7al2D22P1Ni+73PKiZ3m+X9wJw27TM7ACtb5leWZb/dxoBk+xtG0Khu+1GqQ9e/BlZL+o6k59WI\npz+mHVvm7x1GPPfb3lge9/9A1rSsf6z/+ZKeK+nbpaf1Iap23jlDbBvgPtuPtynzJWAv4DO2nxik\nzA7AKtt9LcsGvu6RGOy9ehbwF5LW90/AQVQJfaCdgZVt/ukCIOldklaUXvv1VIfG/e/hyVS13Nsk\nLZN0eFl+HlWt+0JJ90j6uKTNhvk6n0VVO1/d8nq+SFXD7LdqmNucdCZzsrwaeIKqnW4w91B90frt\nUpaNxKNUh1L9/qB1pe3v234l1Q/yNqok0i6e/pjuHmFMw/F5qrh2s7011SGx2jxnyEtaSZpF1Q78\nZeBMSdsNUvQeYGdJrd/X4bzu4V5aaxVwnu3ZLdOWts8apOwu7TpFJL2Eqn34GKqmmtlU7c4CsH2H\n7ddRJbC/By6WtGU5avmQ7T2o2qsPB14/gtfzBFWbbP/r2dr2ni1lcvmxNiZtsrT9IFV73eckHS1p\npqTNJB0m6eOl2FeBD0jaXtKcUv78Ee7yBuClZfzbNsB7+1dImivpKElbUn2pH6E6hB3ou8BzJR0n\naZqk1wJ7UB2SdttWVO2qj5Ra75sHrF8D/LdhbvPTwHW23wR8B/jCIOWupar5nV4+o4Opmh4urLmf\nNcD8Acl2KOcDR0j6E0lTJU2XdLCknTZR9sdUnSZnSdqylD1wE+W2omoXvA+YJunvgK37V0o6QdL2\npfa8vizuk/QySX9Yxks+RHVYvqnvxqBsr6bqwPpHSVtLmiLp2ZLaNaNEi0mbLAFKu9FpwAeovsSr\ngLcB/68U+ShVW9VPgZuA68uykezrMuBrZVvLeXqCm1LiuIeqh/iP+f1khO37qWoW76RqRjgdONz2\nupHENEzvAo6j6lj5EtVraXUmsLgc5h3TbmOSjqLqZOt/nacB+0o6fmBZ209SJcfDgHXA2cDrbd9W\nM/b+ger3S7q+XWHbq4CjqGrP/d+Ld7OJ30tpxjgCeA7wK6oRAK/dxGa/D3yPaqTBSuBxnn7oeyhw\ni6RHqP6JHFvaDv8AuJgqUa4ALqc6NB+u11N1jt1K1Sl4MZtuVohByE7teyKRNB/4tu29Gg5lUhjN\n+53PqrdM6pplRERdGanfMEl/SzVUpf9wbznwA6r2u5lUYwHfaPsBSfsMsnw/4JyyyQlxHnmPmSbp\nAmBfqgHzr6dqtjiCagznf1CNaXQ+q96VmmWDJL0Q+HOqsYyHAQvKqq8A77H9R1RtpR9ss/xc4FTb\ne49V7PE0uwNn234+VdviW6jO4nlhOcSeQdXWDPmselaSZbMOBC4pA5kfpjpTY0tgtu3LS5nFVL3o\n2wyyfHZZfkVZPpLG/xidVbavKo/PpxqT+TJJ10q6ieqMoz3zWfW2HIZHjN7AXlJT9dgvsL2qXPVo\n+phHFR2VmmWzrqIazze9DNA+nGrw+gNlEDNUp1deXsaFbmr5emC9pIPK8t8behNdt4ukF5XHxwH9\npzuuK5/rawDyWfW21CwbZHuZpCVUYy/XULVDPggsBL4gaSbVlYhOKk8ZbPlJwDnlmpLpNBh7PwPe\nKukcqnGMnwe2BW6mOqVyWUvZfFY9KuMsGyZplu1HSgK8AjjFdtuB0xExtlKzbN4iSXtQtWktTqKM\nGJ9Ss4yIqCEdPBERNSRZRkTUkGTZQySd0nQMMTz5zMaGpHMkrZV0c8uy7SRdJumO8nfblnXvlXSn\npJ9J+pM6+0iy7C354fWefGZj41+oLnPX6gxgqe3dgKVlntKheizVfaYOBc5WjfurJ1lGRM8rp5D+\nesDio6hOC6b8Pbpl+YXl3lC/pLoP1P7t9jEhhw5tu/lm3mHmxDu7bN6MLdhz9lYTcvjC6i3mNx1C\nV8zcage2m/uHE+4ze/Shu3nisV+3u63IkPabsqUf+u0toIZ2J0/cQnXB5H6LbC9q87S55SrxUJ0c\nMLc83pHqrq397qLG/ZwmZLLcYeZ0Ljxov6bDiGH4yHPPbTqEGIYfXHDkqLfxkDfyqWkDbym1aYdv\nuP1x2wval9y0cnm8Uf3TmpDJMiJ6gECb1ayctr135iatkTTP9mpJ84C1ZfndVHfl7LcTNW5+lzbL\niGiEpoipM6bWmkZoCdX1FCh/L2lZfqykLSTtCuxGdeO5IaVmGRHNEEyZNqpmz99tSvoqcDAwR9Jd\nVBfGPgu4SNLJVDeJOwbA9i2SLqK66MkG4K3lxnNDSrKMiGYM5zC8jXLP9U05ZJDyHwM+Npx9JFlG\nRCMkdaxmORaSLCOiGR2sWY6FJMuIaEYH2yzHQpJlRDRCgqmb986AnCTLiGiI0JTULCMihibQ1NQs\nIyKGJGDK1NQsIyKGJnIYHhHRjqR08ERE1KEpSZYREUPLYXhERB1KB09ERDtKzTIiop60WUZEtCMx\ndbMky4iIIeUwPCKiphyGR0S0k5plREQduepQRERbEkyZNuI7N465JMuIaEwGpUdEtKMchkdE1JLe\n8IiINjLOMiKipiTLiIh2pPSGR0S0p7RZRkTUohyGR0QMKR08ERE15TA8IqKdDEqPiKgnveEREW2k\nzTIiohZB2iwjItpTDw0d6p20HhETi6re8DpTrc1J/1PSLZJulvRVSdMlbSfpMkl3lL/bjjTcJMuI\naIaEpk2tNbXflHYE/gZYYHsvYCpwLHAGsNT2bsDSMj8iSZYR0RhNUa2ppmnADEnTgJnAPcBRwOKy\nfjFw9EhjTbKMiEYIIU2pNQFzJF3XMp3Sui3bdwOfAH4FrAYetH0pMNf26lLsXmDuSONNB09ENENA\n/VrjOtsLBt1U1RZ5FLArsB74uqQTWsvYtiSPMNoky4hoTgdPd3wF8Evb9wFI+gbwYmCNpHm2V0ua\nB6wd6Q5yGB4Rjelgm+WvgAMkzVQ1HukQYAWwBFhYyiwELhlprKlZRkQzJDS1M6c72r5W0sXA9cAG\n4CfAImAWcJGkk4GVwDEj3UeSZUQ0p4Nn8Nj+IPDBAYufoKpljlqSZUQ0QlLO4OkWSfMl3dx0HBHR\nIVOm1JvGgdQsI6IxuepQIelvgROA+4BVwHLgB8AXqEbY/xx4o+0HJO0zyPL9gHPKJi/tZrwRMYaq\na7Q1HUVtXYtU0guBPwf2Bg4D+geUfgV4j+0/Am7idw2ygy0/FzjV9t7dijUimqGpU2tN40E30/qB\nwCW2H7f9MPAtYEtgtu3LS5nFwEslbTPI8tll+RVl+XmD7UzSKf2nQj3w5FNdeUER0UH9Z/DUmcaB\nCdNmaXsR1bgq9py91YhPaYqIsdJb9w3vZqRXAUeUa8rNAg4HHgUekPSSUuZE4HLbDw6yfD2wXtJB\nZfnxXYw3IsaaVG8aB7pWs7S9TNIS4KfAGqp2yAepTjn6gqSZwC+Ak8pTBlt+EnBOOQE+HTwRE4UY\nN8OC6uj2YfgnbJ9ZEuAVwHLbNwAHDCw4xPLlVJ1E/U7vVrARMZY6d7rjWOh2slwkaQ9gOrDY9vVd\n3l9E9ArRU0OHuposbR/Xze1HRC8bPz3ddUyY3vCI6C2C/qug94Qky4hoxvCulN64JMuIaEhvne6Y\nZBkRzUlveEREGz12IY0ky4hoTtosIyJqSM0yIqKGcXLedx1JlhHRDCnnhkdE1DIlveEREUNLzTIi\noqa0WUZE1JDe8IiIdsbPVdDrSLKMiEZY4JzuGBHRTk53jIioJ8kyIqI9p80yIqKNXHUoIqKm1Cwj\nItpResMjItrqsVvh9k6kETHhWFNqTXVImi3pYkm3SVoh6UWStpN0maQ7yt9tRxprkmVENKScwVNn\nqufTwPdsPw/YG1gBnAEstb0bsLTMj0iSZUQ0plM1S0nbAC8Fvgxg+0nb64GjgMWl2GLg6JHGmjbL\niGiGNJzrWc6RdF3L/CLbi1rmdwXuA86VtDewHHg7MNf26lLmXmDuSMNNsoyIRphhDUpfZ3vBEOun\nAfsCp9q+VtKnGXDIbduSPKJgyWF4RDRJU+pN7d0F3GX72jJ/MVXyXCNpHkD5u3akoSZZRkRjjGpN\nbbdj3wuskrR7WXQIcCuwBFhYli0ELhlprDkMj4iGqPawoJpOBS6QtDnwC+AkqgrhRZJOBlYCx4x0\n40mWEdGcDiZL2zcAm2rXPKQT20+yjIhGWKIvd3eMiKghF9KIiGivw22WXZVkGRENqdfTPV4kWUZE\nY1KzjIhoR0zMNktJW9h+opvBRMTkYUSfeqc3vG0dWNL+km4C7ijze0v6TNcji4gJr5PXs+y2OlH8\nM3A4cD+A7RuBl3UzqIiYHDp1uuNYqHMYPsX2Sj29bWFjl+KJiEmj46c7dlWdZLlK0v6AJU2lOv/y\n9u6GFRGTwUS7b/ibqQ7FdwHWAD8oyyIiRszqrQ6etsnS9lrg2DGIJSImmfHSHllH22Qp6UtUFzV+\nGtundCWiiJg0Jlqb5Q9aHk8H/gxY1Z1wImIymVA1S9tfa52XdB5wZdciiohJwROwN3ygXRnFHdLG\nwq+m7MTfzDir6TBiGN7/tSOaDiGG4YYHft6R7UyomqWkB/hdm+UU4NeM4kblERH9+nroNmBDJktV\nI9H3Bu4ui/psj/hWkhERvyPcQ8lyyEhLYvyu7Y1lSqKMiI4wvXW6Y520foOkF3Q9koiYdHopWQ56\nGC5pmu0NwAuAZZJ+DjxKdRU62953jGKMiAlqvCTCOoZqs/wxsC9w5BjFEhGTyvipNdYxVLIUgO3O\njBGIiGhhoM+908EzVLLcXtJpg620/ckuxBMRk8hEqVlOBWZBD72aiOgpEyVZrrb94TGLJCImGWFP\njGTZO68iInqOgb4eSjNDJctDxiyKiJh8PEE6eGz/eiwDiYjJZ6K0WUZEdNHEabOMiOia/nPDe0WS\nZUQ0JjXLiIga+poOYBiSLCOiEUY91RveO5FGxIRjq9ZUh6Spkn4i6dtlfjtJl0m6o/zddjSxJllG\nRGM6fD3LtwMrWubPAJba3g1Yyihvh5NkGRHNMPTVnNqRtBPwp8D/aVl8FLC4PF4MHD2acNNmGRGN\nGObQoTmSrmuZX2R7Ucv8p4DTga1als21vbo8vpdR3pU2yTIiGjOMoUPrbC/Y1ApJhwNrbS+XdPCm\n92NLGtU9xJIsI6IhYmNnxlkeCBwp6dXAdGBrSecDayTNs71a0jxg7Wh2kjbLiGiE6UxvuO332t7J\n9nzgWODfbZ8ALAEWlmILgUtGE29qlhHRmC7fXPss4CJJJwMrgWNGs7Eky4hoTKfPDbf9Q+CH5fH9\ndPBSk0mWEdGMmsOCxosky4hohIG+vlxIIyKirYlyW4mIiK7qcgdPRyVZRkQjnCulR0TUkA6eiIh6\nchgeEdGGoVOnO46JJMuIaExqlhERNSRZRkS0YUNfDsMjItpLzTIiooaNPXQv3CTLiGhE//Use0WS\nZUQ0wzkMj4ioJWfwRES0UR2GNx1FfUmWEdGYJMuIiHac3vCIiLaqK6U3HUV9SZYR0ZgchkdE1JBk\nGRHRhnvs4r9TxnqHkuZLunmsnxsR44/tWtN4kJplRDRm48amI6hvzGuWxTRJF0haIeliSTMl/Z2k\nZZJulrRIkgAk7SfpRkk3Am9tKN6I6DC7/jQeNJUsdwfOtv184CHgLcBnbb/Q9l7ADODwUvZc4FTb\new+1QUmnSLpO0nVPPbG+m7FHRIf0ud40HjSVLFfZvqo8Ph84CHiZpGsl3QS8HNhT0mxgtu0rStnz\nBtug7UW2F9hesNkWs7safER0Ri/VLJtqsxz48g2cDSywvUrSmcD0MY8qIsaUx0u1sYamapa7SHpR\neXwccGV5vE7SLOA1ALbXA+slHVTWHz+2YUZEt7ic7lhnGg+aqln+DHirpHOAW4HPA9sCNwP3Asta\nyp4EnCPJwKVjHWhEdE9fD9UsxzxZ2v5P4HmbWPWBMg0svxxo7dw5vTuRRcRYyiXaIiLqGEedN3Uk\nWUZEQ0xfD2XLpjp4IiJwX72pHUk7S/r/km6VdIukt5fl20m6TNId5e+2I401yTIiGmHDxo2uNdWw\nAXin7T2AA6g6kPcAzgCW2t4NWFrmRyTJMiIa06kLadhebfv68vhhYAWwI3AUsLgUWwwcPdJY02YZ\nEY0wwzqVcY6k61rmF9letKmCkuYDLwCuBebaXl1W3QvMHUmskGQZEU3xsM7gWWd7QbtC5aSW/wu8\nw/ZD5Xo81e5sl/HaI5LD8IhoTCfPDZe0GVWivMD2N8riNZLmlfXzgLUjjTXJMiIa09fnWlM75ZKO\nXwZW2P5ky6olwMLyeCFwyUhjzWF4RDTCNn31errrOBA4EbhJ0g1l2fuAs4CLJJ0MrASOGekOkiwj\nojGdGpRu+0pAg6w+pBP7SLKMiMaMl/vr1JFkGRGNsHPVoYiIWnqoYplkGRHNsM3G8XJl3xqSLCOi\nMb10W4kky4hoTJJlREQ74+g2t3UkWUZEI0xqlhERNdS7/Np4kWQZEc0w6Q2PiGgnh+EREXUM73qW\njUuyjIiG9NbdHZMsI6IxqVlGRLRhctWhiIj2DBs3pDc8IqKNjLOMiGjLBvelZhkR0VYu/hsRUUMO\nwyMi2rBNXzp4IiLa63OSZUTE0HK6Y0REe8ZJlhERdaSDJyKiHUNfxllGRAzNmL6NG5sOo7Yky4ho\nRjp4IiLqSbKMiGjLGWcZEdGOcxgeEVFPrjoUEdGO0xseEdGWySXaIiLa67GL/05pOoCImKyqc8Pr\nTHVIOlTSzyTdKemMTkebmmVENMYdGjokaSrwOeCVwF3AMklLbN/akR2QZBkRDaku/tuxDp79gTtt\n/wJA0oXAUUDHkqV66aofdUm6D1jZdBxdMAdY13QQMSwT9TN7lu3tR7MBSd+jen/qmA483jK/yPai\nlm29BjjU9pvK/InAf7f9ttHE2GpC1ixH+yGOV5Kus72g6Tiivnxmg7N9aNMxDEc6eCJiIrgb2Lll\nfqeyrGOSLCNiIlgG7CZpV0mbA8cCSzq5gwl5GD6BLWpfJMaZfGZjwPYGSW8Dvg9MBc6xfUsn9zEh\nO3iiOyRtBG6i+ie7Alho+zcj3NbBwLtsHy7pSGAP22cNUnY2cJzts4e5jzOBR2x/YiQxRrTKYXgM\nx2O297G9F/Ak8NetK1UZ9nfK9pLBEmUxG3jLcLcb0UlJljFSPwKeI2l+OWviK8DNwM6SXiXpaknX\nS/q6pFnw2zMsbpN0PfA/+jck6Q2SPlsez5X0TUk3lunFwFnAsyXdIOkfSrl3S1om6aeSPtSyrfdL\nul3SlcDuY/ZuxISXNssYNknTgMOA75VFu1Edkl8jaQ7wAeAVth+V9B7gNEkfB74EvBy4E/jaIJv/\nZ+By239WzsqYBZwB7GV7n7L/V5V97g8IWCLppcCjVA37+1B9t68Hlnf21cdklWQZwzFD0g3l8Y+A\nLwM7ACttX1OWHwDsAVwlCWBz4GrgecAvbd8BIOl84JRN7OPlwOsBbG8EHpS07YAyryrTT8r8LKrk\nuRXwzf52VEkd7Q2NyS3JMobjsf7aXb+SEB9tXQRcZvt1A8o97XmjJOB/2/7igH28o4P7iHiatFlG\np10DHCjpOQCStpT0XOA2YL6kZ5dyrxvk+UuBN5fnTpW0DfAwVa2x3/eBN7a0he4o6ZnAFcDRkmZI\n2go4osOvLSaxJMvoKNv3AW8Avirpp5RDcNuPUx12f6d08KwdZBNvB14m6Saq9sY9bN9PdVh/s6R/\nsH0p8K/A1aXcxcBWtq+nagu9Efg3qoHKER2RcZYRETWkZhkRUUOSZUREDUmWERE1JFlGRNSQZBkR\nUUOSZUREDUmWERE1/Beicqqmc23t1AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10fb80ad0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "no_label = list(df.columns.values)\n",
    "no_label.remove('label')\n",
    "X = df.as_matrix(no_label)\n",
    "\n",
    "X_train_all, X_test_all, y_train_all, y_test_all = train_test_split(X, y, test_size=my_tsize, random_state=my_seed)\n",
    "\n",
    "clf_all = sklearn.ensemble.RandomForestClassifier(n_estimators=35)\n",
    "\n",
    "clf_all.fit(X_train_all, y_train_all) \n",
    "y_pred = clf_all.predict(X_test_all)  \n",
    "cm     = confusion_matrix(y_test_all, y_pred, labels)       \n",
    "plot_cm(cm, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred      = clf_all.predict_proba(X_test_all)\n",
    "\n",
    "y_pred      = pd.DataFrame(np.array(y_pred))\n",
    "y_test_all  = pd.DataFrame(np.array(y_test_all))\n",
    "\n",
    "all = pd.concat([y_pred, y_test_all], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.8 0.2 'good']\n",
      "[0.5428571428571428 0.45714285714285713 'good']\n",
      "[0.9428571428571428 0.05714285714285714 'good']\n",
      "[0.9142857142857143 0.08571428571428572 'good']\n",
      "[0.4 0.6 'bad']\n",
      "[0.7142857142857143 0.2857142857142857 'good']\n",
      "[0.8285714285714286 0.17142857142857143 'good']\n",
      "[0.6 0.4 'good']\n",
      "[0.9142857142857143 0.08571428571428572 'good']\n",
      "[0.7428571428571429 0.2571428571428571 'good']\n",
      "[0.42857142857142855 0.5714285714285714 'bad']\n",
      "[0.5714285714285714 0.42857142857142855 'good']\n",
      "[0.6285714285714286 0.37142857142857144 'good']\n",
      "[0.5714285714285714 0.42857142857142855 'good']\n",
      "[0.4857142857142857 0.5142857142857142 'bad']\n",
      "[0.7428571428571429 0.2571428571428571 'good']\n",
      "[0.5142857142857142 0.4857142857142857 'good']\n",
      "[0.6 0.4 'good']\n",
      "[0.8 0.2 'good']\n",
      "[0.7428571428571429 0.2571428571428571 'good']\n",
      "[0.7714285714285715 0.22857142857142856 'good']\n",
      "[0.2857142857142857 0.7142857142857143 'bad']\n",
      "count: 21\n"
     ]
    }
   ],
   "source": [
    "count = 0\n",
    "for i in range(len(all)):\n",
    "    if abs(all.values[i][0]-all.values[i][1]) < 0.25:\n",
    "        #print all.values[i]\n",
    "        count+=1\n",
    "    if all.values[i][0] > 0.5 and all.values[i][2] == 'good':\n",
    "        print all.values[i]\n",
    "        #print X_test_all[i]\n",
    "    if all.values[i][1] > 0.5 and all.values[i][2] == 'bad':\n",
    "        print all.values[i]\n",
    "print \"count: %d\" % count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix Stats\n",
      "good/good: 89.71% (183/204)\n",
      "good/bad: 10.29% (21/204)\n",
      "bad/good: 2.04% (4/196)\n",
      "bad/bad: 97.96% (192/196)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUsAAAEQCAYAAAA5/laqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG2BJREFUeJzt3XucHWWd5/HPNwmQkAABgpmIQFhBFFlBjAwKOijqgAPC\n7DiIXIyIw4wXVhcV8TKKt13GcRwdFTWuYARGRVaXeBkFMysIAxhAkEsQUCcGCAmJhJuCJP3dP+pp\nPbTdfaq7z+nq0/19v1716lNVz6n6nduvn3qep6pkm4iIGN60pgOIiOgFSZYRETUkWUZE1JBkGRFR\nQ5JlREQNSZYRETUkWXaJpFmSviXpAUlfH8N2jpd0SSdja4qkF0j6WRe2O+L3WtIPJb2+07EM2Mdr\nJV3Rxe3/m6TFLfMflrRe0r2SdpX0sKTp3dr/VDOj6QCaJuk44DTg6cBDwA3AR2yP9Uv+SmA+sKPt\nTaPdiO0LgAvGGEvXSTKwp+07hypj+0fAXl3Y/bDvtaQzgT1sn9CFfTfG9uH9jyXtCrwN2M32urJ4\nTiOBTVJTumYp6TTgE8D/pPqx7Qp8BnhFBza/G3D7WBLlZCKpm/+Y815X390NLYly1Lr8WfUu21Ny\nArYDHgb+epgyW1El03vK9Algq7LuEOAuqv/m64A1wEll3QeA3wGPl32cDJwJnN+y7YWAgRll/rXA\nL6hqt78Ejm9ZfkXL854PrAAeKH+f37Luh8CHgCvLdi4B5g3x2vrjP70l/qOBlwO3A78G3t1S/gDg\nKmBjKftpYMuy7vLyWh4pr/dVLdt/J3AvcF7/svKcp5Z97F/mnwzcBxwyRLzPKK9vI3AL8Iqh3usB\nzztswPob67xXwIHAf5T93ThUXKXsLsA3SvwbgE8P8dl9ElgNPAhcB7xgwPt7bVm3Fvh4WT4TOL9s\nd2P5zOe3vIbXAy8Bfgv0ldf4Jf74+7Ud8MXy2d0NfBiY3hLnlcA/l/18uOnf50ScGg+gsRde/Yg2\n9X+ZhijzQeBq4EnATuXH86Gy7pDy/A8CW1Almd8A25f1Z/LE5Dhw/vdfZmB2+ZHsVdYtAJ5ZHv/+\nBwfsANwPnFie9+oyv2NZ/0Pg58DTgFll/qwhXlt//O8r8f9N+bH/K7AN8MzyA9y9lH8OVQKZUWJf\nCby1ZXumOtQduP1/oPqnM4uWZFnK/A1wK7A18H3gY0PEugVwJ/BuYEvgxVQJbq/B3ttBnv9H64d7\nr4CdqZLGy6mOvl5a5ncaZNvTqZLpP5fPcSZw8MDPrsyfAOxY3sO3Uf0TmVnWXQWcWB7PAQ4sj/8W\n+FZ5j6aXz2Hbltfw+pb3u/W9XcgTk+U3gc+XGJ8E/Bj425Y4NwGnlthmNf37nIjTVD4M3xFY7+EP\n3Y4HPmh7ne37qGoxJ7asf7ysf9z2d6n+q4+2Ta4P2EfSLNtrbN8ySJm/AO6wfZ7tTba/AtwGHNlS\n5lzbt9v+LXAhsN8w+3ycqn32ceCrwDzgk7YfKvu/FdgXwPZ1tq8u+/1Pqh/en9V4Te+3/ViJ5wls\nf4EqCV5D9Q/iPUNs50CqBHKW7d/Z/nfg21T/LMZiqPfqBOC7tr9ru8/2pVS1vpcPso0DqGrF77D9\niO1HPUR7t+3zbW8o7+E/Uf0T6f++PA7sIWme7YdtX92yfEeqf0Sby+fw4EhepKT5Jfa3lhjXUSX3\nY1uK3WP7UyW2P/qsYmq3WW4A5rVpn3kysKplflVZ9vttDEi2v2EUjeq2H6E6dP07YI2k70h6eo14\n+mPauWX+3hHEs8H25vK4/weytmX9b/ufL+lpkr5delofpGrnnTfMtgHus/1omzJfAPYBPmX7sSHK\nPBlYbbuvZdnA1z0aQ71XuwF/LWlj/wQcTJXQB9oFWNXmny4Akt4uaWXptd9IdWjc/x6eTFXLvU3S\nCklHlOXnUdW6vyrpHkkflbTFCF/nblS18zUtr+fzVDXMfqtHuM0pZyony6uAx6ja6YZyD9UXrd+u\nZdloPEJ1KNXvT1pX2v6+7ZdS/SBvo0oi7eLpj+nuUcY0Ep+limtP29tSHRKrzXOGvaSVpDlU7cBf\nBM6UtMMQRe8BdpHU+n0dyese6aW1VgPn2Z7bMs22fdYQZXdt1yki6QVU7cPHUDXVzKVqdxaA7Tts\nv5oqgf0DcJGk2eWo5QO296Zqrz4CeM0oXs9jVG2y/a9nW9vPbCmTy4+1MWWTpe0HqNrrPiPpaElb\nS9pC0uGSPlqKfQV4r6SdJM0r5c8f5S5vAF5Yxr9tB7yrf4Wk+ZKOkjSb6kv9MNUh7EDfBZ4m6ThJ\nMyS9Ctib6pC027ahald9uNR63zBg/Vrgv4xwm58ErrX9euA7wOeGKHcNVc3v9PIZHULV9PDVmvtZ\nCywckGyHcz5wpKQ/lzRd0kxJh0h6yiBlf0zVaXKWpNml7EGDlNuGql3wPmCGpPcB2/avlHSCpJ1K\n7XljWdwn6UWS/msZL/kg1WH5YN+NIdleQ9WB9U+StpU0TdJTJbVrRokWUzZZApR2o9OA91J9iVcD\nbwb+bynyYaq2qp8CNwHXl2Wj2delwNfKtq7jiQluWonjHqoe4j/jj5MRtjdQ1SzeRtWMcDpwhO31\no4lphN4OHEfVsfIFqtfS6kxgaTnMO6bdxiQdRdXJ1v86TwP2l3T8wLK2f0eVHA8H1gNnA6+xfVvN\n2PsHqm+QdH27wrZXA0dR1Z77vxfvYJDfS2nGOBLYA/gV1QiAVw2y2e8D36MaabAKeJQnHvoeBtwi\n6WGqfyLHlrbDPwEuokqUK4HLqA7NR+o1VJ1jt1J1Cl7E4M0KMQTZqX1PJpIWAt+2vU/DoUwJY3m/\n81n1lilds4yIqCsj9Rsm6e+phqr0H+5dB/yAqv1ua6qxgK+zfb+k/YZY/hzgnLLJSXEeeY+ZIekC\nYH+qAfOvoWq2OJJqDOd/UI1pdD6r3pWaZYMkPRf4K6qxjIcDi8qqLwPvtP0sqrbS97dZfi5wqu19\nxyv2eIK9gLNtP4OqbfGNVGfxPLccYs+iamuGfFY9K8myWQcBF5eBzA9RnakxG5hr+7JSZilVL/p2\nQyyfW5ZfXpaPpvE/xma17SvL4/OpxmS+SNI1km6iOuPomfmselsOwyPGbmAvqal67BfZXl2uejRz\n3KOKjkrNsllXUo3nm1kGaB9BNXj9/jKIGarTKy8r40IHW74R2Cjp4LL8j4beRNftKul55fFxQP/p\njuvL5/pKgHxWvS01ywbZXiFpGdXYy7VU7ZAPAIuBz0namupKRCeVpwy1/CTgnHJNyXQajL+fAW+S\ndA7VOMbPAtsDN1OdUrmipWw+qx6VcZYNkzTH9sMlAV4OnGK77cDpiBhfqVk2b4mkvanatJYmUUZM\nTKlZRkTUkA6eiIgakiwjImpIsuwhkk5pOoYYmXxm40PSOZLWSbq5ZdkOki6VdEf5u33LundJulPS\nzyT9eZ19JFn2lvzwek8+s/HxJarL3LU6A1hue09geZmndKgeS3WfqcOAs1Xj/upJlhHR88oppL8e\nsPgoqtOCKX+Pbln+1XJvqF9S3QfqgHb7mJRDh7bfakvvPHvynV325K1nss8O207K4QtrtlrYdAhd\nsfW2O7PjgmdNus/skQfu4tHf/LrdbUWG9Zxps/3g728BNbw7eewWqgsm91tie0mbp80vV4mH6uSA\n+eXxzlR3be13FzXu5zQpk+XOs2dy0aF/2nQYMQIf2uPcpkOIEfjelwa70eXIPOjNfGLGwFtKDe6I\nTbc/antR+5KDK5fHG9M/rUmZLCOiBwi0Rc3Kadt7Zw5qraQFttdIWgCsK8vvprorZ7+nUOPmd2mz\njIhGaJqYPmt6rWmUllFdT4Hy9+KW5cdK2krS7sCeVDeeG1ZqlhHRDMG0GWNq9vzDpqSvAIcA8yTd\nRXVh7LOACyWdTHWTuGMAbN8i6UKqi55sAt5Ubjw3rCTLiGjGSA7D2yj3XB/MoUOU/wjwkZHsI8ky\nIhohqWM1y/GQZBkRzehgzXI8JFlGRDM62GY5HpIsI6IREkzfsncG5CRZRkRDhKalZhkRMTyBpqdm\nGRExLAHTpqdmGRExPJHD8IiIdiSlgyciog5NS7KMiBheDsMjIupQOngiItpRapYREfWkzTIioh2J\n6VskWUZEDCuH4RERNeUwPCKindQsIyLqyFWHIiLakmDajFHfuXHcJVlGRGMyKD0ioh3lMDwiopb0\nhkdEtJFxlhERNSVZRkS0I6U3PCKiPaXNMiKiFuUwPCJiWOngiYioKYfhERHtZFB6REQ96Q2PiGgj\nbZYREbUI0mYZEdGeemjoUO+k9YiYXFT1hteZam1O+h+SbpF0s6SvSJopaQdJl0q6o/zdfrThJllG\nRDMkNGN6ran9prQz8N+BRbb3AaYDxwJnAMtt7wksL/OjkmQZEY3RNNWaapoBzJI0A9gauAc4Clha\n1i8Fjh5trEmWEdEIIaRptSZgnqRrW6ZTWrdl+27gY8CvgDXAA7YvAebbXlOK3QvMH2286eCJiGYI\nqF9rXG970ZCbqtoijwJ2BzYCX5d0QmsZ25bkUUabZBkRzeng6Y4vAX5p+z4ASd8Ang+slbTA9hpJ\nC4B1o91BDsMjojEdbLP8FXCgpK1VjUc6FFgJLAMWlzKLgYtHG2tqlhHRDAlN78zpjravkXQRcD2w\nCfgJsASYA1wo6WRgFXDMaPeRZBkRzengGTy23w+8f8Dix6hqmWOWZBkRjZCUM3i6RdJCSTc3HUdE\ndMi0afWmCSA1y4hoTK46VEj6e+AE4D5gNXAd8APgc1Qj7H8OvM72/ZL2G2L5c4BzyiYv6Wa8ETGO\nqmu0NR1FbV2LVNJzgb8C9gUOB/oHlH4ZeKftZwE38YcG2aGWnwucanvfbsUaEc3Q9Om1pomgm2n9\nIOBi24/afgj4FjAbmGv7slJmKfBCSdsNsXxuWX55WX7eUDuTdEr/qVD3P/Z4V15QRHRQ/xk8daYJ\nYNK0WdpeQjWuin122HbUpzRFxHjprfuGdzPSK4EjyzXl5gBHAI8A90t6QSlzInCZ7QeGWL4R2Cjp\n4LL8+C7GGxHjTao3TQBdq1naXiFpGfBTYC1VO+QDVKccfU7S1sAvgJPKU4ZafhJwTjkBPh08EZOF\nmDDDguro9mH4x2yfWRLg5cB1tm8ADhxYcJjl11F1EvU7vVvBRsR46tzpjuOh28lyiaS9gZnAUtvX\nd3l/EdErRE8NHepqsrR9XDe3HxG9bOL0dNcxaXrDI6K3CPqvgt4Tkiwjohkju1J645IsI6IhvXW6\nY5JlRDQnveEREW302IU0kiwjojlps4yIqCE1y4iIGibIed91JFlGRDOknBseEVHLtPSGR0QMLzXL\niIia0mYZEVFDesMjItqZOFdBryPJMiIaYYFzumNERDs53TEiop4ky4iI9pw2y4iINnLVoYiImlKz\njIhoR+kNj4hoq8duhds7kUbEpGNNqzXVIWmupIsk3SZppaTnSdpB0qWS7ih/tx9trEmWEdGQcgZP\nnameTwLfs/10YF9gJXAGsNz2nsDyMj8qSZYR0ZhO1SwlbQe8EPgigO3f2d4IHAUsLcWWAkePNta0\nWUZEM6SRXM9ynqRrW+aX2F7SMr87cB9wrqR9geuAtwDzba8pZe4F5o823CTLiGiEGdGg9PW2Fw2z\nfgawP3Cq7WskfZIBh9y2LcmjCpYchkdEkzSt3tTeXcBdtq8p8xdRJc+1khYAlL/rRhtqkmVENMao\n1tR2O/a9wGpJe5VFhwK3AsuAxWXZYuDi0caaw/CIaIhqDwuq6VTgAklbAr8ATqKqEF4o6WRgFXDM\naDeeZBkRzelgsrR9AzBYu+ahndh+kmVENMISfbm7Y0REDbmQRkREex1us+yqJMuIaEi9nu6JIsky\nIhqTmmVERDticrZZStrK9mPdDCYipg4j+tQ7veFt68CSDpB0E3BHmd9X0qe6HllETHqdvJ5lt9WJ\n4l+AI4ANALZvBF7UzaAiYmro1OmO46HOYfg026v0xLaFzV2KJyKmjI6f7thVdZLlakkHAJY0ner8\ny9u7G1ZETAWT7b7hb6A6FN8VWAv8oCyLiBg1q7c6eNomS9vrgGPHIZaImGImSntkHW2TpaQvUF3U\n+Alsn9KViCJiyphsbZY/aHk8E/hLYHV3womIqWRS1Sxtf611XtJ5wBVdiygipgRPwt7wgXZnDHdI\nGw+r2Jk38KGmw4gReM+XD2s6hBiBFRt+3pHtTKqapaT7+UOb5TTg14zhRuUREf36eug2YMMmS1Uj\n0fcF7i6L+myP+laSERF/INxDyXLYSEti/K7tzWVKooyIjjC9dbpjnbR+g6Rndz2SiJhyeilZDnkY\nLmmG7U3As4EVkn4OPEJ1FTrb3n+cYoyISWqiJMI6hmuz/DGwP/CKcYolIqaUiVNrrGO4ZCkA250Z\nIxAR0cJAn3ung2e4ZLmTpNOGWmn7412IJyKmkMlSs5wOzIEeejUR0VMmS7JcY/uD4xZJREwxwp4c\nybJ3XkVE9BwDfT2UZoZLloeOWxQRMfV4knTw2P71eAYSEVPPZGmzjIjoosnTZhkR0TX954b3iiTL\niGhMapYRETX0NR3ACCRZRkQjjHqqN7x3Io2IScdWrakOSdMl/UTSt8v8DpIulXRH+bv9WGJNsoyI\nxnT4epZvAVa2zJ8BLLe9J7CcMd4OJ8kyIpph6Ks5tSPpKcBfAP+7ZfFRwNLyeClw9FjCTZtlRDRi\nhEOH5km6tmV+ie0lLfOfAE4HtmlZNt/2mvL4XsZ4V9oky4hozAiGDq23vWiwFZKOANbZvk7SIYPv\nx5Y0pnuIJVlGREPE5s6MszwIeIWklwMzgW0lnQ+slbTA9hpJC4B1Y9lJ2iwjohGmM73htt9l+ym2\nFwLHAv9u+wRgGbC4FFsMXDyWeFOzjIjGdPnm2mcBF0o6GVgFHDOWjSVZRkRjOn1uuO0fAj8sjzfQ\nwUtNJllGRDNqDguaKJIsI6IRBvr6ciGNiIi2JsttJSIiuqrLHTwdlWQZEY1wrpQeEVFDOngiIurJ\nYXhERBuGTp3uOC6SLCOiMalZRkTUkGQZEdGGDX05DI+IaC81y4iIGjb30L1wkywjohH917PsFUmW\nEdEM5zA8IqKWnMETEdFGdRjedBT1JVlGRGOSLCMi2nF6wyMi2qqulN50FPUlWUZEY3IYHhFRQ5Jl\nREQb7rGL/04b7x1KWijp5vF+bkRMPLZrTRNBapYR0ZjNm5uOoL5xr1kWMyRdIGmlpIskbS3pfZJW\nSLpZ0hJJApD0HEk3SroReFND8UZEh9n1p4mgqWS5F3C27WcADwJvBD5t+7m29wFmAUeUsucCp9re\nd7gNSjpF0rWSrn38sY3djD0iOqTP9aaJoKlkudr2leXx+cDBwIskXSPpJuDFwDMlzQXm2r68lD1v\nqA3aXmJ7ke1FW2w1t6vBR0Rn9FLNsqk2y4Ev38DZwCLbqyWdCcwc96giYlx5olQba2iqZrmrpOeV\nx8cBV5TH6yXNAV4JYHsjsFHSwWX98eMbZkR0i8vpjnWmiaCpmuXPgDdJOge4FfgssD1wM3AvsKKl\n7EnAOZIMXDLegUZE9/T1UM1y3JOl7f8Enj7IqveWaWD564DWzp3TuxNZRIynXKItIqKOCdR5U0eS\nZUQ0xPT1ULZsqoMnIgL31ZvakbSLpP8n6VZJt0h6S1m+g6RLJd1R/m4/2liTLCOiETZs3uxaUw2b\ngLfZ3hs4kKoDeW/gDGC57T2B5WV+VJIsI6IxnbqQhu01tq8vjx8CVgI7A0cBS0uxpcDRo401bZYR\n0QgzolMZ50m6tmV+ie0lgxWUtBB4NnANMN/2mrLqXmD+aGKFJMuIaIpHdAbPetuL2hUqJ7X8H+Ct\nth8s1+Opdme7jNcelRyGR0RjOnluuKQtqBLlBba/URavlbSgrF8ArBttrEmWEdGYvj7Xmtopl3T8\nIrDS9sdbVi0DFpfHi4GLRxtrDsMjohG26avX013HQcCJwE2SbijL3g2cBVwo6WRgFXDMaHeQZBkR\njenUoHTbVwAaYvWhndhHkmVENGai3F+njiTLiGiEnasORUTU0kMVyyTLiGiGbTZPlCv71pBkGRGN\n6aXbSiRZRkRjkiwjItqZQLe5rSPJMiIaYVKzjIiood7l1yaKJMuIaIZJb3hERDs5DI+IqGNk17Ns\nXJJlRDSkt+7umGQZEY1JzTIiog2Tqw5FRLRn2LwpveEREW1knGVERFs2uC81y4iItnLx34iIGnIY\nHhHRhm360sETEdFen5MsIyKGl9MdIyLaM06yjIioIx08ERHtGPoyzjIiYnjG9G3e3HQYtSVZRkQz\n0sETEVFPkmVERFvOOMuIiHacw/CIiHpy1aGIiHac3vCIiLZMLtEWEdFej138d1rTAUTEVFWdG15n\nqkPSYZJ+JulOSWd0OtrULCOiMe7Q0CFJ04HPAC8F7gJWSFpm+9aO7IAky4hoSHXx34518BwA3Gn7\nFwCSvgocBXQsWaqXrvpRl6T7gFVNx9EF84D1TQcRIzJZP7PdbO80lg1I+h7V+1PHTODRlvkltpe0\nbOuVwGG2X1/mTwT+1PabxxJjq0lZsxzrhzhRSbrW9qKm44j68pkNzfZhTccwEungiYjJ4G5gl5b5\np5RlHZNkGRGTwQpgT0m7S9oSOBZY1skdTMrD8ElsSfsiMcHkMxsHtjdJejPwfWA6cI7tWzq5j0nZ\nwRPdIWkzcBPVP9mVwGLbvxnltg4B3m77CEmvAPa2fdYQZecCx9k+e4T7OBN42PbHRhNjRKschsdI\n/Nb2frb3AX4H/F3rSlVG/J2yvWyoRFnMBd440u1GdFKSZYzWj4A9JC0sZ018GbgZ2EXSyyRdJel6\nSV+XNAd+f4bFbZKuB/5b/4YkvVbSp8vj+ZK+KenGMj0fOAt4qqQbJP1jKfcOSSsk/VTSB1q29R5J\nt0u6Athr3N6NmPTSZhkjJmkGcDjwvbJoT6pD8qslzQPeC7zE9iOS3gmcJumjwBeAFwN3Al8bYvP/\nAlxm+y/LWRlzgDOAfWzvV/b/srLPAwAByyS9EHiEqmF/P6rv9vXAdZ199TFVJVnGSMySdEN5/CPg\ni8CTgVW2ry7LDwT2Bq6UBLAlcBXwdOCXtu8AkHQ+cMog+3gx8BoA25uBByRtP6DMy8r0kzI/hyp5\nbgN8s78dVVJHe0NjakuyjJH4bX/trl9JiI+0LgIutf3qAeWe8LwxEvC/bH9+wD7e2sF9RDxB2iyj\n064GDpK0B4Ck2ZKeBtwGLJT01FLu1UM8fznwhvLc6ZK2Ax6iqjX2+z7wupa20J0lPQm4HDha0ixJ\n2wBHdvi1xRSWZBkdZfs+4LXAVyT9lHIIbvtRqsPu75QOnnVDbOItwIsk3UTV3ri37Q1Uh/U3S/pH\n25cA/wpcVcpdBGxj+3qqttAbgX+jGqgc0REZZxkRUUNqlhERNSRZRkTUkGQZEVFDkmVERA1JlhER\nNSRZRkTUkGQZEVHD/wcqV6qOSuifqwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10fd04d90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import sklearn.ensemble\n",
    "gdc = sklearn.ensemble.GradientBoostingClassifier(n_estimators=35)\n",
    "gdc.fit(X_train_all, y_train_all) \n",
    "y_pred = gdc.predict(X_test_all)  \n",
    "cm     = confusion_matrix(y_test_all, y_pred, labels)       \n",
    "plot_cm(cm, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}