module6_som.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# CPSC 4970 AI + ML: Module 6 -- combining SOM with MLP\n",
    "\n",
    "[sklearn-som documentation](https://pypi.org/project/sklearn-som/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn_som.som import SOM\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.compose import make_column_transformer\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import ConfusionMatrixDisplay"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Fit 5x5 SOM to Iris data set"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(150, 5)\n",
      "Index(['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)',\n",
      "       'petal width (cm)', 'target'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "df = load_iris(as_frame=True)['frame']\n",
    "print(df.shape)\n",
    "print(df.columns)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(df.iloc[:, 0:-1], df['target'])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[17 18 14 11  8 19  6  1 17  1  9  9  1  4  3 12  0 12  0  0  9  6  1 23\n",
      " 12 18 14  6  1 12 14 14  6 14 13  8 14  3  9  9  0  4 18  3 19 18  3  1\n",
      "  9  0  8 13 19 14  9  2  2 13 19 18  9 14  1  1  9  0  1 18  1 14  3 19\n",
      " 13  6 17 17  0 17  1  9  1  0  1 14  5  6 22  1  0  9 18  1 19 19 14  0\n",
      " 23  9  1 22 13  1 13  1 19 12  8  6  3  1  9  0]\n"
     ]
    }
   ],
   "source": [
    "som = SOM(m=5, n=5, dim=df.shape[1]-1)\n",
    "som.fit(X_train.values)\n",
    "print(som.predict(X_train.values))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(112, 4)\n",
      "(112, 5)\n",
      "(38, 4)\n",
      "(38, 5)\n",
      "Training accuracy: 1.0\n",
      "Testing accuracy: 0.8947368421052632\n"
     ]
    }
   ],
   "source": [
    "aug_X_train = np.append(X_train, som.predict(X_train.values).reshape(-1, 1), axis=1)\n",
    "print(X_train.shape)\n",
    "print(aug_X_train.shape)\n",
    "aug_X_test = np.append(X_test, som.predict(X_test.values).reshape(-1, 1), axis=1)\n",
    "print(X_test.shape)\n",
    "print(aug_X_test.shape)\n",
    "\n",
    "pipeline = Pipeline(\n",
    "    [('preproc', make_column_transformer(\n",
    "        (OneHotEncoder(handle_unknown='ignore'), [aug_X_train.shape[1]-1]),\n",
    "        (StandardScaler(), slice(0, aug_X_train.shape[1]-1)))),\n",
    "     ('classifier', MLPClassifier(max_iter=2000))]\n",
    ")\n",
    "\n",
    "cv_params = {\n",
    "    'classifier__hidden_layer_sizes': [(10, ), (20, ), (40, ), (80, ),\n",
    "                                       (10, 5), (20, 10), (40, 20), (80, 40)],\n",
    "    'classifier__alpha': [0.0001, 0.001, 0.01, 0.1, 0.2, 0.4]\n",
    "}\n",
    "\n",
    "cv = GridSearchCV(pipeline, cv_params, n_jobs=-1)\n",
    "cv.fit(aug_X_train, y_train)\n",
    "print(\"Training accuracy:\", accuracy_score(y_train, cv.predict(aug_X_train)))\n",
    "print(\"Testing accuracy:\", accuracy_score(y_test, cv.predict(aug_X_test)))"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'classifier__alpha': 0.0001, 'classifier__hidden_layer_sizes': (80, 40)}\n"
     ]
    },
    {
     "data": {
      "text/plain": "<sklearn.metrics._plot.confusion_matrix.ConfusionMatrixDisplay at 0x1454769b0>"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": "<Figure size 640x480 with 2 Axes>",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe8AAAG2CAYAAABWCA7yAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/av/WaAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAx6klEQVR4nO3deXgUZdb38V+HpJOGJMMWHBJGNlEREBAEcQARGeVVZHFcAJ9ABMUtQRFGBWWT8UFA1EcRHAQVAYVh13FURGVEB4Mi6yAxYRMMS1iCZF+63j+Q1gADCV3d1VX5fq6rL+k7VdWnaMnpc+67q1yGYRgCAAC2EWZ1AAAAoGJI3gAA2AzJGwAAmyF5AwBgMyRvAABshuQNAIDNkLwBALAZkjcAADZD8gYAwGZI3gAABEhRUZF69Oih1NRU39jGjRvVt29ftW7dWjfddJMWLVpU4eOSvAEACIDCwkI99thjSk9P941lZWXpvvvuU7t27bRs2TINHTpUEyZM0OrVqyt07HCTYwUAoNLLyMjQ8OHDdfrtQ1atWqXatWvrsccekyQ1aNBAqampev/999WlS5dyH5/kDQCAydatW6f27dtr2LBhatWqlW+8U6dOatq06Rnb5+TkVOj4JG8AAMqhqKhIRUVFZcbcbrfcbvcZ2/bv3/+sx6hXr57q1avne37kyBF98MEHSklJqVAsJG8AgGMZpYfkqlLHlGMVFxerQ4cOZRJ4cnJyhRPvKQUFBUpJSVHt2rV11113VWhf2yfvWel3qdibb3UYQRMR5tG9TRZWuvP+qF11q0MIOk9MlBbsm6m+9YYo/0SB1eEggCrre33qvAPJVaWOvIc6SUbF2tJnHiha1eqs0dq1a8sMn63qLo/c3Fw99NBD2r17t9555x15PJ4K7W/75F3szVeRN8/qMIKusp133olIq0OwTP6JAuWdqDwf1Coz3uvA8Bon/E/eMhQmKTo62u94cnJydO+99+rHH3/UnDlz1KBBgwofw/bJGwCAcyk1vJLh9fMoXlMSptfrVXJysvbt26e5c+eqcePGF3QckjcAwNG8MiQZ593u3Pzd/6TFixcrNTVVM2bMUGxsrLKysiRJERERql69ermPQ/IGACBIPv74Y3m9Xt1///1lxtu1a6e5c+eW+zgkbwCAo3nlleR/2/xCpaWl+f48e/ZsP+M4ieQNAHC0UsOQjNBom5uFa5sDAGAzVN4AAEcLpQVrZiF5AwAczStDhp/J1xViyZu2OQAANkPlDQBwNCdW3iRvAICjlRrGGffVrqhQS960zQEAsBkqbwCAo3nl/1pxlxmBmIjkDQBwtFLmvAEAsJdSEy6wFmqVN3PeAADYDJU3AMDRmPMGAMBmvHLJ62f6DQux9E3bHAAAm6HyBgA4mtc4+XASkjcAwNFKTWibG7TNAQCAP6i8AQCO5sTKm+QNAHA0r+GS1/A3+YZW8qZtDgCAzVB5AwAcjbY5AAA2U6owef1sNBsh1qgmeQMAHM0wYc7bFWKVd2h9lAAAAOdF5Q0AcLRSuVTqd+UcWpU3yRsA4GilRphKDX8bzaHVqA6taAAAwHlReQMAHO3kLUH9q1VDbcEayRsA4GhOnPOmbQ4AgM1QeQMAHM2JC9ZI3gAAR/OacHnUUJvzDq2PEgAA4LyovAEAjuZVmEr9Xm0eWrUuyRsA4GhmzHmTvAEACCKvCXcV83d/s4VWNAAA4LyovAEAjlZquFTqsFuCkrwBAI5W6sAFa6EVDQAAOC8qbwCAo3mNMHn9XG0eagvWSN4AAEejbQ4AACxH5Q0AcDSv5Pdq87DQWmxO8gYAOBsXaQEAAJaj8gYAOJoZ1zYP8/t+4OYKrWhwhtIi6V+9YnV43a+fs/bvOqg190Tow7bVtfrWWGV9xWcwp4mI9OrhZ3fIe7CNZq9Zrz/ff8jqkBAgvNeBd+p+3v4+QgnJO4SVFkob/lJNJzKq+MYMQxrXZ4qi4gx1XPiz6vUs0rePRCs/M7T+x4J/7hudqcbNc+Wq+bZmjm+oux87qI63ZFsdFgKA9zrwTlXe/j5CiaXRFBYWatSoUWrbtq06duyoN954w8pwQsqJjDB91S9GuT+WfYuyUl3K3HFArceVKKaxV5fcV6AaLUv047JIiyKF2SI9pere76hmP1tfrohmSl1VU4um11HPew5bHRpMxnvtfEVFRerRo4dSU1N9Y3v37lVSUpJatWqlm2++WV9++WWFj2tp8p48ebK2bt2qOXPmaOzYsZo2bZo++ugjK0MKGUe+DVetdiXq+M6JMuNHN4WpyVWNFF7117EaV5UoeyOtc6do3KxA4RGG0jbE+Mb+s66aLm+dJ5fLsDAymI33OjhOXaTF30dFFRYW6rHHHlN6erpvzDAMPfzww6pdu7aWLFmiXr16KTk5WZmZmRU6tmW/8fPy8rRo0SK9/vrratasmZo1a6b09HTNnz9f3bt3tyqskNGgb9FZxwuyXLo4vkaZschaXuUfDK2WDi5czTrFOn40XCXFv76nx7LCFekxFFujVMeP8kHNKXivg8NruOT183veFd0/IyNDw4cPl2GU/RD29ddfa+/evVqwYIGqVq2qxo0ba+3atVqyZIlSUlLKfXzLfuNv375dJSUlat26tW+sTZs22rRpk7xer1VhhbzSfCkiMqLMWJhb8p4918OGIj1eFReV/UVx6nlEJP82nIT32rnWrVun9u3ba+HChWXGN23apCuuuEJVq/7aPm3Tpo02btxYoeNb9rEuKytLNWrUkNvt9o3Vrl1bhYWFys7OVs2aNct1nIgwT6BCDCkRYZFyh7kVEVVFxYXFZc47rDhM4R6X3GFVz3EEe6saUzneZ0lyhUXJHSl5YqIknfxvbI2TaxrCqlRT1RiqMaeo7O/1qfMONK8J1zav8sv+OTk5ZcbdbneZPHZK//79z3qcrKws1alTp8xYrVq1dODAgQrFY9n/Gfn5+Wec8KnnRUXlLyPvbbLw/BvZ3FLdoV5/+F+1vKyZ3rliqdZ/sklP/ea85xgLVaVBuh687GkLowysB49bHUHwGEXfyTh6t97dO12StGDfTBmFX8s4NkTz9syTy8UUiVPwXgeHKXcV+2X/zp07Kzc31zeenJxcoXb3f8t9Fcl7koXJOzIy8oxgTz2Piir/p7FZ6Xep2JtvamyhJ1Ir9o7Sv9MMHa0XpYzvovTaljtluAskSWs+iVCtq7yakdbT4jgD56N21a0OIWjcUaV6O9Wr53r01/gP/q6+9YaoR2KGWnaI0NOJSVaHBxNV9vfaExOlBftmWh1GhXzxxRdlnp+t6j6XyMhIZWdnlxkrKiqqUN6TLEzeF110kY4dO6aSkhKFh58MIysrS1FRUYqNjS33cYq9+Sry5gUqzBARqWJvoYq8JarRVor7Q4JSn9yjxg8U6ODnETq62a0rJ+SoyOvc1al5JyrPV+HyTkir/l5T9zyRJqN4s1q0y1SvezI1ddgflHfC6R9UKxfe6+AolUulfl5k5dT+0dHRfh3noosuUkZGRpmxw4cPn9FKPx/LejJNmzZVeHh4mUn69evXq0WLFgoLo1X037iqSOOXP66CLJfW3BGrn/4RqbYv58gT79zEXRn9bXy8dvynmoyjAzRk7C7Nff73+urD6laHhQDgvQ68U21zfx9maNmypf7zn/+ooKDAN7Z+/Xq1bNmyQsexLEt6PB717t1b48aN0+bNm7Vq1Sq98cYbGjBggFUhhawe/zmm2u1KfM8TLqmrznOLdfOGbF234mfFdSg5x96wo8L8ML385CUKu2ij7u3cRstmxVkdEgKE97pyadeunerWrauRI0cqPT1dM2fO1ObNm3X77bdX6DiWlrgjR45Us2bNNHDgQI0fP14pKSm68cYbrQwJAOAwpfq1dX7hD3NUqVJF06dPV1ZWlm677Ta99957evXVVxUfH1+h41j6PQSPx6NJkyZp0qRJVoYBAHAwM1ebX4i0tLQyz+vXr6958+b5FY+zv0QIAKj0zLixCDcmAQAAfqHyBgA4mmHC/biNELufN8kbAOBotM0BAIDlqLwBAI5mxS1BA43kDQBwtFIT7irm7/5mC61oAADAeVF5AwAcjbY5AAA241WYvH42mv3d32yhFQ0AADgvKm8AgKOVGi6V+tn29nd/s5G8AQCOxpw3AAA2Y5hwVzGDK6wBAAB/UHkDABytVC6V+nljEX/3NxvJGwDgaF7D/zlrr2FSMCahbQ4AgM1QeQMAHM1rwoI1f/c3G8kbAOBoXrnk9XPO2t/9zRZaHyUAAMB5UXkDAByNK6wBAGAzTpzzDq1oAADAeVF5AwAczSsTrm0eYgvWSN4AAEczTFhtbpC8AQAIHifeVYw5bwAAbIbKGwDgaE5cbU7yBgA4Gm1zAABgOSpvAICjOfHa5iRvAICj0TYHAACWo/IGADiaEytvkjcAwNGcmLxpmwMAYDNU3gAAR3Ni5U3yBgA4miH/v+plmBOKaUjeAABHc2LlzZw3AAA2Q+UNAHA0J1beJG8AgKM5MXnTNgcAwGaovAEAjubEypvkDQBwNMNwyfAz+fq7v9lomwMAYDNU3gAAR+N+3gAA2IwT57xpmwMAYLL9+/fr/vvv11VXXaWuXbvqrbfeMvX4VN4AAEezYsHao48+qvj4eC1dulQZGRkaMWKEEhIS9Kc//cmvOE6h8gYAONqptrm/j/I6fvy4Nm7cqAcffFANGjRQt27d1KlTJ61du9a0cyJ5AwAc7VTl7e+jvKKiouTxeLR06VIVFxdr586d+u6779S0aVPTzonkDQBAOeXk5JR5FBUVnbFNZGSkxowZo4ULF6ply5b6f//v/6lz58664447TIvD9nPeH7WrrrwTkVaHETRVYzx68HjlO++PMzdaHULwuaIlSct+2CIZORYHE1y3tO9hdQhBFR598t9yeL14hecUWhxN8Jw670AzTFhtfqry7ty5s3Jzc33jycnJSklJOWP7HTt26Prrr9c999yj9PR0TZgwQR06dFDPnj39iuMU2ydvAADOxZBkGP4fQ5K++OKLMuNut/uMbdeuXavFixfrX//6l6KiotSiRQsdPHhQM2bMMC150zYHAKCcoqOjyzzOlry3bt2q+vXrKyoqyjd2xRVXKDMz07Q4qLwBAI4W7Cus1alTR3v27FFRUZEvue/cuVP16tXzK4bfovIGADhasFebd+3aVREREXr66ae1a9cuffbZZ3rttdeUmJho2jlReQMAYKKYmBi99dZbevbZZ3X77berZs2aevDBB3XXXXeZ9hokbwCAo1lxbfNLLrlEb775pl+veS4kbwCAoxmGCavN/dzfbMx5AwBgM1TeAABHs+LGJIFG8gYAOBrJGwAAm7FiwVqgMecNAIDNUHkDABzNiavNSd4AAEc7mbz9nfM2KRiT0DYHAMBmqLwBAI7GanMAAGzG0K/34/bnGKGEtjkAADZD5Q0AcDTa5gAA2I0D++YkbwCAs5lQeSvEKm/mvAEAsBkqbwCAo3GFNQAAbMaJC9ZomwMAYDNU3gAAZzNc/i84C7HKm+QNAHA0J8550zYHAMBmqLwBAM7GRVoAALAXVpsDAADLUXkDAJwvxNre/ipX8p42bVq5D5icnHzBwQAAYDYnts3LlbxTU1PLdTCXK7RODgCASrtgbe7cuYGOAwAAlNMFLVjbu3evJk2apIceekiHDh3S4sWLtX79erNjAwDABC6THqGjwsn7m2++Uc+ePfXTTz9pzZo1Kiws1M6dOzVw4ECtXLkyEDECAHDhDJMeIaTCyXvKlCkaPny4Xn75ZYWHn+y6P/744xoxYoRefvll0wMEAABlVTh5//DDD7ruuuvOGL/hhhv0448/mhIUAACmofKWEhIStGXLljPGV69erYSEBFOCAgDANKfuKubvI4RU+CItjz76qJ588klt2bJFpaWlWr58ufbt26cPPvhAkydPDkSMAADgNypcef/pT3/S/PnzdeTIETVp0kSffvqpioqKNH/+fN18882BiBEAgAt26pag/j5CyQVdHvXyyy+nygYA2ENlvUjL6ZYvX64FCxZox44dioiIUKNGjZSUlKRu3bqZHR8AADhNhZP3Sy+9pHfeeUcDBgzQ/fffL6/Xq82bN+vxxx/X0KFDlZSUFIAwAQC4QGYsOLP7grWFCxdq0qRJuv76631jN9xwgy6//HI9++yzJG8AQEhxGScf/h4jlFQ4eRuGobp1654x3rBhQxUWFpoSFAAApnHgnHeFV5snJydr7Nix2rFjh29s//79evbZZ/XAAw+YGhwAADhTuSrvyy+/vMztPg3DUI8ePeTxeBQWFqbc3Fy5XC5lZGRo8ODBAQsWAIAKq6xz3m+//Xag4wAAIDAc2DYvV/Ju165duQ526NAhv4IBAADnV+EFazt37tTzzz+vjIwMlZaWSjrZRi8qKtLRo0e1bds204MEAOCCObDyrvCCtdGjR+vo0aMaPHiwDh8+rEGDBql79+7KycnRs88+G4gYAQC4cA68q1iFK+8tW7Zo4cKFatq0qZYvX65GjRrp7rvvVsOGDbV48WL16dMnEHECAIBfVLjyDg8PV0xMjCSpUaNG+v777yVJ1157rdLS0syNDj4RkV4Nm7pX8775Rt5Df1TPezKtDgkBUFTo0pDrL9Omf0f7xr5fH6lHb22iXpe00OCOl+vD+TUtjBCBYhhFmjprlVpcdcTqUJzHgbcErXDybt26tWbPnq2CggI1b95cn332mQzD0NatWxUZGRmIGCHpvtGZuvTKPI0ZeIVcsWN1V/JP6nhLttVhwURFBS4991B97Unz+MaOHjimp++O15XX5mj6yjQljjig6aPrKXVVrIWRwmwREaUysofp4oY/Wx2KI526wpq/j1BS4eQ9cuRIffnll3rnnXfUq1cvHTlyRO3atdNjjz2m/v37X1AQRUVF6tGjh1JTUy9of6eL9JSqe7+jmjEmQTu3VZMr6kYtm1VXPe85bHVoMMmeHyL1SI9Llbmn7Afgr5Z/oxpxpRo0cr8SGhWpS+9sdbv9qD5fVt2aQGG6PzQ8of+dtloq/dHqUGCioqIijR8/XldffbWuvfZavfDCCzJMvK9ohee8L7nkEq1cuVIFBQXyeDxasmSJ1q1bp+rVq6tVq1YVDqCwsFDDhw9Xenp6hfetLBo3K1B4hKFt31aVO+rk2PfrY3T7/T/J5TJkhFg7BxW3eW20Wv7xhJKe2K9el7T0jV/dvZUaXXzwjO1zT1QJZngIoBatj2rrxjg1/OPfZRxsef4dUHEWrDb/61//qtTUVM2ePVu5ubkaNmyY4uPj1bdvXz8DOalcyTsz8+zzq8eOHZMkXXrppb7t4uPjy/3iGRkZGj58uKmfRpyoZp1iHT8arpLiMF/yPn7YrUiPodgapTp+9ILu7IoQcuvAs89z/r5BHdXxFPp+cWQfDtfqFTWUOPxAEKNDIP1zaX1VjY5Uz2RPqC1oxgXKzs7WkiVL9Oabb+rKK6+UJA0aNEibNm0KbvLu2rXrGZdH/e3z346dWsBWHuvWrVP79u01bNiwC6raK4tIj1fFRWX/vk89j4j0WhESLFCY79Iz9zZQzbhi3fw/TJkA5eWSCXcVq8C269evV3R0dJkLnA0ZMsS/AE5TruT96aefmvqip1zoHPlveWKiTIgktLnCouSOlKrGeHznG10jQpIUVqWaqsZUgsrbFX3+bZzE5ZFcvyxJcVVTfq5L45Lq6qedbr2w4idFVatmbXwBVjW6ci1+9VT79XwjPRGV5vx/e952kZOTU+a52+2W2+0uM7Z3714lJCRo+fLleu2111RcXKzbbrtNDz74oMLCKrzU7KzK9Vs/ISHBlBcLhAX7ZlodQsAZRd/JOHq3lme/IZfr5Fs29dMUGceGaN6eeXK5zPmfAaHiDoXVeF1hdZpJkvKjVmrUbf+rzIwDmvL5WP2h2R8sji/wlvzH6gis88ybQ+SKbG91GM5i4o1JOnfurNzcXN9wcnKyUlJSymyal5enPXv2aMGCBZo4caKysrI0ZswYeTweDRo0yL84fmH7kq1vvSHKP1FgdRgB5Y4q1dupXo2/4Xbt/qGOFuybqb//7xhd0SZCTycmWR1eUCz7YYvVIQTRJfIeu0/eQ2FS7S807ta+OpBRRVMW/6SLa/eU98z1a45zR9ebrA4hqDzVIjVv3XhJ0ph7ZmrbpmUWRxQcvz3vgDJxwdoXX3xRZvj0qls6eT2UnJwcTZ061Vf8ZmZm6t133yV5n5J/okB5J/KtDiOg8k5Iq/5eU0PG7NSrT4fJKPhEN//PXk19tJ7jz93HyDn/Nk5i5EuG9NHsz7T5K7fGvbVL0bF5OvpL4g6POLlY0anycgqtDsEyhfnFlfr8Q1109Pmn8OLi4hQZGVmma92wYUPt37/ftDhsn7wri7+Nj1fKxH16Zs42GT+P14JX6umrD6tbHRYCbM3Sr+X1ujRmQKMy41d2yNGUJRkWRQXYTJC/KtayZUsVFhZq165datiwoaSTN/Uycwr6gpJ3aWmp1qxZo927d+u2227Trl271KhRI99lU2G+wvwwPf/oxZo+2qMVx9/WP+YMkFRJqu5K5uPMjb/8KVoTP3xa3oOtK1/noZK644bbqLoDwIwrpFVk/0aNGqlLly4aOXKkxo0bp6ysLM2cOVMPPvigf0H8RoWT9/79+zV48GBlZ2fr+PHjuuGGGzRr1ixt2LBBs2fP1mWXXWZacAAA2NHzzz+vCRMmqF+/fvJ4PLr77ruVmJho2vErnLyfeeYZtWnTRuPGjVPbtm0lSS+88IKeeuop/fWvf9XcuXMvKBBuagIACAgLrrAWExOjyZMn+/mi/12Fv2P07bffatCgQapS5dfLM0ZEROihhx7S1q1bTQ0OAAC/OfB+3hVO3lFRUTpy5MxLOe7atatcq/AAAIB/Kpy8+/btqzFjxmj16tWSTibtJUuWaPTo0br99tvNjg8AAL848ZagFZ7zfvjhhxUbG6tx48YpPz9fQ4YMUa1atZSUlKTBgwcHIkYAAC6ciVdYCxUX9FWxxMREJSYmKi8vT6WlpXxFDAAQuixYsBZoFU7ey5cvP+fPe/fufYGhAACA8qhw8n755ZfLPC8tLdWRI0cUHh6uK6+8kuQNAAgpwb5ISzBUOHl/9tlnZ4zl5uZqzJgxXKAFABB6HNg2N+VektWqVVNKSorefPNNMw4HAADOwbQbk2zfvl1er9eswwEAYA4zvuoVYpV3hZN3YmKiXK6yS+Zzc3OVlpampKQks+ICAMAcDmybVzh5t2/f/owxt9utESNGqEOHDqYEBQAA/rsKJ+/s7GwNGDBAF198cSDiAQDAXA6svCu8YO29995TWJgp69wAAAg4Lo8qKSkpSePHj1dSUpLi4+MVGRlZ5ufx8fGmBQcAAM50wRdpWbNmjST5Fq8ZhiGXy6Xvv//exPAAAMDpypW8v/nmG7Vu3Vrh4eH69NNPAx0TAADmceCcd7mS94ABA/Tll1+qVq1aSkhICHRMAACYxomXRy3XyjPDCLGoAQCoxMo95336hVkAALANh9Wg5U7ef/7zn8v1FTHmxAEAIaWyznlL0j333KOYmJhAxgIAAMqhXMnb5XLplltuUa1atQIdDwAApnLigrVyJW8WrAEAbMuBbfNyrTbv06fPGVdSAwAA1ihX5T1x4sRAxwEAQEBU2rY5AAC2VVnb5gAAIHRQeQMAnM2BlTfJGwDgaMx5AwBgNw6svJnzBgDAZqi8AQDO5sDKm+QNAHA0J8550zYHAMBmqLwBAM5G2xwAAHuhbQ4AACxH5Q0AcDba5gAA2IwDkzdtcwAAbIbKGwDgaK5fHv4eI5SQvAEAzubAtjnJGwDgaHxVDAAAWI7KGwDgbLTNAQCwoRBLvv6ibQ4AgM1QeQMAHI0FawAA2I1h0uMCDRkyRE8++eSFH+AsSN4AAATIBx98oH/961+mH5e2OQDA0axqm2dnZ2vy5Mlq0aKFfy9+FiRvAICzWfRVsUmTJqlXr146dOiQny9+JtrmAACYbO3atfr222/10EMPBeT4VN6whU4P3291CEFX1ePWJ/Okm4bfo7z8IqvDCao1qX+zOoTgckVLek6LPvtYMnKsjiZ4fjnvgL+MiW3znJyy74/b7Zbb7S4zVlhYqLFjx2rMmDGKiory74X/C5I3AMDZTGybd+7cWbm5ub7h5ORkpaSklNl02rRpat68uTp16uTni/53JG8AgLOZmLy/+OKLMsOnV93SyRXmhw8fVuvWrSVJRUUnO2cff/yxNmzY4GcgJ5G8AQAop+jo6PNuM3fuXJWUlPieP//885KkESNGmBYHyRsA4GjB/qpYQkJCmefVqlWTJNWvX9+/IH6D5A0AcDbuKgYAACriuefMX1FP8gYAOJrLMOQy/Cud/d3fbCRvAICzObBtzhXWAACwGSpvAICjOfF+3iRvAICz0TYHAABWo/IGADgabXMAAOzGgW1zkjcAwNGcWHkz5w0AgM1QeQMAnI22OQAA9hNqbW9/0TYHAMBmqLwBAM5mGCcf/h4jhJC8AQCOxmpzAABgOSpvAICzsdocAAB7cXlPPvw9RiihbQ4AgM1QeQMAnI22OQAA9uLE1eYkbwCAsznwe97MeQMAYDNU3gAAR6NtDgCA3ThwwRptcwAAbIbKGwDgaLTNAQCwG1abAwAAq1F5AwAcjbY5AAB2w2pzAABgNSpvAICj0TYHAMBuvMbJh7/HCCEkbwCAszHnDQAArEblDQBwNJdMmPM2JRLzkLwBAM7GFdYAAIDVqLwBAI7GV8UAALAbVpsDAACrUXkDABzNZRhy+bngzN/9zUbyBgA4m/eXh7/HCCG0zQEAsBkqbwCAo9E2BwDAbhy42pzkDQBwNq6wBgAArEbytomISK+GTd2red98I++hP6rnPZlWh4QASog7rmfvWyHvwVaa+9Rb6tdtk9UhwWRFhS4Nuf4ybfp3tG/s+/WRevTWJup1SQsN7ni5Ppxf08IInePUFdb8fYQSS5P3wYMHNXToULVr106dOnXSxIkTVVhYaGVIIeu+0Zm69Mo8jRl4hVyxY3VX8k/qeEu21WEhAFwuQ5Mf/EjHczxy1Vqul5d00YDu36lb2wyrQ4NJigpceu6h+tqT5vGNHT1wTE/fHa8rr83R9JVpShxxQNNH11PqqlgLI3WIU21zfx8VEOj8ZlnyNgxDQ4cOVX5+vubPn68XX3xRn3/+uV566SWrQgpZkZ5Sde93VDPGJGjntmpyRd2oZbPqquc9h60ODQFQMyZf6ftq6ZWlXeQKb6BvtjfQ+rQEXdn4gNWhwQR7fojUIz0uVeaeyDLjXy3/RjXiSjVo5H4lNCpSl97Z6nb7UX2+rLo1geKCBSO/WZa8d+7cqY0bN2rixIlq0qSJ2rZtq6FDh+of//iHVSGFrMbNChQeYWjbt1V9Y9+vj9HlrfPkCrVeDvx25OeqGvdGN+UXumUYhq5osF8tL9mvDel1rQ4NJti8Nlot/3hCL733Q5nxq7u30vCXDp6xfe6JKsEKzbFcXnMe5RWM/GbZavO4uDjNmjVLtWvXLjOek5NjUUShq2adYh0/Gq6S4jC5o06OHT/sVqTHUGyNUh0/ypcGnMrIul4vJmfqqy0X618bGlodDkxw68AjZx3/fYM6quMp9H0lKftwuFavqKHE4XRc/Bbk1ebByG+W/daPjY1Vp06dfM+9Xq/mzZuna665pkLH8cREmR1ayImpmaOS4jBVjfH4zrfKL1k8tqZbxcWR59rdGTxuqyMIuqoet1w1XtFfX5qrIbeu0rC71mnGis5WhxUcrujzb+MULo/k+qUJ6qomSSrMd+mZe+NVs06pbk4scO7fxy/nayenJ2C32y23u+zvJ7Py27m4DCM0vrw2adIkzZ8/X4sXL9all15qdTghxSj4UMbPExRW59+/jpVkyDh8s1x11skVVt264BAURsFHMrKHy3XRBrlcle+DjFP9KewOPf/ZOLXs0sw3lp+TrzG9J2v31r16cc0E1WvCdIm/enWbrLy8Ir+OUbWqWytWPa6rrrpKubm5vvHk5GSlpKScc99A5LeQ6LdOmTJFc+bM0YsvvljhE+tbb4jyTxQEKLLQcFnrE3p23mH9uWaiIqt6tGDfTE3480j95aUw9auZIsNwWR1iwOXd2tbqEIKmenSemtY/oE07L9eK1x9Qr/teU+3YA3r9L8W6/f4X9XOe5/wHsbmPp75pdQhBcom8x+6T91CYwup8qRM7Ounp/r9T5q4ITVqcqfjYm+U9cxrcOVzVFFbny8C/jImXR/3iiy/KjJ9edZ/On/x2LpYn7wkTJujdd9/VlClTdNNNN1V4//wTBco7kR+AyELHtm/CVFLsUv1LD2vX93GSpEZXHFHaRo9yf3b2B5dT8vL9+9RsJw0uOqIxA/+p//lrHUknz73+JZk6diJKB45UkVQJ/i6MSrT2xciXjJOt1WcGxerAnnBNWZKui5v8Ov+N0BEdXf4pDH/z27lY+j3vadOmacGCBXrhhRd0yy23WBlKSCvMD9OqRTU19LmfdEmLHBkFn6jXoP1aPqv2+XeG7WzfE6e0vXF67K5PZZRk6OrLd+vBPql6+6PWVoeGAPpo9mfa/JVHjz6/V9G/K9XRQ+E6eihcPx9jtbnfLPied6Dzm2WV944dOzR9+nQNGTJEbdq0UVZWlu9ncXFxVoUVsv42Pl4pE/fpmTnbZPw8XgteqaevPqxudVgIAK8RppF/u1Ej+q2VceRODbvDqyWrm2vx6uZWh4YAWrP0a3m9Lo0Z0KjM+JUdcjRlCRfo8Ysh/+/HXYHcHYz8ZtmCtZkzZ2rq1Kln/VlaWlq5j9PrdwMc3zb/raoxHq04/nalO++8Pu2tDiHoqnrc+mTeUP3pf16uVNMGkrTm1b9ZHUJwuaIVdtEGeQ+2rlxTBr+cd6D16TJRebl+Llir5tay1SPLta1Z+e1cLKu8hwwZoiFDhlj18gAABEQw8pvlC9YAAAgoQyZcpMWUSExD8gYAOBv38wYAAFaj8gYAOJtX/q8293d/k5G8AQCOZuYV1kIFbXMAAGyGyhsA4GwOXLBG8gYAOJsDkzdtcwAAbIbKGwDgbA6svEneAABn46tiAADYC18VAwAAlqPyBgA4G3PeAADYjNc4+fD3GCGEtjkAADZD5Q0AcDba5gAA2I0JyVuhlbxpmwMAYDNU3gAAZ6NtDgCAzbDaHAAAWI3KGwDgbIb35MPfY4QQkjcAwNmY8wYAwGaY8wYAAFaj8gYAOBttcwAAbMaQCcnblEhMQ9scAACbofIGADgbbXMAAGzG6z358PcYIYS2OQAANkPlDQBwNtrmAADYjAOTN21zAABshsobAOBsDrw8KskbAOBohuGV4eddwfzd32wkbwCAsxkmVN7MeQMAAH9QeQMAnM2Bq81J3gAAZ+MKawAAwGpU3gAAZ6NtDgCAvRherww/297+7m822uYAANgMlTcAwNlomwMAYDMOvDwqbXMAAGyGyhsA4GyGIfl7bXLa5gAABI/hNWT42fb2d3+z0TYHADib4TXnUQGFhYUaNWqU2rZtq44dO+qNN94w9ZSovAEAMNnkyZO1detWzZkzR5mZmXriiScUHx+v7t27m3J8kjcAwNGC3TbPy8vTokWL9Prrr6tZs2Zq1qyZ0tPTNX/+fNOSN21zAICzBbltvn37dpWUlKh169a+sTZt2mjTpk3ymnSlNttX3p6YKKtDCKpT51vZzlset9URBF3VX865aiU8d7mirY4guFzVyv63sgjS+VaN9Zh2jJycnDLjbrdbbnfZf6NZWVmqUaNGmfHatWursLBQ2dnZqlmzpt/x2D55L9g30+oQLFFZz7syWvH6A1aHYIGhVgdgibA6X1odgiO9u/dvphwnNzdXHTp0UFFRkW8sOTlZKSkpZbbLz88/I6Gfev7bff1h++QNAEAwREREaO3atWXGTk/SkhQZGXlGkj71PCrKnK4pyRsAgHI4W4v8bC666CIdO3ZMJSUlCg8/mWazsrIUFRWl2NhYU2JhwRoAACZq2rSpwsPDtXHjRt/Y+vXr1aJFC4WFmZN2Sd4AAJjI4/God+/eGjdunDZv3qxVq1bpjTfe0IABA0x7DZdhhNgFWwEAsLn8/HyNGzdOK1euVHR0tAYPHqykpCTTjk/yBgDAZmibAwBgMyRvAABshuQNAIDNkLxtItC3l0NoKioqUo8ePZSammp1KAiQgwcPaujQoWrXrp06deqkiRMnqrCw0OqwEOK4SItNBPr2cgg9hYWFGj58uNLT060OBQFiGIaGDh2q2NhYzZ8/X8ePH9eoUaMUFhamJ554wurwEMJI3jYQjNvLIbRkZGRo+PDh4ssgzrZz505t3LhRX331lWrXri1JGjp0qCZNmkTyxjnRNreBYNxeDqFl3bp1at++vRYuXGh1KAiguLg4zZo1y5e4Tzn9zlXA6ai8bSAYt5dDaOnfv7/VISAIYmNj1alTJ99zr9erefPm6ZprrrEwKtgBydsGgnF7OQDWmzJlirZt26bFixdbHQpCHMnbBoJxezkA1poyZYrmzJmjF198UZdeeqnV4SDEkbxtIBi3lwNgnQkTJujdd9/VlClTdNNNN1kdDmyABWs2EIzbywGwxrRp07RgwQK98MILuuWWW6wOBzbBb34bCMbt5QAE344dOzR9+nTdd999atOmjbKysnwP4Fxom9vEyJEjNW7cOA0cOFDR0dFKSUnRjTfeaHVYAPzw6aefqrS0VDNmzNCMGTPK/CwtLc2iqGAH3BIUAACboW0OAIDNkLwBALAZkjcAADZD8gYAwGZI3gAA2AzJGwAAmyF5AwBgMyRvVBpdu3bVZZdd5ns0a9ZM3bt311tvvWXq6yQmJuqVV16RJD355JN68sknz7tPUVGR/v73v1/way5dulRdu3at8M9O98orrygxMfGC47jsssuUmpp6wfsDKB+usIZKZdSoUbr55pslSSUlJfr666/11FNPqXr16urdu7fpr/fUU0+Va7sPPvhAr732mu68807TYwDgPFTeqFRiYmIUFxenuLg41a1bV3369FGHDh20cuXKgL1eTEzMebfjQocAKoLkjUovPDxcERERkk62vCdMmKAbbrhBXbp0UU5Ojvbv368HHnhALVu2VNeuXTVt2jSVlpb69v/kk0900003qVWrVnrmmWfK/Oz0tvmKFSvUvXt3tWzZUn379tW2bduUmpqqkSNH6qefftJll12mffv2yTAMvfrqq+rYsaPatm2rBx54QJmZmb7jHDx4UPfee69atWqlPn366Mcffyz3+X766afq3bu3WrRoobZt2+qxxx5Tbm6u7+fFxcV66qmn1LJlS3Xr1k3//Oc/fT87X1wAgoPkjUqruLhYK1eu1FdffaUbbrjBN7506VJNmTJF06ZNU7Vq1ZScnKxatWpp2bJlmjhxot5//3299tprkqSMjAw9+uij6tevn5YsWaKSkhKtX7/+rK+3Zs0aPfXUUxo4cKDee+89NW/eXPfff79at26tUaNG6fe//72+/PJL1a1bV/PmzdP777+vqVOnauHChapVq5YGDRqk4uJiSdIjjzwir9erRYsW6b777tOcOXPKdc4//vijHnnkEfXv318ffvihXnrpJf373/8uM9++YcMG399Dv379NGLECO3Zs0eSzhsXgOBgzhuVytixYzVhwgRJUkFBgaKiojRw4ED17NnTt02XLl101VVXSZLWrl2rzMxMLVq0SGFhYWrUqJGeeOIJjRw5Ug8//LCWLFmitm3bKikpSZI0evRoff7552d97YULF6pHjx7q16+fJOnxxx9XRESEjh8/rpiYGFWpUkVxcXGSpFmzZmns2LFq3769JOmZZ55Rx44dtWbNGv3hD3/Qhg0b9Pnnnys+Pl5NmjTR1q1b9dFHH533/L1er55++mnf3Hq9evV07bXXKj093bdNnTp1NG7cOEVERKhx48ZavXq1Fi1apBEjRpwzrvIuigPgP5I3KpWhQ4f6bqUaGRmpuLg4ValSpcw2CQkJvj/v2LFD2dnZatOmjW/M6/WqoKBAx44d044dO9S0aVPfzyIiIso8/61du3apb9++vudut1tPPPHEGdvl5ubqwIEDGjZsmMLCfm2OFRQUaPfu3SosLFT16tUVHx/v+1mLFi3KlbwbNGggt9utGTNmKD09Xenp6crIyFCvXr182zRt2tQ3jSBJzZo1044dO84bF4DgIXmjUqlVq5bq169/zm0iIyN9fy4pKVGjRo00ffr0M7Y7tRDt9MVmv018vxUeXr5/bqfmzP/v//5PDRs2LPOz3/3ud1q7dm25X/N027dvV79+/dS1a1dfx+D0lvtvE7N08sNKRETEeeMCEDzMeQPn0LBhQ2VmZqpmzZqqX7++6tevr3379unll1+Wy+VSkyZNtGXLFt/2Xq9X27dvP+ux6tevX+ZnpaWl6tq1q9avXy+Xy+Ubj42NVa1atZSVleV7zbp162rKlCnatWuXLr30Uh0/ftw3Dy1J33//fbnOZ8WKFbr66qs1depU9e/fX1deeaX27NlT5sPAb1vokrR582Y1atTovHEBCB6SN3AOHTt2VEJCgv7yl78oLS1N3377rUaPHi2Px6MqVarozjvv1NatWzVjxgzt3LlTkyZN+q+rrxMTE/Xee+9p2bJl2rNnjyZOnCjDMNSsWTN5PB4dP35cu3fvVklJiZKSkvTSSy/ps88+0+7du/X000/ru+++U6NGjdS4cWN16NBBo0aN0vbt27Vq1SrNmzevXOdTvXp1paWlafPmzdq1a5eee+45bdmyRUVFRb5tMjMzNWHCBO3YsUOvvvqqtm3b5punP1dcAIKHtjlwDlWqVNGMGTM0YcIE3Xnnnapataq6d+/um6uuX7++ZsyYoYkTJ2rGjBnq1q2brrvuurMe6+qrr9bYsWP16quvKisrS82bN9drr72mqKgoXXPNNapfv75uvfVWvfPOOxo8eLByc3M1ZswY5eTkqHnz5po9e7avPf3iiy9q9OjR6tu3r+Lj45WYmKilS5ee93wSExO1bds2JSUlKTIyUldffbUefvhhffDBB75trrvuOmVnZ6tPnz5KSEjQjBkzdNFFF0nSeeMCEBwug6tDAABgK7TNAQCwGZI3AAA2Q/IGAMBmSN4AANgMyRsAAJsheQMAYDMkbwAAbIbkDQCAzZC8AQCwGZI3AAA2Q/IGAMBmSN4AANjM/we+KQspb5Lc4gAAAABJRU5ErkJggg==\n"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(cv.best_params_)\n",
    "ConfusionMatrixDisplay.from_estimator(cv, aug_X_test, y_test)"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}