7b) Clustering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7b) Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn\n",
    "from sklearn.cluster import KMeans \n",
    "from sklearn.cluster import SpectralClustering\n",
    "from sklearn.cluster import AgglomerativeClustering\n",
    "from sklearn.cluster import DBSCAN\n",
    "from sklearn.cluster import MiniBatchKMeans\n",
    "from sklearn.metrics import silhouette_score\n",
    "\n",
    "#use pandas to load csv file\n",
    "import pandas as pandas\n",
    "#use numpy to calculate stuff\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load data and merge both tables to one, ignore_index to reindex\n",
    "redwinedata = pandas.read_csv('data/winequality-red.csv', sep =';')\n",
    "whitewinedata = pandas.read_csv('data/winequality-white.csv', sep =';')\n",
    "concat_data = redwinedata.append(whitewinedata, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop the quality label and data\n",
    "concat_data = concat_data.drop('quality', axis=1)\n",
    "winearray = concat_data.values\n",
    "# normalize the data \n",
    "winearray_norm = sklearn.preprocessing.scale(winearray)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### K-Means (with/without minibatch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[2, 0.2554064489554629], [3, 0.23387486968415436], [4, 0.22123871017610744], [5, 0.1243876329737778], [6, 0.179467005043885], [7, 0.17701427984357385]]\n"
     ]
    }
   ],
   "source": [
    "# http://scikit-learn.org/stable/auto_examples/cluster/plot_kmeans_silhouette_analysis.html\n",
    "batchkmeans_sil_scores = []\n",
    "\n",
    "for nr_clusters in range(2, 8):\n",
    "    clusterer = MiniBatchKMeans(n_clusters=nr_clusters)\n",
    "    cluster_labels = clusterer.fit_predict(winearray_norm)\n",
    "    #print(cluster_labels)\n",
    "    silhouette_avg = silhouette_score(winearray_norm, cluster_labels)\n",
    "    #print(\"For n_clusters =\", nr_clusters,\"The average silhouette_score is :\", silhouette_avg)\n",
    "    batchkmeans_sil_scores.append([nr_clusters, silhouette_avg])\n",
    "\n",
    "print(batchkmeans_sil_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[2, 0.2766102466709183], [3, 0.23507051305667923], [4, 0.24752476415510705], [5, 0.18031988651384204], [6, 0.18626179598840933], [7, 0.1742929153287071]]\n"
     ]
    }
   ],
   "source": [
    "# http://scikit-learn.org/stable/auto_examples/cluster/plot_kmeans_silhouette_analysis.html\n",
    "kmeans_sil_scores = []\n",
    "\n",
    "for nr_clusters in range(2, 8):\n",
    "    clusterer = sklearn.cluster.KMeans(n_clusters=nr_clusters, random_state=10)\n",
    "    cluster_labels = clusterer.fit_predict(winearray_norm)\n",
    "    #print(cluster_labels)\n",
    "    silhouette_avg = silhouette_score(winearray_norm, cluster_labels)\n",
    "    #print(\"For n_clusters =\", nr_clusters,\"The average silhouette_score is :\", silhouette_avg)\n",
    "    kmeans_sil_scores.append([nr_clusters, silhouette_avg])\n",
    "\n",
    "print(kmeans_sil_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Average Linkage Agglomerative Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nr of samples per Cluster: [6496    1]\n",
      "Nr of samples per Cluster: [6495    1    1]\n",
      "Nr of samples per Cluster: [  29 6466    1    1]\n",
      "Nr of samples per Cluster: [  28 6466    1    1    1]\n",
      "Nr of samples per Cluster: [6466    7   21    1    1    1]\n",
      "Nr of samples per Cluster: [6464    7   21    1    1    1    2]\n",
      "[[2, 0.7827338879667882], [3, 0.7264973604547182], [4, 0.6177452770210612], [5, 0.6069273341713797], [6, 0.5192470945503151], [7, 0.4475012710042327]]\n"
     ]
    }
   ],
   "source": [
    "average_sil_scores = []\n",
    "\n",
    "for nr_clusters in range(2, 8):\n",
    "    clusterer = AgglomerativeClustering(linkage=\"average\", affinity=\"cityblock\", n_clusters=nr_clusters)\n",
    "    cluster_labels = clusterer.fit_predict(winearray_norm)\n",
    "    print(\"Nr of samples per Cluster: \"+str(np.bincount(cluster_labels)))\n",
    "    silhouette_avg = silhouette_score(winearray_norm, cluster_labels)\n",
    "    average_sil_scores.append([nr_clusters, silhouette_avg])\n",
    "\n",
    "print(average_sil_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This looks suspiciously better. This is why I also print the number of elements per clusters above. And indeed it becomes clear that it just puts nearly all the values into one cluster which is not the intended behaviour. Another try with other affinity types, e.g. cosine, the results are very similar to K-means:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nr of samples per Cluster: [4846 1651]\n",
      "Nr of samples per Cluster: [2327 1651 2519]\n",
      "Nr of samples per Cluster: [2519 1651 2052  275]\n",
      "Nr of samples per Cluster: [2052 1651 2482  275   37]\n",
      "Nr of samples per Cluster: [1679 1651 2482  275   37  373]\n",
      "Nr of samples per Cluster: [2482 1651  373  275   37 1561  118]\n",
      "[[2, 0.2662649268569845], [3, 0.19405970145838983], [4, 0.11613055600380129], [5, 0.08760391368250511], [6, 0.07933900330106827], [7, 0.09020528542502969]]\n"
     ]
    }
   ],
   "source": [
    "average_sil_scores = []\n",
    "\n",
    "for nr_clusters in range(2, 8):\n",
    "    clusterer = AgglomerativeClustering(linkage=\"average\", affinity=\"cosine\", n_clusters=nr_clusters)\n",
    "    cluster_labels = clusterer.fit_predict(winearray_norm)\n",
    "    print(\"Nr of samples per Cluster: \"+str(np.bincount(cluster_labels)))\n",
    "    silhouette_avg = silhouette_score(winearray_norm, cluster_labels)\n",
    "    average_sil_scores.append([nr_clusters, silhouette_avg])\n",
    "\n",
    "print(average_sil_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ward Linkage Agglomerative Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[2, 0.2668673064992951], [3, 0.20047387790440627], [4, 0.20673579244071638], [5, 0.13432744813769548], [6, 0.1439585385849316], [7, 0.12321328875640453]]\n"
     ]
    }
   ],
   "source": [
    "ward_sil_scores = []\n",
    "\n",
    "for nr_clusters in range(2, 8):\n",
    "    clusterer =  clustering = AgglomerativeClustering(linkage='ward', n_clusters=nr_clusters)\n",
    "    cluster_labels = clusterer.fit_predict(winearray_norm)\n",
    "    silhouette_avg = silhouette_score(winearray_norm, cluster_labels)\n",
    "    ward_sil_scores.append([nr_clusters, silhouette_avg])\n",
    "\n",
    "print(ward_sil_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Summarize the results in a table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Clusters  K-Means  Minibatch-K-Mean  Aggl. Average  Aggl. Ward\n",
      "0         2    0.277             0.255          0.266       0.267\n",
      "1         3    0.235             0.234          0.194       0.200\n",
      "2         4    0.248             0.221          0.116       0.207\n",
      "3         5    0.180             0.124          0.088       0.134\n",
      "4         6    0.186             0.179          0.079       0.144\n",
      "5         7    0.174             0.177          0.090       0.123\n"
     ]
    }
   ],
   "source": [
    "table_index = [];\n",
    "table_kmeans = [];\n",
    "for member in kmeans_sil_scores:\n",
    "    table_index.append(member[0])\n",
    "    table_kmeans.append(member[1])\n",
    "\n",
    "table_batchkmeans = [];\n",
    "for member in batchkmeans_sil_scores:\n",
    "    table_batchkmeans.append(member[1])  \n",
    "    \n",
    "table_ward = [];\n",
    "for member in ward_sil_scores:\n",
    "    table_ward.append(member[1])\n",
    "\n",
    "table_average = [];\n",
    "for member in average_sil_scores:\n",
    "    table_average.append(member[1])     \n",
    "        \n",
    "\n",
    "df = pandas.DataFrame({'Clusters':table_index, 'K-Means':table_kmeans, 'Minibatch-K-Mean':table_batchkmeans, 'Aggl. Ward':table_ward, 'Aggl. Average':table_average}).round(3)\n",
    "print (df[['Clusters', 'K-Means', 'Minibatch-K-Mean', 'Aggl. Average', 'Aggl. Ward']])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Surprisingly, the silhoutte score results were overall rather bad and very similar, which K-Means having the best outcome while also beeing very efficient.    \n",
    "\n",
    "All the algorithms show that the more clusters are defined, the less accurate the clustering is. This hints to the fact that the algorithms are able to pick up the difference between white and red wines. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Basic statistics for K-Means\n",
    "As the redwine and whitewine data was appended to each other and never reordered, it is easy to check if the clustering picked up the difference between red and white wines by splitting the resulting array according to the sizes of the sample for white and red wines.    \n",
    "\n",
    "It shows very clearly that the clustering found out the difference between red and white wines, because the first 1599 (sample size of red whines) nearly all went into one cluster and the rest into the second cluster."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overall number of samples per Cluster: [1641 4856]\n",
      "Number of samples per Cluster in Redwine only: [1575   24]\n",
      "Number of samples per Cluster in Whitewine only: [  66 4832]\n"
     ]
    }
   ],
   "source": [
    "clusterer = KMeans(n_clusters=2, random_state=10)\n",
    "cluster_labels = clusterer.fit_predict(winearray_norm)\n",
    "silhouette_avg = silhouette_score(winearray_norm, cluster_labels)\n",
    "\n",
    "print(\"Overall number of samples per Cluster: \"+str(np.bincount(cluster_labels)))\n",
    "cluster_red = cluster_labels[:1599]\n",
    "cluster_white = cluster_labels[1599:]\n",
    "print(\"Number of samples per Cluster in Redwine only: \"+str(np.bincount(cluster_red)))\n",
    "print(\"Number of samples per Cluster in Whitewine only: \"+str(np.bincount(cluster_white)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here is some additional data about the clusters. More or less all of the features have very different data in both clusters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "first cluster:\n",
      "            0        1        2        3        4        5        6        7   \\\n",
      "count  1641.00  1641.00  1641.00  1641.00  1641.00  1641.00  1641.00  1641.00   \n",
      "mean      0.83     1.17    -0.34    -0.59     0.92    -0.83    -1.19     0.68   \n",
      "std       1.34     1.08     1.33     0.46     1.40     0.58     0.63     0.72   \n",
      "min      -2.02    -1.33    -2.19    -0.98    -1.23    -1.66    -1.94    -1.63   \n",
      "max       6.70     7.53     4.69    12.69    15.84     2.73     1.97    14.77   \n",
      "\n",
      "            8        9        10  \n",
      "count  1641.00  1641.00  1641.00  \n",
      "mean      0.57     0.84    -0.07  \n",
      "std       0.97     1.14     0.90  \n",
      "min      -2.98    -1.42    -1.75  \n",
      "max       4.92     9.87     3.70  \n",
      "\n",
      "second cluster:\n",
      "            0        1        2        3        4        5        6        7   \\\n",
      "count  4856.00  4856.00  4856.00  4856.00  4856.00  4856.00  4856.00  4856.00   \n",
      "mean     -0.28    -0.40     0.11     0.20    -0.31     0.28     0.40    -0.23   \n",
      "std       0.65     0.57     0.83     1.05     0.55     0.96     0.75     0.97   \n",
      "min      -2.63    -1.58    -2.19    -1.02    -1.34    -1.61    -1.89    -2.53   \n",
      "max       5.39     3.10     9.23     5.50     5.25    14.56     5.74     5.20   \n",
      "\n",
      "            8        9        10  \n",
      "count  4856.00  4856.00  4856.00  \n",
      "mean     -0.19    -0.28     0.03  \n",
      "std       0.93     0.76     1.03  \n",
      "min      -3.10    -2.09    -2.09  \n",
      "max       3.74     3.69     3.11  \n"
     ]
    }
   ],
   "source": [
    "#convert nparray to dataframe and merge with normalized data (converted to dataframe) \n",
    "cluster_labels_dataframe = pandas.DataFrame({'cluster_nr':cluster_labels})\n",
    "concat_with_cluster_nr = pandas.concat([pandas.DataFrame(winearray_norm), cluster_labels_dataframe], axis=1)\n",
    "\n",
    "cluster1_data = concat_with_cluster_nr[concat_with_cluster_nr.cluster_nr == 0]\n",
    "print(\"first cluster:\")\n",
    "print(cluster1_data.describe().round(2).drop(['25%', '50%', '75%']).drop('cluster_nr', axis=1))\n",
    "\n",
    "cluster2_data = concat_with_cluster_nr[concat_with_cluster_nr.cluster_nr == 1]\n",
    "print(\"\")\n",
    "print(\"second cluster:\")\n",
    "print(cluster2_data.describe().round(2).drop(['25%', '50%', '75%']).drop('cluster_nr', axis=1))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}