test_scikit_majority.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Preparing Data\n",
    "=============="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import cPickle as pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas import factorize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bossa/control_tasks.json       bossa/tasks_export.json\r\n",
      "bossa/control_tasks_runs.json  bossa/tasks_runs_export.json\r\n"
     ]
    }
   ],
   "source": [
    "!ls bossa/*json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BOSSA Results\n",
    "-------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Processing `results_bossa.json` to get a *dictionary* with keys the task ids, and values in as the average value of the scores. To do that, we first convert scores from categorical (`neg`, `neu`, `pos`) to a numeric scale."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>result_id</th>\n",
       "      <th>seconds</th>\n",
       "      <th>task_id</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>11203</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>52775</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    result_id   seconds  task_id score\n",
       "50      11203  0.000025    52775     1"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bossa_results = pd.read_json(\"bossa/tasks_runs_export.json\")\n",
    "bossa_results.rename(columns={\"created\": \"start_time\", \"id\": \"result_id\", \"info\": \"score\"}, inplace=True)\n",
    "bossa_results[['start_time']]= bossa_results[['start_time']].apply(pd.to_datetime, dayfirst=True)\n",
    "bossa_results[['finish_time']]= bossa_results[['finish_time']].apply(pd.to_datetime, dayfirst=True)\n",
    "bossa_results['score'] = pd.Categorical(bossa_results['score'], categories=['vneg', 'neg', 'neu', 'pos', 'vpos'])\n",
    "bossa_results['score'].cat.rename_categories([-2, -1, 0, 1, 2], inplace=True)\n",
    "# Normalize everything to -1, 0, 1\n",
    "# bossa_results['score'] = bossa_results['score'].astype(float).apply(lambda x: -1 if x < 0 else 1 if x > 0 else 0)\n",
    "bossa_results[\"seconds\"] = (bossa_results[\"finish_time\"] - bossa_results[\"start_time\"]).astype('timedelta64[us]') / 1e6\n",
    "bossa_results = bossa_results[[\"result_id\", \"seconds\", \"task_id\", \"score\"]]\n",
    "bossa_results.ix[[50]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The information about the sentence comes in a dictionary inside the cells of the serie `info`, so we expand it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task_id</th>\n",
       "      <th>info</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>52851</td>\n",
       "      <td>{u'search_words': u'founder', u'appears_in_sen...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    task_id                                               info\n",
       "50    52851  {u'search_words': u'founder', u'appears_in_sen..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bossa_tasks = pd.read_json(\"bossa/tasks_export.json\")\n",
    "bossa_tasks[['created']]= bossa_tasks[['created']].apply(pd.to_datetime, dayfirst=True)\n",
    "bossa_tasks.rename(columns={'id': 'task_id'}, inplace=True)\n",
    "bossa_tasks = bossa_tasks[['task_id', 'info']]\n",
    "bossa_tasks.ix[[50]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And finally we merge the `DataFrame` with the scores with the one containing the sentences."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>result_id</th>\n",
       "      <th>seconds</th>\n",
       "      <th>task_id</th>\n",
       "      <th>score</th>\n",
       "      <th>info</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>11195</td>\n",
       "      <td>0.000021</td>\n",
       "      <td>52776</td>\n",
       "      <td>2</td>\n",
       "      <td>{u'search_words': u'executive', u'appears_in_s...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    result_id   seconds  task_id  score  \\\n",
       "50      11195  0.000021    52776      2   \n",
       "\n",
       "                                                 info  \n",
       "50  {u'search_words': u'executive', u'appears_in_s...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bossa_tasks_scores = pd.merge(bossa_results, bossa_tasks, on='task_id')\n",
    "bossa_tasks_scores.ix[[50]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's now expand the column `info` into as many new columns as keys has the dictionary `info`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'search_words',\n",
       " u'appears_in_sentence',\n",
       " u'url',\n",
       " u'media',\n",
       " u'appears_in_noun_phrases',\n",
       " u'noun_phrases',\n",
       " u'sentence_id',\n",
       " u'text',\n",
       " u'sentence',\n",
       " u'pub_date',\n",
       " u'is_company']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bossa_tasks_scores.ix[50].info.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>result_id</th>\n",
       "      <th>seconds</th>\n",
       "      <th>task_id</th>\n",
       "      <th>score</th>\n",
       "      <th>search_words</th>\n",
       "      <th>appears_in_sentence</th>\n",
       "      <th>url</th>\n",
       "      <th>media</th>\n",
       "      <th>appears_in_noun_phrases</th>\n",
       "      <th>noun_phrases</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>text</th>\n",
       "      <th>sentence</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>is_company</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>11195</td>\n",
       "      <td>0.000021</td>\n",
       "      <td>52776</td>\n",
       "      <td>2</td>\n",
       "      <td>executive</td>\n",
       "      <td>0</td>\n",
       "      <td>http://dealbook.nytimes.com/2013/05/17/a-toeho...</td>\n",
       "      <td>nyt</td>\n",
       "      <td>0</td>\n",
       "      <td>[chinese investors, overseas companies, politi...</td>\n",
       "      <td>14</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>2013-05-17T11:47:51Z</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>11205</td>\n",
       "      <td>0.000018</td>\n",
       "      <td>52776</td>\n",
       "      <td>-1</td>\n",
       "      <td>executive</td>\n",
       "      <td>0</td>\n",
       "      <td>http://dealbook.nytimes.com/2013/05/17/a-toeho...</td>\n",
       "      <td>nyt</td>\n",
       "      <td>0</td>\n",
       "      <td>[chinese investors, overseas companies, politi...</td>\n",
       "      <td>14</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>2013-05-17T11:47:51Z</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>11207</td>\n",
       "      <td>0.000017</td>\n",
       "      <td>52776</td>\n",
       "      <td>1</td>\n",
       "      <td>executive</td>\n",
       "      <td>0</td>\n",
       "      <td>http://dealbook.nytimes.com/2013/05/17/a-toeho...</td>\n",
       "      <td>nyt</td>\n",
       "      <td>0</td>\n",
       "      <td>[chinese investors, overseas companies, politi...</td>\n",
       "      <td>14</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>2013-05-17T11:47:51Z</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>11209</td>\n",
       "      <td>0.000017</td>\n",
       "      <td>52776</td>\n",
       "      <td>-2</td>\n",
       "      <td>executive</td>\n",
       "      <td>0</td>\n",
       "      <td>http://dealbook.nytimes.com/2013/05/17/a-toeho...</td>\n",
       "      <td>nyt</td>\n",
       "      <td>0</td>\n",
       "      <td>[chinese investors, overseas companies, politi...</td>\n",
       "      <td>14</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>2013-05-17T11:47:51Z</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    result_id   seconds  task_id  score search_words appears_in_sentence  \\\n",
       "50      11195  0.000021    52776      2    executive                   0   \n",
       "51      11205  0.000018    52776     -1    executive                   0   \n",
       "52      11207  0.000017    52776      1    executive                   0   \n",
       "53      11209  0.000017    52776     -2    executive                   0   \n",
       "\n",
       "                                                  url media  \\\n",
       "50  http://dealbook.nytimes.com/2013/05/17/a-toeho...   nyt   \n",
       "51  http://dealbook.nytimes.com/2013/05/17/a-toeho...   nyt   \n",
       "52  http://dealbook.nytimes.com/2013/05/17/a-toeho...   nyt   \n",
       "53  http://dealbook.nytimes.com/2013/05/17/a-toeho...   nyt   \n",
       "\n",
       "   appears_in_noun_phrases                                       noun_phrases  \\\n",
       "50                       0  [chinese investors, overseas companies, politi...   \n",
       "51                       0  [chinese investors, overseas companies, politi...   \n",
       "52                       0  [chinese investors, overseas companies, politi...   \n",
       "53                       0  [chinese investors, overseas companies, politi...   \n",
       "\n",
       "   sentence_id                                               text  \\\n",
       "50          14  Chinese investors are increasingly opting to b...   \n",
       "51          14  Chinese investors are increasingly opting to b...   \n",
       "52          14  Chinese investors are increasingly opting to b...   \n",
       "53          14  Chinese investors are increasingly opting to b...   \n",
       "\n",
       "                                             sentence              pub_date  \\\n",
       "50  Chinese investors are increasingly opting to b...  2013-05-17T11:47:51Z   \n",
       "51  Chinese investors are increasingly opting to b...  2013-05-17T11:47:51Z   \n",
       "52  Chinese investors are increasingly opting to b...  2013-05-17T11:47:51Z   \n",
       "53  Chinese investors are increasingly opting to b...  2013-05-17T11:47:51Z   \n",
       "\n",
       "   is_company  \n",
       "50          0  \n",
       "51          0  \n",
       "52          0  \n",
       "53          0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def json_to_series(info):\n",
    "    keys, values = zip(*info.iteritems())\n",
    "    return pd.Series(values, index=keys)\n",
    "\n",
    "bossa_info = bossa_tasks_scores[\"info\"].apply(json_to_series)\n",
    "bossa_info.reset_index()\n",
    "bossa = pd.concat([bossa_tasks_scores, bossa_info], axis=1)\n",
    "bossa.pop(\"info\")\n",
    "# bossa['id'] = bossa['id'].astype(float)\n",
    "bossa.ix[50:53]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Aggregate\n",
    "---------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now aggregate calculating the average per `sentence_id` using a group by. In the process, we lose the source of the data, that's why we first have to save it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bossa.to_csv(\"sentiment/scores_ungrouped.csv\", encoding=\"utf8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, we aggregate and create a new `DataFrame` for the different sentences and their score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score    8996\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sentence</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>'We must hope after so much prevarication that this time Google's proposals represent a genuine attempt to address the concerns identified,' said David Wood, the legal counsel for Icomp, an industry group backed by Microsoft and a number of other companies.</th>\n",
       "      <td>-0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>'We must push our leaders to step up and commit to action,' said Hugh Evans, the founder and chief executive of the charity.</th>\n",
       "      <td>-0.285714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>'We need them to tell the story of how we are making decisions and putting the organization together,' said George Postolos, the Astros' president and chief executive, who added that the team would not want a broadcaster who was uncomfortable explaining the front office's strategy.</th>\n",
       "      <td>-0.666667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                               score\n",
       "sentence                                                                                                                                                                                                                                                                                            \n",
       "'We must hope after so much prevarication that this time Google's proposals represent a genuine attempt to address the concerns identified,' said David Wood, the legal counsel for Icomp, an industry group backed by Microsoft and a number of other companies.                          -0.333333\n",
       "'We must push our leaders to step up and commit to action,' said Hugh Evans, the founder and chief executive of the charity.                                                                                                                                                               -0.285714\n",
       "'We need them to tell the story of how we are making decisions and putting the organization together,' said George Postolos, the Astros' president and chief executive, who added that the team would not want a broadcaster who was uncomfortable explaining the front office's strategy. -0.666667"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences = bossa.groupby(['sentence'])[['score']].aggregate(np.average)\n",
    "sentences.to_csv(\"sentiment/scores.csv\", encoding=\"utf8\")\n",
    "print(sentences.count())\n",
    "sentences[1001:1004]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "def majority(series): #receives a Pandas Series\n",
    "    return Counter(map(lambda x: 1 if x > 0 else -1 if x < 0 else 0, series)).most_common(1)[0][0]\n",
    "\n",
    "# score_calculate = np.average\n",
    "score_calculate = majority\n",
    "\n",
    "sentences = bossa.groupby(['sentence'])[['score']].aggregate(score_calculate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8996"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sentences)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sentence Classifier\n",
    "-------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "from sklearn.naive_bayes import MultinomialNB, BernoulliNB\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.svm import LinearSVC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create the tranining and testing sets (data and labels) from a randomized version of the set of assessed sentences."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sentence    8996\n",
       "score       8996\n",
       "dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences.reset_index().count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2415"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_scores = sentences.reset_index()\n",
    "scores = raw_scores\n",
    "len(scores[scores.score==0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We could consider 3 classes, but it toruns out that using binary classficication seems to produce better results. Still, try multi-classs classifiers is something worth trying."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/antonio/.virtualenvs/lexisnexis/lib/python2.7/site-packages/IPython/kernel/__main__.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "raw_scores = sentences.reset_index()\n",
    "scores = raw_scores\n",
    "scores = scores[scores.score!=0]  # We ignore the neutral sentences\n",
    "scores['sentiment'] = scores['score'].apply(lambda s: 'pos' if s > 0 else 'neg')\n",
    "# scores['sentiment'] = scores['score'].apply(lambda s: 'pos' if s > 0 else 'neg' if s < 0 else 0)\n",
    "\n",
    "# percentage = 0.85  #  percentage for training, rest for for testing\n",
    "# # We split to have enough representativenesss for both positive and negative sentiments\n",
    "# sent_min = min(\n",
    "#     scores[scores.sentiment=='pos'].sentiment.count(),\n",
    "#     scores[scores.sentiment=='neg'].sentiment.count(),\n",
    "# )\n",
    "# scores = scores[[\"sentence\", \"sentiment\"]]\n",
    "# train_data = np.array([])\n",
    "# train_labels = np.array([])\n",
    "# test_data = np.array([])\n",
    "# test_labels = np.array([])\n",
    "# for sent in ('pos', 'neg'):\n",
    "#     sent_scores = scores[scores['sentiment']==sent]\n",
    "#     sent_scores = sent_scores.reindex(np.random.permutation(sent_scores.index))\n",
    "#     sent_sentences_count = int(sent_scores['sentence'].count())\n",
    "#     sent_train = sent_scores[[\"sentence\", \"sentiment\"]][:int(sent_sentences_count * percentage)]\n",
    "#     sent_test = sent_scores[[\"sentence\", \"sentiment\"]][int(sent_sentences_count * percentage) + 1:]\n",
    "#     print sent, sent_min, sent_train.sentiment.count(), sent_test.sentiment.count()\n",
    "#     train_data = np.append(train_data, sent_train[\"sentence\"])\n",
    "#     train_labels = np.append(train_labels, sent_train[\"sentiment\"])\n",
    "#     test_data = np.append(test_data, sent_test[\"sentence\"])\n",
    "#     test_labels = np.append(test_labels, sent_test[\"sentiment\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6581"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "document_df = scores[['sentence', 'sentiment']]\n",
    "document_df = document_df.reindex(np.random.permutation(document_df.index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5484</th>\n",
       "      <td>Mr. Mandis left Goldman in 2004 to join a hedg...</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1005</th>\n",
       "      <td>'We originally thought that these contactless ...</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4691</th>\n",
       "      <td>JPMorgan Chase and Morgan Stanley are coordina...</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4819</th>\n",
       "      <td>Johnson &amp; Johnson is facing a wave of lawsuits...</td>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1332</th>\n",
       "      <td>A local builder, manager and owner has bought ...</td>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               sentence sentiment\n",
       "5484  Mr. Mandis left Goldman in 2004 to join a hedg...       pos\n",
       "1005  'We originally thought that these contactless ...       neg\n",
       "4691  JPMorgan Chase and Morgan Stanley are coordina...       pos\n",
       "4819  Johnson & Johnson is facing a wave of lawsuits...       neg\n",
       "1332  A local builder, manager and owner has bought ...       pos"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "document_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#SCIKIT-LEARN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>\n",
    "#         Olivier Grisel <olivier.grisel@ensta.org>\n",
    "#         Mathieu Blondel <mathieu@mblondel.org>\n",
    "#         Lars Buitinck <L.J.Buitinck@uva.nl>\n",
    "# License: BSD 3 clause\n",
    "\n",
    "from __future__ import print_function\n",
    "\n",
    "import logging\n",
    "import numpy as np\n",
    "from optparse import OptionParser\n",
    "import sys\n",
    "from time import time\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import HashingVectorizer\n",
    "from sklearn.feature_selection import SelectKBest, chi2\n",
    "from sklearn.linear_model import RidgeClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.linear_model import Perceptron\n",
    "from sklearn.linear_model import PassiveAggressiveClassifier\n",
    "from sklearn.naive_bayes import BernoulliNB, MultinomialNB, GaussianNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neighbors import NearestCentroid\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.utils.extmath import density\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Display progress logs on stdout\n",
    "logging.basicConfig(level=logging.INFO,\n",
    "                    format='%(asctime)s %(levelname)s %(message)s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading sentences for categories:\n",
      "['pos', 'neg']\n",
      "Data loaded:\n",
      "Train set: 5922 samples\n",
      "Test set: 659 samples\n"
     ]
    }
   ],
   "source": [
    "categories = [\n",
    "    'pos',\n",
    "    'neg',\n",
    "]\n",
    "print(\"Loading sentences for categories:\")\n",
    "print(categories)\n",
    "data_train, data_test = train_test_split(document_df, train_size=0.9, test_size=0.1, random_state=100)\n",
    "print('Data loaded:')\n",
    "print('Train set: {} samples'.format(len(data_train)))\n",
    "print('Test set: {} samples'.format(len(data_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6581"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(document_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pos    4388\n",
       "neg    2193\n",
       "dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "document_df['sentiment'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pos    3941\n",
       "neg    1981\n",
       "dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train['sentiment'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, ..., 1, 0, 0])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels = np.array(map(lambda x: 1 if x == 'pos' else 0, data_train['sentiment']))\n",
    "train_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0,\n",
       "       1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1,\n",
       "       1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1,\n",
       "       0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0,\n",
       "       1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0,\n",
       "       0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0,\n",
       "       1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1,\n",
       "       0, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0,\n",
       "       1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1,\n",
       "       0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0,\n",
       "       1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1,\n",
       "       1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1,\n",
       "       1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0,\n",
       "       1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0,\n",
       "       0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1,\n",
       "       1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0,\n",
       "       0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1,\n",
       "       0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0,\n",
       "       1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1,\n",
       "       0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1,\n",
       "       0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1,\n",
       "       1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0,\n",
       "       0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0,\n",
       "       1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_labels = np.array(map(lambda x: 1 if x == 'pos' else 0, data_test['sentiment']))\n",
    "test_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5922 documents - 1.078MB (training set)\n",
      "659 documents - 0.116MB (test set)\n",
      "2 categories\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def size_mb(docs):\n",
    "    return sum(len(s.encode('utf-8')) for s in docs) / 1e6\n",
    "\n",
    "data_train_size_mb = size_mb(data_train['sentence'])\n",
    "data_test_size_mb = size_mb(data_test['sentence'])\n",
    "\n",
    "print(\"%d documents - %0.3fMB (training set)\" % (\n",
    "    len(data_train), data_train_size_mb))\n",
    "print(\"%d documents - %0.3fMB (test set)\" % (\n",
    "    len(data_test), data_test_size_mb))\n",
    "print(\"%d categories\" % len(categories))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting features from the training data using a sparse vectorizer\n",
      "done in 0.262330s at 4.109MB/s\n",
      "n_samples: 5922, n_features: 18232\n",
      "\n",
      "Extracting features from the test data using the same vectorizer\n",
      "done in 0.023716s at 4.896MB/s\n",
      "n_samples: 659, n_features: 18232\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# split a training set and a test set\n",
    "y_train = train_labels\n",
    "y_test = test_labels\n",
    "\n",
    "print(\"Extracting features from the training data using a sparse vectorizer\")\n",
    "t0 = time()\n",
    "vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,\n",
    "                             stop_words='english')\n",
    "X_train = vectorizer.fit_transform(data_train.sentence.tolist())\n",
    "duration = time() - t0\n",
    "print(\"done in %fs at %0.3fMB/s\" % (duration, data_train_size_mb / duration))\n",
    "print(\"n_samples: %d, n_features: %d\" % X_train.shape)\n",
    "print()\n",
    "\n",
    "print(\"Extracting features from the test data using the same vectorizer\")\n",
    "t0 = time()\n",
    "X_test = vectorizer.transform(data_test.sentence.tolist())\n",
    "duration = time() - t0\n",
    "print(\"done in %fs at %0.3fMB/s\" % (duration, data_test_size_mb / duration))\n",
    "print(\"n_samples: %d, n_features: %d\" % X_test.shape)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting 100 best features by a chi-squared test\n",
      "done in 0.025049s\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# mapping from integer feature name to original token string\n",
    "feature_names = vectorizer.get_feature_names()\n",
    "opts_select_chi2 = 100\n",
    "\n",
    "print(\"Extracting %d best features by a chi-squared test\" %\n",
    "      opts_select_chi2)\n",
    "t0 = time()\n",
    "ch2 = SelectKBest(chi2, k=opts_select_chi2)\n",
    "X_train = ch2.fit_transform(X_train, y_train)\n",
    "X_test = ch2.transform(X_test)\n",
    "if feature_names:\n",
    "    # keep selected feature names\n",
    "    feature_names = [feature_names[i] for i\n",
    "                     in ch2.get_support(indices=True)]\n",
    "print(\"done in %fs\" % (time() - t0))\n",
    "print()\n",
    "\n",
    "feature_names = np.asarray(feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# feature_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def trim(s):\n",
    "    \"\"\"Trim string to fit on terminal (assuming 80-column display)\"\"\"\n",
    "    return s if len(s) < 80 else s[:75] + \"...\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train0 = X_train\n",
    "X_test0 = X_test\n",
    "\n",
    "###############################################################################\n",
    "# Benchmark classifiers\n",
    "def benchmark(clf):\n",
    "    print('_' * 80)\n",
    "    print(\"Training: \")\n",
    "    print(clf)\n",
    "    \n",
    "    if type(clf) == GaussianNB:\n",
    "        X_train = X_train0.todense()\n",
    "        X_test = X_test0.todense()\n",
    "    else:\n",
    "        X_train = X_train0\n",
    "        X_test = X_test0\n",
    "    \n",
    "    t0 = time()\n",
    "#     print(X_train)\n",
    "#     print(y_train)\n",
    "    clf.fit(X_train, y_train)\n",
    "    train_time = time() - t0\n",
    "    print(\"train time: %0.3fs\" % train_time)\n",
    "\n",
    "    t0 = time()\n",
    "    pred = clf.predict(X_test)\n",
    "    test_time = time() - t0\n",
    "    print(\"test time:  %0.3fs\" % test_time)\n",
    "\n",
    "    score = metrics.accuracy_score(y_test, pred)\n",
    "    print(\"accuracy:   %0.3f\" % score)\n",
    "\n",
    "#     if hasattr(clf, 'coef_'):\n",
    "#         print(\"dimensionality: %d\" % clf.coef_.shape[1])\n",
    "#         print(\"density: %f\" % density(clf.coef_))\n",
    "\n",
    "#         print('qqq',clf.coef_)\n",
    "        \n",
    "#         if feature_names is not None:\n",
    "#             print(\"top 10 keywords per class:\")\n",
    "#             for i, category in enumerate(categories):\n",
    "#                 print('>>>',i, category)\n",
    "#                 print('www',np.argsort(clf.coef_[i]))\n",
    "#                 top10 = np.argsort(clf.coef_[i])[-10:]\n",
    "#                 print(trim(\"%s: %s\"\n",
    "#                       % (category, \" \".join(feature_names[top10]))))\n",
    "#         print()\n",
    "\n",
    "#     print(\"classification report:\")\n",
    "#     print(metrics.classification_report(y_test, pred,\n",
    "#                                         target_names=categories))\n",
    "\n",
    "#     print(\"confusion matrix:\")\n",
    "#     print(metrics.confusion_matrix(y_test, pred))\n",
    "\n",
    "#     print()\n",
    "    clf_descr = str(clf).split('(')[0]\n",
    "    return clf_descr, score, train_time, test_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "Ridge Classifier\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "RidgeClassifier(alpha=1.0, class_weight=None, copy_X=True, fit_intercept=True,\n",
      "        max_iter=None, normalize=False, solver='lsqr', tol=0.01)\n",
      "train time: 0.003s\n",
      "test time:  0.000s\n",
      "accuracy:   0.689\n",
      "dimensionality: 1000\n",
      "density: 1.000000\n",
      "================================================================================\n",
      "Perceptron\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "Perceptron(alpha=0.0001, class_weight=None, eta0=1.0, fit_intercept=True,\n",
      "      n_iter=50, n_jobs=1, penalty=None, random_state=0, shuffle=True,\n",
      "      verbose=0, warm_start=False)\n",
      "train time: 0.013s\n",
      "test time:  0.000s\n",
      "accuracy:   0.639\n",
      "dimensionality: 1000\n",
      "density: 0.984000\n",
      "================================================================================\n",
      "Passive-Aggressive\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "PassiveAggressiveClassifier(C=1.0, fit_intercept=True, loss='hinge',\n",
      "              n_iter=50, n_jobs=1, random_state=None, shuffle=True,\n",
      "              verbose=0, warm_start=False)\n",
      "train time: 0.015s\n",
      "test time:  0.000s\n",
      "accuracy:   0.657\n",
      "dimensionality: 1000\n",
      "density: 1.000000\n",
      "================================================================================\n",
      "kNN\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n",
      "           metric_params=None, n_neighbors=10, p=2, weights='uniform')\n",
      "train time: 0.001s\n",
      "test time:  0.054s\n",
      "accuracy:   0.659\n",
      "================================================================================\n",
      "Random forest\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
      "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
      "            min_samples_leaf=1, min_samples_split=2,\n",
      "            min_weight_fraction_leaf=0.0, n_estimators=100, n_jobs=1,\n",
      "            oob_score=False, random_state=None, verbose=0,\n",
      "            warm_start=False)\n",
      "train time: 21.358s\n",
      "test time:  0.088s\n",
      "accuracy:   0.666\n",
      "================================================================================\n",
      "L2 penalty\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "LinearSVC(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='l2', max_iter=1000, multi_class='ovr',\n",
      "     penalty='l2', random_state=None, tol=0.001, verbose=0)\n",
      "train time: 0.022s\n",
      "test time:  0.000s\n",
      "accuracy:   0.692\n",
      "dimensionality: 1000\n",
      "density: 1.000000\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "SGDClassifier(alpha=0.0001, average=False, class_weight=None, epsilon=0.1,\n",
      "       eta0=0.0, fit_intercept=True, l1_ratio=0.15,\n",
      "       learning_rate='optimal', loss='hinge', n_iter=50, n_jobs=1,\n",
      "       penalty='l2', power_t=0.5, random_state=None, shuffle=True,\n",
      "       verbose=0, warm_start=False)\n",
      "train time: 0.027s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/antonio/.virtualenvs/lexisnexis/local/lib/python2.7/site-packages/sklearn/svm/classes.py:192: DeprecationWarning: loss='l2' has been deprecated in favor of loss='squared_hinge' as of 0.16. Backward compatibility for the loss='l2' will be removed in 1.0\n",
      "  DeprecationWarning)\n",
      "/home/antonio/.virtualenvs/lexisnexis/local/lib/python2.7/site-packages/sklearn/svm/classes.py:192: DeprecationWarning: loss='l2' has been deprecated in favor of loss='squared_hinge' as of 0.16. Backward compatibility for the loss='l2' will be removed in 1.0\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "test time:  0.000s\n",
      "accuracy:   0.683\n",
      "dimensionality: 1000\n",
      "density: 0.999000\n",
      "================================================================================\n",
      "L1 penalty\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "LinearSVC(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='l2', max_iter=1000, multi_class='ovr',\n",
      "     penalty='l1', random_state=None, tol=0.001, verbose=0)\n",
      "train time: 0.008s\n",
      "test time:  0.000s\n",
      "accuracy:   0.681\n",
      "dimensionality: 1000\n",
      "density: 0.773000\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "SGDClassifier(alpha=0.0001, average=False, class_weight=None, epsilon=0.1,\n",
      "       eta0=0.0, fit_intercept=True, l1_ratio=0.15,\n",
      "       learning_rate='optimal', loss='hinge', n_iter=50, n_jobs=1,\n",
      "       penalty='l1', power_t=0.5, random_state=None, shuffle=True,\n",
      "       verbose=0, warm_start=False)\n",
      "train time: 0.033s\n",
      "test time:  0.000s\n",
      "accuracy:   0.697\n",
      "dimensionality: 1000\n",
      "density: 0.638000\n",
      "================================================================================\n",
      "Elastic-Net penalty\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "SGDClassifier(alpha=0.0001, average=False, class_weight=None, epsilon=0.1,\n",
      "       eta0=0.0, fit_intercept=True, l1_ratio=0.15,\n",
      "       learning_rate='optimal', loss='hinge', n_iter=50, n_jobs=1,\n",
      "       penalty='elasticnet', power_t=0.5, random_state=None, shuffle=True,\n",
      "       verbose=0, warm_start=False)\n",
      "train time: 0.036s\n",
      "test time:  0.000s\n",
      "accuracy:   0.680\n",
      "dimensionality: 1000\n",
      "density: 0.996000\n",
      "================================================================================\n",
      "NearestCentroid (aka Rocchio classifier)\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "NearestCentroid(metric='euclidean', shrink_threshold=None)\n",
      "train time: 0.004s\n",
      "test time:  0.000s\n",
      "accuracy:   0.659\n",
      "================================================================================\n",
      "Naive Bayes\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True)\n",
      "train time: 0.003s\n",
      "test time:  0.000s\n",
      "accuracy:   0.662\n",
      "dimensionality: 1000\n",
      "density: 1.000000\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "BernoulliNB(alpha=0.01, binarize=0.0, class_prior=None, fit_prior=True)\n",
      "train time: 0.003s\n",
      "test time:  0.001s\n",
      "accuracy:   0.662\n",
      "dimensionality: 1000\n",
      "density: 1.000000\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "GaussianNB()\n",
      "train time: 0.080s\n",
      "test time:  0.011s\n",
      "accuracy:   0.630\n",
      "================================================================================\n",
      "LinearSVC with L1-based feature selection\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "Pipeline(steps=[('feature_selection', LinearSVC(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
      "     multi_class='ovr', penalty='l1', random_state=None, tol=0.001,\n",
      "     verbose=0)), ('classification', LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
      "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
      "     verbose=0))])\n",
      "train time: 0.024s\n",
      "test time:  0.001s\n",
      "accuracy:   0.684\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "for clf, name in (\n",
    "        (RidgeClassifier(tol=1e-2, solver=\"lsqr\"), \"Ridge Classifier\"),\n",
    "        (Perceptron(n_iter=50), \"Perceptron\"),\n",
    "        (PassiveAggressiveClassifier(n_iter=50), \"Passive-Aggressive\"),\n",
    "        (KNeighborsClassifier(n_neighbors=10), \"kNN\"),\n",
    "        (RandomForestClassifier(n_estimators=100), \"Random forest\")\n",
    "    ):\n",
    "    print('=' * 80)\n",
    "    print(name)\n",
    "    results.append(benchmark(clf))\n",
    "\n",
    "for penalty in [\"l2\", \"l1\"]:\n",
    "    print('=' * 80)\n",
    "    print(\"%s penalty\" % penalty.upper())\n",
    "    # Train Liblinear model\n",
    "    results.append(benchmark(LinearSVC(loss='l2', penalty=penalty,\n",
    "                                            dual=False, tol=1e-3)))\n",
    "\n",
    "    # Train SGD model\n",
    "    results.append(benchmark(SGDClassifier(alpha=.0001, n_iter=50,\n",
    "                                           penalty=penalty)))\n",
    "\n",
    "# Train SGD with Elastic Net penalty\n",
    "print('=' * 80)\n",
    "print(\"Elastic-Net penalty\")\n",
    "results.append(benchmark(SGDClassifier(alpha=.0001, n_iter=50,\n",
    "                                       penalty=\"elasticnet\")))\n",
    "\n",
    "# Train NearestCentroid without threshold\n",
    "print('=' * 80)\n",
    "print(\"NearestCentroid (aka Rocchio classifier)\")\n",
    "results.append(benchmark(NearestCentroid()))\n",
    "\n",
    "# Train sparse Naive Bayes classifiers\n",
    "print('=' * 80)\n",
    "print(\"Naive Bayes\")\n",
    "results.append(benchmark(MultinomialNB(alpha=.01)))\n",
    "results.append(benchmark(BernoulliNB(alpha=.01)))\n",
    "\n",
    "# Train Gaussian Naive Bayes classifiers\n",
    "results.append(benchmark(GaussianNB()))\n",
    "# print(\"Number of mislabeled points out of a total %d points : %d\" % (X_train.data.shape[0],(y_test != y_pred).sum()))\n",
    "\n",
    "\n",
    "print('=' * 80)\n",
    "print(\"LinearSVC with L1-based feature selection\")\n",
    "# The smaller C, the stronger the regularization.\n",
    "# The more regularization, the more sparsity.\n",
    "results.append(benchmark(Pipeline([\n",
    "  ('feature_selection', LinearSVC(penalty=\"l1\", dual=False, tol=1e-3)),\n",
    "  ('classification', LinearSVC())\n",
    "])))\n",
    "\n",
    "# make some plots\n",
    "\n",
    "# indices = np.arange(len(results))\n",
    "\n",
    "# results = [[x[i] for x in results] for i in range(4)]\n",
    "\n",
    "# clf_names, score, training_time, test_time = results\n",
    "# training_time = np.array(training_time) / np.max(training_time)\n",
    "# test_time = np.array(test_time) / np.max(test_time)\n",
    "\n",
    "# plt.figure(figsize=(12, 8))\n",
    "# plt.title(\"Score\")\n",
    "# plt.barh(indices, score, .2, label=\"score\", color='r')\n",
    "# plt.barh(indices + .3, training_time, .2, label=\"training time\", color='g')\n",
    "# plt.barh(indices + .6, test_time, .2, label=\"test time\", color='b')\n",
    "# plt.yticks(())\n",
    "# plt.legend(loc='best')\n",
    "# plt.subplots_adjust(left=.25)\n",
    "# plt.subplots_adjust(top=.95)\n",
    "# plt.subplots_adjust(bottom=.05)\n",
    "\n",
    "# for i, c in zip(indices, clf_names):\n",
    "#     plt.text(-.3, i, c)\n",
    "\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}