sentiment_raw_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preparing Data\n",
    "=============="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import cPickle as pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas import factorize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bossa/tasks_export.json  bossa/tasks_runs_export.json\r\n"
     ]
    }
   ],
   "source": [
    "!ls bossa/*json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "BOSSA Results\n",
    "-------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Processing `results_bossa.json` to get a *dictionary* with keys the task ids, and values in as the average value of the scores. To do that, we first convert scores from categorical (`neg`, `neu`, `pos`) to a numeric scale."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>result_id</th>\n",
       "      <th>seconds</th>\n",
       "      <th>task_id</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>11203</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>52775</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    result_id   seconds  task_id score\n",
       "50      11203  0.000025    52775     1"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bossa_results = pd.read_json(\"bossa/tasks_runs_export.json\")\n",
    "bossa_results.rename(columns={\"created\": \"start_time\", \"id\": \"result_id\", \"info\": \"score\"}, inplace=True)\n",
    "bossa_results[['start_time']]= bossa_results[['start_time']].apply(pd.to_datetime, dayfirst=True)\n",
    "bossa_results[['finish_time']]= bossa_results[['finish_time']].apply(pd.to_datetime, dayfirst=True)\n",
    "bossa_results['score'] = pd.Categorical(bossa_results['score'], categories=['vneg', 'neg', 'neu', 'pos', 'vpos'])\n",
    "bossa_results['score'].cat.rename_categories([-2, -1, 0, 1, 2], inplace=True)\n",
    "# Normalize everything to -1, 0, 1\n",
    "# bossa_results['score'] = bossa_results['score'].astype(float).apply(lambda x: -1 if x < 0 else 1 if x > 0 else 0)\n",
    "bossa_results[\"seconds\"] = (bossa_results[\"finish_time\"] - bossa_results[\"start_time\"]).astype('timedelta64[us]') / 1e6\n",
    "bossa_results = bossa_results[[\"result_id\", \"seconds\", \"task_id\", \"score\"]]\n",
    "bossa_results.ix[[50]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The information about the sentence comes in a dictionary inside the cells of the serie `info`, so we expand it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>task_id</th>\n",
       "      <th>info</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>52851</td>\n",
       "      <td>{u'search_words': u'founder', u'appears_in_sen...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    task_id                                               info\n",
       "50    52851  {u'search_words': u'founder', u'appears_in_sen..."
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bossa_tasks = pd.read_json(\"bossa/tasks_export.json\")\n",
    "bossa_tasks[['created']]= bossa_tasks[['created']].apply(pd.to_datetime, dayfirst=True)\n",
    "bossa_tasks.rename(columns={'id': 'task_id'}, inplace=True)\n",
    "bossa_tasks = bossa_tasks[['task_id', 'info']]\n",
    "bossa_tasks.ix[[50]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And finally we merge the `DataFrame` with the scores with the one containing the sentences."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>result_id</th>\n",
       "      <th>seconds</th>\n",
       "      <th>task_id</th>\n",
       "      <th>score</th>\n",
       "      <th>info</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>11195</td>\n",
       "      <td>0.000021</td>\n",
       "      <td>52776</td>\n",
       "      <td>2</td>\n",
       "      <td>{u'search_words': u'executive', u'appears_in_s...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    result_id   seconds  task_id  score  \\\n",
       "50      11195  0.000021    52776      2   \n",
       "\n",
       "                                                 info  \n",
       "50  {u'search_words': u'executive', u'appears_in_s...  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bossa_tasks_scores = pd.merge(bossa_results, bossa_tasks, on='task_id')\n",
    "bossa_tasks_scores.ix[[50]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's now expand the column `info` into as many new columns as keys has the dictionary `info`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'search_words',\n",
       " u'appears_in_sentence',\n",
       " u'url',\n",
       " u'media',\n",
       " u'appears_in_noun_phrases',\n",
       " u'noun_phrases',\n",
       " u'sentence_id',\n",
       " u'text',\n",
       " u'sentence',\n",
       " u'pub_date',\n",
       " u'is_company']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bossa_tasks_scores.ix[50].info.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>result_id</th>\n",
       "      <th>seconds</th>\n",
       "      <th>task_id</th>\n",
       "      <th>score</th>\n",
       "      <th>search_words</th>\n",
       "      <th>appears_in_sentence</th>\n",
       "      <th>url</th>\n",
       "      <th>media</th>\n",
       "      <th>appears_in_noun_phrases</th>\n",
       "      <th>noun_phrases</th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>text</th>\n",
       "      <th>sentence</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>is_company</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>11195</td>\n",
       "      <td>0.000021</td>\n",
       "      <td>52776</td>\n",
       "      <td>2</td>\n",
       "      <td>executive</td>\n",
       "      <td>0</td>\n",
       "      <td>http://dealbook.nytimes.com/2013/05/17/a-toeho...</td>\n",
       "      <td>nyt</td>\n",
       "      <td>0</td>\n",
       "      <td>[chinese investors, overseas companies, politi...</td>\n",
       "      <td>14</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>2013-05-17T11:47:51Z</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>11205</td>\n",
       "      <td>0.000018</td>\n",
       "      <td>52776</td>\n",
       "      <td>-1</td>\n",
       "      <td>executive</td>\n",
       "      <td>0</td>\n",
       "      <td>http://dealbook.nytimes.com/2013/05/17/a-toeho...</td>\n",
       "      <td>nyt</td>\n",
       "      <td>0</td>\n",
       "      <td>[chinese investors, overseas companies, politi...</td>\n",
       "      <td>14</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>2013-05-17T11:47:51Z</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>11207</td>\n",
       "      <td>0.000017</td>\n",
       "      <td>52776</td>\n",
       "      <td>1</td>\n",
       "      <td>executive</td>\n",
       "      <td>0</td>\n",
       "      <td>http://dealbook.nytimes.com/2013/05/17/a-toeho...</td>\n",
       "      <td>nyt</td>\n",
       "      <td>0</td>\n",
       "      <td>[chinese investors, overseas companies, politi...</td>\n",
       "      <td>14</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>2013-05-17T11:47:51Z</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>11209</td>\n",
       "      <td>0.000017</td>\n",
       "      <td>52776</td>\n",
       "      <td>-2</td>\n",
       "      <td>executive</td>\n",
       "      <td>0</td>\n",
       "      <td>http://dealbook.nytimes.com/2013/05/17/a-toeho...</td>\n",
       "      <td>nyt</td>\n",
       "      <td>0</td>\n",
       "      <td>[chinese investors, overseas companies, politi...</td>\n",
       "      <td>14</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>Chinese investors are increasingly opting to b...</td>\n",
       "      <td>2013-05-17T11:47:51Z</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    result_id   seconds  task_id  score search_words appears_in_sentence  \\\n",
       "50      11195  0.000021    52776      2    executive                   0   \n",
       "51      11205  0.000018    52776     -1    executive                   0   \n",
       "52      11207  0.000017    52776      1    executive                   0   \n",
       "53      11209  0.000017    52776     -2    executive                   0   \n",
       "\n",
       "                                                  url media  \\\n",
       "50  http://dealbook.nytimes.com/2013/05/17/a-toeho...   nyt   \n",
       "51  http://dealbook.nytimes.com/2013/05/17/a-toeho...   nyt   \n",
       "52  http://dealbook.nytimes.com/2013/05/17/a-toeho...   nyt   \n",
       "53  http://dealbook.nytimes.com/2013/05/17/a-toeho...   nyt   \n",
       "\n",
       "   appears_in_noun_phrases                                       noun_phrases  \\\n",
       "50                       0  [chinese investors, overseas companies, politi...   \n",
       "51                       0  [chinese investors, overseas companies, politi...   \n",
       "52                       0  [chinese investors, overseas companies, politi...   \n",
       "53                       0  [chinese investors, overseas companies, politi...   \n",
       "\n",
       "   sentence_id                                               text  \\\n",
       "50          14  Chinese investors are increasingly opting to b...   \n",
       "51          14  Chinese investors are increasingly opting to b...   \n",
       "52          14  Chinese investors are increasingly opting to b...   \n",
       "53          14  Chinese investors are increasingly opting to b...   \n",
       "\n",
       "                                             sentence              pub_date  \\\n",
       "50  Chinese investors are increasingly opting to b...  2013-05-17T11:47:51Z   \n",
       "51  Chinese investors are increasingly opting to b...  2013-05-17T11:47:51Z   \n",
       "52  Chinese investors are increasingly opting to b...  2013-05-17T11:47:51Z   \n",
       "53  Chinese investors are increasingly opting to b...  2013-05-17T11:47:51Z   \n",
       "\n",
       "   is_company  \n",
       "50          0  \n",
       "51          0  \n",
       "52          0  \n",
       "53          0  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def json_to_series(info):\n",
    "    keys, values = zip(*info.iteritems())\n",
    "    return pd.Series(values, index=keys)\n",
    "\n",
    "bossa_info = bossa_tasks_scores[\"info\"].apply(json_to_series)\n",
    "bossa_info.reset_index()\n",
    "bossa = pd.concat([bossa_tasks_scores, bossa_info], axis=1)\n",
    "bossa.pop(\"info\")\n",
    "# bossa['id'] = bossa['id'].astype(float)\n",
    "bossa.ix[50:53]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Aggregate\n",
    "---------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We now aggregate calculating the average per `sentence_id` using a group by. In the process, we lose the source of the data, that's why we first have to save it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bossa.to_csv(\"sentiment/scores_ungrouped.csv\", encoding=\"utf8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, we aggregate and create a new `DataFrame` for the different sentences and their score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score    8996\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sentence</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>'We must hope after so much prevarication that this time Google's proposals represent a genuine attempt to address the concerns identified,' said David Wood, the legal counsel for Icomp, an industry group backed by Microsoft and a number of other companies.</th>\n",
       "      <td>-0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>'We must push our leaders to step up and commit to action,' said Hugh Evans, the founder and chief executive of the charity.</th>\n",
       "      <td>-0.285714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>'We need them to tell the story of how we are making decisions and putting the organization together,' said George Postolos, the Astros' president and chief executive, who added that the team would not want a broadcaster who was uncomfortable explaining the front office's strategy.</th>\n",
       "      <td>-0.666667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                               score\n",
       "sentence                                                                                                                                                                                                                                                                                            \n",
       "'We must hope after so much prevarication that this time Google's proposals represent a genuine attempt to address the concerns identified,' said David Wood, the legal counsel for Icomp, an industry group backed by Microsoft and a number of other companies.                          -0.333333\n",
       "'We must push our leaders to step up and commit to action,' said Hugh Evans, the founder and chief executive of the charity.                                                                                                                                                               -0.285714\n",
       "'We need them to tell the story of how we are making decisions and putting the organization together,' said George Postolos, the Astros' president and chief executive, who added that the team would not want a broadcaster who was uncomfortable explaining the front office's strategy. -0.666667"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences = bossa.groupby(['sentence'])[['score']].aggregate(np.average)\n",
    "sentences.to_csv(\"sentiment/scores.csv\", encoding=\"utf8\")\n",
    "print sentences.count()\n",
    "sentences[1001:1004]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sentence Classifier\n",
    "-------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "from sklearn.cross_validation import cross_val_score\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "from sklearn.naive_bayes import MultinomialNB, BernoulliNB\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.svm import LinearSVC"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create the tranining and testing sets (data and labels) from a randomized version of the set of assessed sentences."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sentence    8996\n",
       "score       8996\n",
       "dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences.reset_index().count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We could consider 3 classes, but it toruns out that using binary classficication seems to produce better results. Still, try multi-classs classifiers is something worth trying."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'scores' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-20-1d78b1156d7b>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mscores\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mscores\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msentiment\u001b[0m\u001b[1;33m==\u001b[0m\u001b[1;34m'neg'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msentiment\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcount\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'scores' is not defined"
     ]
    }
   ],
   "source": [
    "scores[scores.sentiment=='neg'].sentiment.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.random.permutation([1,2,3,4,5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pos 2939 4281 755\n",
      "neg 2939 2498 440\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/antonio/.virtualenvs/lexisnexis/lib/python2.7/site-packages/IPython/kernel/__main__.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "raw_scores = sentences.reset_index()\n",
    "scores = raw_scores\n",
    "scores = scores[scores.score!=0]  # We ignore the neutral sentences\n",
    "scores['sentiment'] = scores['score'].apply(lambda s: 'pos' if s > 0 else 'neg')\n",
    "percentage = 0.85  #  percentage for training, rest for for testing\n",
    "# We split to have enough representativenesss for both positive and negative sentiments\n",
    "sent_min = min(\n",
    "    scores[scores.sentiment=='pos'].sentiment.count(),\n",
    "    scores[scores.sentiment=='neg'].sentiment.count(),\n",
    ")\n",
    "scores = scores[[\"sentence\", \"sentiment\"]]\n",
    "train_data = np.array([])\n",
    "train_labels = np.array([])\n",
    "test_data = np.array([])\n",
    "test_labels = np.array([])\n",
    "for sent in ('pos', 'neg'):\n",
    "    sent_scores = scores[scores['sentiment']==sent]\n",
    "    sent_scores = sent_scores.reindex(np.random.permutation(sent_scores.index))\n",
    "    sent_sentences_count = int(sent_scores['sentence'].count())\n",
    "    sent_train = sent_scores[[\"sentence\", \"sentiment\"]][:int(sent_sentences_count * percentage)]\n",
    "    sent_test = sent_scores[[\"sentence\", \"sentiment\"]][int(sent_sentences_count * percentage) + 1:]\n",
    "    print sent, sent_min, sent_train.sentiment.count(), sent_test.sentiment.count()\n",
    "    train_data = np.append(train_data, sent_train[\"sentence\"])\n",
    "    train_labels = np.append(train_labels, sent_train[\"sentiment\"])\n",
    "    test_data = np.append(test_data, sent_test[\"sentence\"])\n",
    "    test_labels = np.append(test_labels, sent_test[\"sentiment\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract feacture vectors from training data, and create and train a `MultinomialNB` classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sentence_clf = Pipeline([('vect', CountVectorizer()),\n",
    "                     ('tfidf', TfidfTransformer()),\n",
    "                     ('clf', MultinomialNB()),\n",
    "])\n",
    "sentence_clf = sentence_clf.fit(train_data, train_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluating the performance of the classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.64184100418410039"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# np.mean(sentence_clf.predict(test_data) == test_labels)\n",
    "sentence_clf.score(test_data, test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A performance of 64% is not very high, so we try now with a different pipeline using a `SGDClassifier` classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.63263598326359838"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgdc_clf = Pipeline([('vect', CountVectorizer()),\n",
    "                     ('tfidf', TfidfTransformer()),\n",
    "                     ('clf', SGDClassifier(loss='hinge', penalty='l2',\n",
    "                                           alpha=1e-3, n_iter=1000)),\n",
    "])\n",
    "sgdc_clf = sgdc_clf.fit(train_data, train_labels)\n",
    "sgdc_clf.score(test_data, test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Not an improve really. Let's try now a grid search to find the best combination of calssifier and parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performing grid search...\n",
      "('pipeline:', ['vect', 'tfidf', 'clf'])\n",
      "Fitting 3 folds for each of 384 candidates, totalling 1152 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 jobs       | elapsed:    0.5s\n",
      "[Parallel(n_jobs=-1)]: Done  50 jobs       | elapsed:   17.8s\n",
      "[Parallel(n_jobs=-1)]: Done 200 jobs       | elapsed:  1.3min\n",
      "[Parallel(n_jobs=-1)]: Done 450 jobs       | elapsed:  3.1min\n",
      "[Parallel(n_jobs=-1)]: Done 800 jobs       | elapsed:  5.5min\n",
      "[Parallel(n_jobs=-1)]: Done 1138 out of 1152 | elapsed:  7.8min remaining:    5.8s\n",
      "[Parallel(n_jobs=-1)]: Done 1152 out of 1152 | elapsed:  7.9min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done in 474.295s\n",
      "()\n",
      "Best score: 0.679\n",
      "Best parameters set:\n",
      "\tclf__alpha: 0.001\n",
      "\tclf__loss: 'modified_huber'\n",
      "\tclf__penalty: 'l2'\n",
      "\ttfidf__use_idf: True\n",
      "\tvect__ngram_range: (1, 1)\n",
      "\tvect__stop_words: None\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "from time import time\n",
    "\n",
    "pipeline = Pipeline([('vect', CountVectorizer()),\n",
    "                     ('tfidf', TfidfTransformer()),\n",
    "                     ('clf', SGDClassifier()),\n",
    "])\n",
    "parameters = {'vect__ngram_range': [(1, 1), (1, 2), (1, 3), (2, 3)],\n",
    "              'vect__stop_words': (None, stopwords.words('spanish')),\n",
    "              'tfidf__use_idf': (True, False),\n",
    "              'clf__alpha': (1e-1, 1e-2, 1e-3, 1e-4),\n",
    "              'clf__penalty': ('l1', 'l2', 'elasticnet'),\n",
    "              'clf__loss': ('log', 'modified_huber')  #, 'squared_hinge', 'perceptron', 'squared_loss', 'huber', 'epsilon_insensitive', 'squared_epsilon_insensitive')\n",
    "}\n",
    "parameters = {\n",
    "    'vect__max_df': (0.5, 0.75, 1.0),\n",
    "    'vect__max_features': (None, 5000, 10000, 50000),\n",
    "    'vect__ngram_range': ((1, 1), (1, 2), (1, 3), (2, 3)),  # unigrams or bigrams or trigrams\n",
    "    'vect__stop_words': (None, stopwords.words('english')),\n",
    "    'tfidf__use_idf': (True, False),\n",
    "    'tfidf__norm': ('l1', 'l2'),\n",
    "    'clf__alpha': (0.00001, 0.000001),\n",
    "    'clf__penalty': ('l1', 'l2', 'elasticnet'),\n",
    "    'clf__n_iter': (10, 50, 80),\n",
    "    'clf__loss': ('log', 'modified_huber'),\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1)\n",
    "print(\"Performing grid search...\")\n",
    "print(\"pipeline:\", [name for name, _ in pipeline.steps])\n",
    "#print(\"parameters:\")\n",
    "#pprint(parameters)\n",
    "t0 = time()\n",
    "grid_search.fit(train_data, train_labels)\n",
    "print(\"done in %0.3fs\" % (time() - t0))\n",
    "print()\n",
    "\n",
    "print(\"Best score: %0.3f\" % grid_search.best_score_)\n",
    "print(\"Best parameters set:\")\n",
    "best_parameters = grid_search.best_estimator_.get_params()\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'Pipeline' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-7d1def28e8fe>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtime\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m pipeline = Pipeline([('vect', CountVectorizer()),\n\u001b[0m\u001b[0;32m      5\u001b[0m                      \u001b[1;33m(\u001b[0m\u001b[1;34m'tfidf'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mTfidfTransformer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m                      \u001b[1;33m(\u001b[0m\u001b[1;34m'clf'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mSGDClassifier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'Pipeline' is not defined"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "from time import time\n",
    "\n",
    "pipeline = Pipeline([('vect', CountVectorizer()),\n",
    "                     ('tfidf', TfidfTransformer()),\n",
    "                     ('clf', SGDClassifier()),\n",
    "])\n",
    "parameters = {'vect__ngram_range': [(1, 1), (1, 2), (1, 3), (2, 3)],\n",
    "              'vect__stop_words': (None, stopwords.words('english')),\n",
    "              'tfidf__use_idf': (True, False),\n",
    "              'clf__alpha': (1e-1, 1e-2, 1e-3, 1e-4),\n",
    "              'clf__penalty': ('l1', 'l2', 'elasticnet'),\n",
    "              'clf__loss': ('log', 'modified_huber')  #, 'squared_hinge', 'perceptron', 'squared_loss', 'huber', 'epsilon_insensitive', 'squared_epsilon_insensitive')\n",
    "}\n",
    "parameters2 = {\n",
    "    'vect__max_df': (0.5, 0.75, 1.0),\n",
    "    'vect__max_features': (None, 5000, 10000, 50000),\n",
    "    'vect__ngram_range': ((1, 1), (1, 2), (1, 3), (2, 3)),  # unigrams or bigrams or trigrams\n",
    "    'vect__stop_words': (None, stopwords.words('english')),\n",
    "    'tfidf__use_idf': (True, False),\n",
    "    'tfidf__norm': ('l1', 'l2'),\n",
    "    'clf__alpha': (0.00001, 0.000001),\n",
    "    'clf__penalty': ('l1', 'l2', 'elasticnet'),\n",
    "    'clf__n_iter': (10, 50, 80),\n",
    "    'clf__loss': ('log', 'modified_huber'),\n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1)\n",
    "print(\"Performing grid search...\")\n",
    "print(\"pipeline:\", [name for name, _ in pipeline.steps])\n",
    "#print(\"parameters:\")\n",
    "#pprint(parameters)\n",
    "t0 = time()\n",
    "grid_search.fit(train_data, train_labels)\n",
    "print(\"done in %0.3fs\" % (time() - t0))\n",
    "print()\n",
    "\n",
    "print(\"Best score: %0.3f\" % grid_search.best_score_)\n",
    "print(\"Best parameters set:\")\n",
    "best_parameters = grid_search.best_estimator_.get_params()\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clf__alpha: 0.0001\n",
      "clf__loss: 'log'\n",
      "clf__penalty: 'l2'\n",
      "tfidf__use_idf: True\n",
      "vect__ngram_range: (1, 1)\n",
      "vect__stop_words: None\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.66956778285882879"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parameters = {'vect__ngram_range': [(1, 1), (1, 2), (1, 3), (2, 3)],\n",
    "              'vect__stop_words': (None, stopwords.words('spanish')),\n",
    "              'tfidf__use_idf': (True, False),\n",
    "              'clf__alpha': (1e-1, 1e-2, 1e-3, 1e-4),\n",
    "              'clf__penalty': ('l1', 'l2', 'elasticnet'),\n",
    "              'clf__loss': ('log', 'modified_huber')  #, 'squared_hinge', 'perceptron', 'squared_loss', 'huber', 'epsilon_insensitive', 'squared_epsilon_insensitive')\n",
    "}\n",
    "gs_clf = GridSearchCV(sgdc_clf, parameters, n_jobs=-1)\n",
    "gs_clf = gs_clf.fit(train_data, train_labels)\n",
    "best_parameters, score, _ = max(gs_clf.grid_scores_, key=lambda x: x[1])\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print(\"%s: %r\" % (param_name, best_parameters[param_name]))\n",
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set the best parameters for `SDGClassifier` and run a cross-validation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scores: [ 0.67133382  0.6740413   0.67625369  0.67601476  0.6797048 ]  mean: 0.675470  std: 0.000000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.69790794979079496"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgdc_clf.set_params(**best_parameters)\n",
    "sgdc_clf = sgdc_clf.fit(train_data, train_labels)\n",
    "scores =  cross_val_score(sgdc_clf, train_data, train_labels, cv=5)\n",
    "print(\"scores: %s  mean: %f  std: %f\" % (str(scores), np.mean(scores), np.std(score)))\n",
    "sgdc_clf.score(test_data, test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's save it for later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scores: [ 0.67133382  0.6740413   0.67625369  0.67601476  0.6797048 ]  mean: 0.675470  std: 0.002757\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.69790794979079496"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgdc_scores =  cross_val_score(sgdc_clf, train_data, train_labels, cv=5)\n",
    "print(\"scores: %s  mean: %f  std: %f\" % (str(sgdc_scores), np.mean(sgdc_scores), np.std(sgdc_scores)))\n",
    "with open(\"sentiment/sgdc_clf.pickle\", \"wb\") as sgdc_file:\n",
    "    pickle.dump(sgdc_clf, sgdc_file)\n",
    "sgdc_clf.score(test_data, test_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's take a look to the precision, recall and F-score of the classifier."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 0.6803653 ,  0.70184426]),\n",
       " array([ 0.33863636,  0.90728477]),\n",
       " array([ 0.4522003 ,  0.79145003]))"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision, recall, fscore, support = precision_recall_fscore_support(test_labels, sgdc_clf.predict(test_data))\n",
    "precision, recall, fscore"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Try several more classifiers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SGDClassifier\n",
      "\tscores: [ 0.36845984  0.36873156  0.36873156  0.36826568  0.36826568]  mean: 0.368491  std: 0.000000\n",
      "\tscore: 0.36820083682\n",
      "LinearSVC\n",
      "\tscores: [ 0.66764923  0.6600295   0.65634218  0.66715867  0.66568266]  mean: 0.663372  std: 0.000000\n",
      "\tscore: 0.684518828452\n",
      "MultinomialNB\n",
      "\tscores: [ 0.64112012  0.6379056   0.6379056   0.64280443  0.6398524 ]  mean: 0.639918  std: 0.000000\n",
      "\tscore: 0.638493723849\n",
      "BernoulliNB\n",
      "\tscores: [ 0.66322771  0.67182891  0.66666667  0.66346863  0.67232472]  mean: 0.667503  std: 0.000000\n",
      "\tscore: 0.680334728033\n"
     ]
    }
   ],
   "source": [
    "for classifier in [SGDClassifier, LinearSVC, MultinomialNB, BernoulliNB]:\n",
    "    sentence_clf = Pipeline([('vect', CountVectorizer()),\n",
    "                         ('tfidf', TfidfTransformer()),\n",
    "                         ('clf', classifier()),\n",
    "    ])\n",
    "    sentence_clf = sentence_clf.fit(train_data, train_labels)\n",
    "    scores =  cross_val_score(sentence_clf, train_data, train_labels, cv=5)\n",
    "    print(classifier.__name__)\n",
    "    print(\"\\tscores: %s  mean: %f  std: %f\" % (str(scores), np.mean(scores), np.std(score)))\n",
    "    print(\"\\tscore: %s\" % sentence_clf.score(test_data, test_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's run `GridSeachCV` to find the best parameters for the ones showing better socres. Staring from `LinearSVC`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.687866108787\n",
      "clf__C: 1\n",
      "clf__loss: 'l1'\n",
      "tfidf__smooth_idf: False\n",
      "tfidf__use_idf: True\n",
      "vect__ngram_range: (1, 1)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.67001032600678567"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_clf = Pipeline([('vect', CountVectorizer(strip_accents='unicode', decode_error='replace', stop_words=stopwords.words('spanish'))),\n",
    "                     ('tfidf', TfidfTransformer(use_idf=False, smooth_idf=True)),\n",
    "                     ('clf', LinearSVC(C=1.0, loss='l1')),\n",
    "])\n",
    "sentence_clf = sentence_clf.fit(train_data, train_labels)\n",
    "print sentence_clf.score(test_data, test_labels)\n",
    "parameters = {'vect__ngram_range': [(1, 1), (1, 2), (1, 3), (2, 3), (1, 4), (2, 4), (3, 4), (1, 5), (2, 5), (3, 5), (4, 5)],\n",
    "              'tfidf__use_idf': (True, False),\n",
    "              'tfidf__smooth_idf': (True, False),\n",
    "              'clf__C': (10, 1, 1e-1, 1e-2),\n",
    "              'clf__loss': ('l1', 'l2'),\n",
    "}\n",
    "gs_clf = GridSearchCV(sentence_clf, parameters, n_jobs=-1)\n",
    "gs_clf = gs_clf.fit(train_data, train_labels)\n",
    "best_parameters, score, _ = max(gs_clf.grid_scores_, key=lambda x: x[1])\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print(u\"%s: %r\" % (param_name, best_parameters[param_name]))\n",
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It seems like `MultinomialNB` is also good, let's see the search grid."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.636820083682\n",
      "clf__alpha: 0.1\n",
      "tfidf__smooth_idf: True\n",
      "tfidf__use_idf: True\n",
      "vect__ngram_range: (1, 1)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.66956778285882879"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_clf = Pipeline([('vect', CountVectorizer(strip_accents='unicode', decode_error='replace', stop_words=stopwords.words('spanish'))),\n",
    "                     ('tfidf', TfidfTransformer()),\n",
    "                     ('clf', MultinomialNB()),\n",
    "])\n",
    "sentence_clf = sentence_clf.fit(train_data, train_labels)\n",
    "print sentence_clf.score(test_data, test_labels)\n",
    "parameters = {'vect__ngram_range': [(1, 1), (1, 2), (1, 3), (2, 3), (1, 4), (2, 4), (3, 4), (1, 5), (2, 5), (3, 5), (4, 5)],\n",
    "              'tfidf__use_idf': (True, False),\n",
    "              'tfidf__smooth_idf': (True, False),\n",
    "              'clf__alpha': (10, 1, 1e-1, 1e-2, 1e-3),\n",
    "}\n",
    "gs_clf = GridSearchCV(sentence_clf, parameters, n_jobs=-1)\n",
    "gs_clf = gs_clf.fit(train_data, train_labels)\n",
    "best_parameters, score, _ = max(gs_clf.grid_scores_, key=lambda x: x[1])\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print(u\"%s: %r\" % (param_name, best_parameters[param_name]))\n",
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We create the two classifiers, run cross-validation, and save them as pickled objects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scores: [ 0.6705969   0.66961652  0.67330383  0.66273063  0.66568266]  mean: 0.668386  std: 0.003738\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.67029288702928869"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "multinb1_clf = Pipeline([('vect', CountVectorizer(strip_accents='unicode', decode_error='replace', stop_words=stopwords.words('spanish'))),\n",
    "                     ('tfidf', TfidfTransformer()),\n",
    "                     ('clf', MultinomialNB()),\n",
    "])\n",
    "multinb2_clf = multinb1_clf.set_params(**best_parameters)\n",
    "multinb_clf = multinb1_clf.fit(train_data, train_labels)\n",
    "multinb_scores =  cross_val_score(multinb_clf, train_data, train_labels, cv=5)\n",
    "print(\"scores: %s  mean: %f  std: %f\" % (str(multinb_scores), np.mean(multinb_scores), np.std(multinb_scores)))\n",
    "with open(\"sentiment/multinb_clf.pickle\", \"wb\") as multinb_file:\n",
    "    pickle.dump(multinb_clf, multinb_file)\n",
    "multinb_clf.score(test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 0.98076042,  0.92232376]),\n",
       " array([ 0.85708567,  0.99018921]),\n",
       " array([ 0.9147618 ,  0.95505238]))"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision, recall, fscore, support = precision_recall_fscore_support(train_labels, multinb_clf.predict(train_data))\n",
    "precision, recall, fscore"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Doing the same for `LinearSVC`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scores: [ 0.66617539  0.66519174  0.65486726  0.67158672  0.66568266]  mean: 0.664701  std: 0.005431\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.67866108786610879"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linear1_clf = Pipeline([('vect', CountVectorizer(strip_accents='unicode', decode_error='replace', stop_words=stopwords.words('spanish'))),\n",
    "                     ('tfidf', TfidfTransformer()),\n",
    "                     ('clf', LinearSVC()),\n",
    "])\n",
    "clf__alpha = best_parameters.pop(\"clf__alpha\")\n",
    "linear2_clf = linear1_clf.set_params(**best_parameters)\n",
    "linear_clf = linear2_clf.fit(train_data, train_labels)\n",
    "linear_scores =  cross_val_score(linear_clf, train_data, train_labels, cv=5)\n",
    "print(\"scores: %s  mean: %f  std: %f\" % (str(linear_scores), np.mean(linear_scores), np.std(linear_scores)))\n",
    "with open(\"sentiment/linear_clf.pickle\", \"wb\") as linear_file:\n",
    "    pickle.dump(linear_clf, linear_file)\n",
    "linear_clf.score(test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 0.98076042,  0.92232376]),\n",
       " array([ 0.85708567,  0.99018921]),\n",
       " array([ 0.9147618 ,  0.95505238]))"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision, recall, fscore, support = precision_recall_fscore_support(train_labels, multinb_clf.predict(train_data))\n",
    "precision, recall, fscore"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's try with `BernoulliNB`, the one that showed the best results so far."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.680334728033\n",
      "clf__alpha: 1\n",
      "tfidf__smooth_idf: True\n",
      "tfidf__use_idf: True\n",
      "vect__ngram_range: (1, 1)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.66337217878743182"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence_clf = Pipeline([('vect', CountVectorizer(strip_accents='unicode', decode_error='replace', stop_words=stopwords.words('spanish'))),\n",
    "                     ('tfidf', TfidfTransformer()),\n",
    "                     ('clf', BernoulliNB()),\n",
    "])\n",
    "sentence_clf = sentence_clf.fit(train_data, train_labels)\n",
    "print sentence_clf.score(test_data, test_labels)\n",
    "parameters = {'vect__ngram_range': [(1, 1), (1, 2), (1, 3), (2, 3), (1, 4), (2, 4), (3, 4), (1, 5), (2, 5), (3, 5), (4, 5)],\n",
    "              'tfidf__use_idf': (True, False),\n",
    "              'tfidf__smooth_idf': (True, False),\n",
    "              'clf__alpha': (10, 1, 1e-1, 1e-2, 1e-3),\n",
    "}\n",
    "gs_clf = GridSearchCV(sentence_clf, parameters, n_jobs=-1)\n",
    "gs_clf = gs_clf.fit(train_data, train_labels)\n",
    "best_parameters, score, _ = max(gs_clf.grid_scores_, key=lambda x: x[1])\n",
    "for param_name in sorted(parameters.keys()):\n",
    "    print(u\"%s: %r\" % (param_name, best_parameters[param_name]))\n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scores: [ 0.6705969   0.66961652  0.67330383  0.66273063  0.66568266]  mean: 0.668386  std: 0.003738\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.67029288702928869"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bernoulli1_clf = Pipeline([('vect', CountVectorizer(strip_accents='unicode', decode_error='replace', stop_words=stopwords.words('spanish'))),\n",
    "                     ('tfidf', TfidfTransformer()),\n",
    "                     ('clf', BernoulliNB()),\n",
    "])\n",
    "bernoulli2_clf = bernoulli1_clf.set_params(**best_parameters)\n",
    "bernoulli_clf = bernoulli2_clf.fit(train_data, train_labels)\n",
    "multinb_scores =  cross_val_score(multinb_clf, train_data, train_labels, cv=5)\n",
    "print(\"scores: %s  mean: %f  std: %f\" % (str(multinb_scores), np.mean(multinb_scores), np.std(multinb_scores)))\n",
    "with open(\"sentiment/bernoulli_clf.pickle\", \"wb\") as multinb_file:\n",
    "    pickle.dump(multinb_clf, multinb_file)\n",
    "multinb_clf.score(test_data, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 0.98076042,  0.92232376]),\n",
       " array([ 0.85708567,  0.99018921]),\n",
       " array([ 0.9147618 ,  0.95505238]))"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision, recall, fscore, support = precision_recall_fscore_support(train_labels, multinb_clf.predict(train_data))\n",
    "precision, recall, fscore"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}