nytimes_csv.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#CSV builder for The New York Times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# import logging\n",
    "# import math\n",
    "# import os\n",
    "# from collections import defaultdict\n",
    "# from datetime import timedelta, date, datetime\n",
    "# from dateutil import parser\n",
    "# from time import sleep, time\n",
    "\n",
    "# import requests\n",
    "# import yaml\n",
    "# from joblib import Parallel, delayed\n",
    "# from pymongo import MongoClient\n",
    "# from pymongo.errors import BulkWriteError, DuplicateKeyError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from blaze import Data, DataFrame, by, join, merge\n",
    "from nltk.tokenize import sent_tokenize\n",
    "from pymongo import MongoClient\n",
    "from slugify import slugify"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##MongoDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "client = MongoClient()\n",
    "db = client.nytimes3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for d in db.articles.find():\n",
    "#     if len(d['q_info']) > 1:\n",
    "#         k = list(d['q_info'].keys())[0]\n",
    "#         if len(d['q_info'][k]) > 1:\n",
    "#             print(d['_id'])\n",
    "#             print(d['q_info'])\n",
    "#             break\n",
    "\n",
    "# 52c8a3eb38f0d862ec32236f\n",
    "# {\n",
    "# 'amazon_com': [{\n",
    "#     'q': {'page': 0, 'end_date': '20140228', 'term': '\"amazon.com\"', 'begin_date': '20140101', 'api_key': '3439a9084efa80c4f5fb1d290dfc1b44:11:70233981'},\n",
    "#     'term_category': 3,\n",
    "#     'snippet': 'Jeff Bezos, vacationing in the Galapagos Islands this week, felt some serious pain. And it wasn’t just because UPS had failed to make all of Amazon’s shipments by Christmas.'\n",
    "#     }, {\n",
    "#     'q': {'page': 0, 'begin_date': '20140101', 'end_date': '20140228', 'term': '\"amazon\"', 'api_key': 'a5c709f3168b829711241b243457e9d6:13:70235641'},\n",
    "#     'term_category': 2,\n",
    "#     'snippet': 'The Ecuadorean Navy offered Jeff Bezos same-day shipping on an <strong>Amazon</strong> Prime package: Mr. Bezos himself. Amazon’s chief executive suffered a kidney stone attack visiting the Galapagos Islands this week, according to the local'\n",
    "#     }],\n",
    "# 'executive': [{\n",
    "#     'q': {'page': 10, 'end_date': '20140116', 'term': '\"executive\"', 'begin_date': '20140101', 'api_key': 'a5c709f3168b829711241b243457e9d6:13:70235641'},\n",
    "#     'term_category': 1,\n",
    "#     'snippet': 'The Ecuadorean Navy offered Jeff Bezos same-day shipping on an Amazon Prime package: Mr. Bezos himself. Amazon’s chief <strong>executive</strong> suffered a kidney stone attack visiting the Galapagos Islands this week, according to the local'\n",
    "#     }]\n",
    "# }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# db.articles.find_one({'_id':'52c8a3eb38f0d862ec32236f'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "total_rows_df = DataFrame()\n",
    "for doc in db.articles.find():\n",
    "    rows_df = DataFrame()\n",
    "    \n",
    "    # Texts\n",
    "    common_texts = []\n",
    "    if doc['abstract']:\n",
    "        common_texts.append(doc['abstract'])\n",
    "    if doc['headline'] and doc['headline']['main']:\n",
    "        common_texts.append(doc['headline']['main'])\n",
    "    if doc['lead_paragraph']:\n",
    "        common_texts.append(doc['lead_paragraph'])\n",
    "    # add snippet as variable field\n",
    "    \n",
    "    # Fix fields\n",
    "#     row_id = 'row_id'\n",
    "    article_id = doc['_id']\n",
    "    pub_date = doc['pub_date']\n",
    "    section_name = doc['section_name']\n",
    "    web_url = doc['web_url']\n",
    "    \n",
    "    # Variable fields\n",
    "    for term in doc['q_info']:\n",
    "        texts = list(common_texts)\n",
    "        q_info = doc['q_info'][term]\n",
    "        term_category = q_info[0]['term_category']\n",
    "        search_terms = []\n",
    "        for info in q_info:\n",
    "            search_terms.append(info['q']['term'])\n",
    "            if info['snippet']:\n",
    "                texts.append(info['snippet'])\n",
    "        \n",
    "        for text in texts:\n",
    "            sentences = sent_tokenize(text)\n",
    "            for sentence in sentences:\n",
    "                if any([slugify(search_term) in slugify(sentence) for search_term in search_terms]):\n",
    "                    row_df = DataFrame(\n",
    "                        [[article_id, pub_date, section_name, web_url, term_category, term, sentence]],\n",
    "                        columns=['article_id', 'pub_date', 'section_name', 'web_url', 'term_category', 'term', 'sentence']\n",
    "                    )\n",
    "                    rows_df = rows_df.append(row_df)\n",
    "    \n",
    "    total_rows_df = total_rows_df.append(rows_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "total_rows_df['term'] = total_rows_df['term'].apply(lambda x: x.replace('_', '.'))\n",
    "total_rows_df['sentence'] = total_rows_df['sentence'].apply(lambda x: x.replace('<strong>', '').replace('</strong>', ''))\n",
    "total_rows_df['sentence'] = total_rows_df['sentence'].apply(lambda x: x.replace(',', '')) # csv delimiter problems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "total_rows_df.to_csv('total_rows.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "total_rows_data = Data('total_rows.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "total_rows_distinct = total_rows_data.distinct()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_id</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>section_name</th>\n",
       "      <th>web_url</th>\n",
       "      <th>term_category</th>\n",
       "      <th>term</th>\n",
       "      <th>sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>52ce1f2c38f0d81799290366</td>\n",
       "      <td>2014-01-01 00:00:00</td>\n",
       "      <td>Paid Death Notices</td>\n",
       "      <td>http://query.nytimes.com/gst/fullpage.html?res...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>An entrepreneur and connoisseur she was also p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>52ce1f2c38f0d81799290366</td>\n",
       "      <td>2014-01-01 00:00:00</td>\n",
       "      <td>Paid Death Notices</td>\n",
       "      <td>http://query.nytimes.com/gst/fullpage.html?res...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>An entrepreneur and connoisseur she was also p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>52c4045238f0d82e35d8e45c</td>\n",
       "      <td>2014-01-01 07:00:45</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>http://boss.blogs.nytimes.com/2014/01/01/the-b...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>Colleen DeBaise — and the Story Exchange — bro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>52c491cb38f0d82d0ea3260c</td>\n",
       "      <td>2014-01-01 17:08:02</td>\n",
       "      <td>U.S.</td>\n",
       "      <td>http://www.nytimes.com/video/us/10000000262887...</td>\n",
       "      <td>1</td>\n",
       "      <td>executive</td>\n",
       "      <td>Mike Duggan a former hospital executive and pr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>52c491cb38f0d82d0ea3260c</td>\n",
       "      <td>2014-01-01 17:08:02</td>\n",
       "      <td>U.S.</td>\n",
       "      <td>http://www.nytimes.com/video/us/10000000262887...</td>\n",
       "      <td>1</td>\n",
       "      <td>executive</td>\n",
       "      <td>Mike Duggan a former hospital executive and pr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>52e0941e38f0d87ee88528f8</td>\n",
       "      <td>2014-01-15 00:00:00</td>\n",
       "      <td>Paid Death Notices</td>\n",
       "      <td>http://query.nytimes.com/gst/fullpage.html?res...</td>\n",
       "      <td>1</td>\n",
       "      <td>executive</td>\n",
       "      <td>Alisa R. Doctoroff President; Linda Mirels Cha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>52e0941e38f0d87ee88528f8</td>\n",
       "      <td>2014-01-15 00:00:00</td>\n",
       "      <td>Paid Death Notices</td>\n",
       "      <td>http://query.nytimes.com/gst/fullpage.html?res...</td>\n",
       "      <td>1</td>\n",
       "      <td>executive</td>\n",
       "      <td>Alisa R. Doctoroff President; Linda Mirels Cha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>52c4b55038f0d82d0ea32638</td>\n",
       "      <td>2014-01-02 00:00:00</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>http://www.nytimes.com/2014/01/02/business/hig...</td>\n",
       "      <td>1</td>\n",
       "      <td>executive</td>\n",
       "      <td>analysts and industry executives say.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>52c4b55038f0d82d0ea32638</td>\n",
       "      <td>2014-01-02 00:00:00</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>http://www.nytimes.com/2014/01/02/business/hig...</td>\n",
       "      <td>1</td>\n",
       "      <td>executive</td>\n",
       "      <td>“People were pulling back when they had to let...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>52c4b55038f0d82d0ea32638</td>\n",
       "      <td>2014-01-02 00:00:00</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>http://www.nytimes.com/2014/01/02/business/hig...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>DETROIT — Matt Hlavin an entrepreneur in Cleve...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>52c8a15c38f0d862ec32236d</td>\n",
       "      <td>2014-01-05 00:00:00</td>\n",
       "      <td>Real Estate</td>\n",
       "      <td>http://www.nytimes.com/2014/01/05/realestate/r...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>to the thrift and simplicity of our grandfathe...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "                  article_id            pub_date        section_name  \\\n",
       "0   52ce1f2c38f0d81799290366 2014-01-01 00:00:00  Paid Death Notices   \n",
       "1   52ce1f2c38f0d81799290366 2014-01-01 00:00:00  Paid Death Notices   \n",
       "2   52c4045238f0d82e35d8e45c 2014-01-01 07:00:45        Business Day   \n",
       "3   52c491cb38f0d82d0ea3260c 2014-01-01 17:08:02                U.S.   \n",
       "4   52c491cb38f0d82d0ea3260c 2014-01-01 17:08:02                U.S.   \n",
       "5   52e0941e38f0d87ee88528f8 2014-01-15 00:00:00  Paid Death Notices   \n",
       "6   52e0941e38f0d87ee88528f8 2014-01-15 00:00:00  Paid Death Notices   \n",
       "7   52c4b55038f0d82d0ea32638 2014-01-02 00:00:00        Business Day   \n",
       "8   52c4b55038f0d82d0ea32638 2014-01-02 00:00:00        Business Day   \n",
       "9   52c4b55038f0d82d0ea32638 2014-01-02 00:00:00        Business Day   \n",
       "10  52c8a15c38f0d862ec32236d 2014-01-05 00:00:00         Real Estate   \n",
       "\n",
       "                                              web_url  term_category  \\\n",
       "0   http://query.nytimes.com/gst/fullpage.html?res...              1   \n",
       "1   http://query.nytimes.com/gst/fullpage.html?res...              1   \n",
       "2   http://boss.blogs.nytimes.com/2014/01/01/the-b...              1   \n",
       "3   http://www.nytimes.com/video/us/10000000262887...              1   \n",
       "4   http://www.nytimes.com/video/us/10000000262887...              1   \n",
       "5   http://query.nytimes.com/gst/fullpage.html?res...              1   \n",
       "6   http://query.nytimes.com/gst/fullpage.html?res...              1   \n",
       "7   http://www.nytimes.com/2014/01/02/business/hig...              1   \n",
       "8   http://www.nytimes.com/2014/01/02/business/hig...              1   \n",
       "9   http://www.nytimes.com/2014/01/02/business/hig...              1   \n",
       "10  http://www.nytimes.com/2014/01/05/realestate/r...              1   \n",
       "\n",
       "            term                                           sentence  \n",
       "0   entrepreneur  An entrepreneur and connoisseur she was also p...  \n",
       "1   entrepreneur  An entrepreneur and connoisseur she was also p...  \n",
       "2   entrepreneur  Colleen DeBaise — and the Story Exchange — bro...  \n",
       "3      executive  Mike Duggan a former hospital executive and pr...  \n",
       "4      executive  Mike Duggan a former hospital executive and pr...  \n",
       "5      executive  Alisa R. Doctoroff President; Linda Mirels Cha...  \n",
       "6      executive  Alisa R. Doctoroff President; Linda Mirels Cha...  \n",
       "7      executive              analysts and industry executives say.  \n",
       "8      executive  “People were pulling back when they had to let...  \n",
       "9   entrepreneur  DETROIT — Matt Hlavin an entrepreneur in Cleve...  \n",
       "..."
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_rows_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_id</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>section_name</th>\n",
       "      <th>web_url</th>\n",
       "      <th>term_category</th>\n",
       "      <th>term</th>\n",
       "      <th>sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>52ce1f2c38f0d81799290366</td>\n",
       "      <td>2014-01-01 00:00:00</td>\n",
       "      <td>Paid Death Notices</td>\n",
       "      <td>http://query.nytimes.com/gst/fullpage.html?res...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>An entrepreneur and connoisseur she was also p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>52c4045238f0d82e35d8e45c</td>\n",
       "      <td>2014-01-01 07:00:45</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>http://boss.blogs.nytimes.com/2014/01/01/the-b...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>Colleen DeBaise — and the Story Exchange — bro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>52c491cb38f0d82d0ea3260c</td>\n",
       "      <td>2014-01-01 17:08:02</td>\n",
       "      <td>U.S.</td>\n",
       "      <td>http://www.nytimes.com/video/us/10000000262887...</td>\n",
       "      <td>1</td>\n",
       "      <td>executive</td>\n",
       "      <td>Mike Duggan a former hospital executive and pr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>52e0941e38f0d87ee88528f8</td>\n",
       "      <td>2014-01-15 00:00:00</td>\n",
       "      <td>Paid Death Notices</td>\n",
       "      <td>http://query.nytimes.com/gst/fullpage.html?res...</td>\n",
       "      <td>1</td>\n",
       "      <td>executive</td>\n",
       "      <td>Alisa R. Doctoroff President; Linda Mirels Cha...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>52c4b55038f0d82d0ea32638</td>\n",
       "      <td>2014-01-02 00:00:00</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>http://www.nytimes.com/2014/01/02/business/hig...</td>\n",
       "      <td>1</td>\n",
       "      <td>executive</td>\n",
       "      <td>analysts and industry executives say.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>52c4b55038f0d82d0ea32638</td>\n",
       "      <td>2014-01-02 00:00:00</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>http://www.nytimes.com/2014/01/02/business/hig...</td>\n",
       "      <td>1</td>\n",
       "      <td>executive</td>\n",
       "      <td>“People were pulling back when they had to let...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>52c4b55038f0d82d0ea32638</td>\n",
       "      <td>2014-01-02 00:00:00</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>http://www.nytimes.com/2014/01/02/business/hig...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>DETROIT — Matt Hlavin an entrepreneur in Cleve...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>52c8a15c38f0d862ec32236d</td>\n",
       "      <td>2014-01-05 00:00:00</td>\n",
       "      <td>Real Estate</td>\n",
       "      <td>http://www.nytimes.com/2014/01/05/realestate/r...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>to the thrift and simplicity of our grandfathe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>52c6e22438f0d82ae415dd89</td>\n",
       "      <td>2014-01-05 00:00:00</td>\n",
       "      <td>Books</td>\n",
       "      <td>http://www.nytimes.com/2014/01/05/books/review...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>squander this thing we call ‘the day’ as quick...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>52c6e23b38f0d82ae415dd8c</td>\n",
       "      <td>2014-01-05 00:00:00</td>\n",
       "      <td>Books</td>\n",
       "      <td>http://www.nytimes.com/2014/01/05/books/review...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>) an entrepreneur (superior service!).</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>52c6ea6e38f0d82ae415ddb2</td>\n",
       "      <td>2014-01-05 00:00:00</td>\n",
       "      <td>Real Estate</td>\n",
       "      <td>http://www.nytimes.com/2014/01/05/realestate/a...</td>\n",
       "      <td>1</td>\n",
       "      <td>executive</td>\n",
       "      <td>“My designers did exactly what I wanted in th...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "                  article_id            pub_date        section_name  \\\n",
       "0   52ce1f2c38f0d81799290366 2014-01-01 00:00:00  Paid Death Notices   \n",
       "1   52c4045238f0d82e35d8e45c 2014-01-01 07:00:45        Business Day   \n",
       "2   52c491cb38f0d82d0ea3260c 2014-01-01 17:08:02                U.S.   \n",
       "3   52e0941e38f0d87ee88528f8 2014-01-15 00:00:00  Paid Death Notices   \n",
       "4   52c4b55038f0d82d0ea32638 2014-01-02 00:00:00        Business Day   \n",
       "5   52c4b55038f0d82d0ea32638 2014-01-02 00:00:00        Business Day   \n",
       "6   52c4b55038f0d82d0ea32638 2014-01-02 00:00:00        Business Day   \n",
       "7   52c8a15c38f0d862ec32236d 2014-01-05 00:00:00         Real Estate   \n",
       "8   52c6e22438f0d82ae415dd89 2014-01-05 00:00:00               Books   \n",
       "9   52c6e23b38f0d82ae415dd8c 2014-01-05 00:00:00               Books   \n",
       "10  52c6ea6e38f0d82ae415ddb2 2014-01-05 00:00:00         Real Estate   \n",
       "\n",
       "                                              web_url  term_category  \\\n",
       "0   http://query.nytimes.com/gst/fullpage.html?res...              1   \n",
       "1   http://boss.blogs.nytimes.com/2014/01/01/the-b...              1   \n",
       "2   http://www.nytimes.com/video/us/10000000262887...              1   \n",
       "3   http://query.nytimes.com/gst/fullpage.html?res...              1   \n",
       "4   http://www.nytimes.com/2014/01/02/business/hig...              1   \n",
       "5   http://www.nytimes.com/2014/01/02/business/hig...              1   \n",
       "6   http://www.nytimes.com/2014/01/02/business/hig...              1   \n",
       "7   http://www.nytimes.com/2014/01/05/realestate/r...              1   \n",
       "8   http://www.nytimes.com/2014/01/05/books/review...              1   \n",
       "9   http://www.nytimes.com/2014/01/05/books/review...              1   \n",
       "10  http://www.nytimes.com/2014/01/05/realestate/a...              1   \n",
       "\n",
       "            term                                           sentence  \n",
       "0   entrepreneur  An entrepreneur and connoisseur she was also p...  \n",
       "1   entrepreneur  Colleen DeBaise — and the Story Exchange — bro...  \n",
       "2      executive  Mike Duggan a former hospital executive and pr...  \n",
       "3      executive  Alisa R. Doctoroff President; Linda Mirels Cha...  \n",
       "4      executive              analysts and industry executives say.  \n",
       "5      executive  “People were pulling back when they had to let...  \n",
       "6   entrepreneur  DETROIT — Matt Hlavin an entrepreneur in Cleve...  \n",
       "7   entrepreneur  to the thrift and simplicity of our grandfathe...  \n",
       "8   entrepreneur  squander this thing we call ‘the day’ as quick...  \n",
       "9   entrepreneur             ) an entrepreneur (superior service!).  \n",
       "..."
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_rows_distinct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>term</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>executive</td>\n",
       "      <td>2960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>google</td>\n",
       "      <td>600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>amazon.com</td>\n",
       "      <td>185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e-bay</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "           term  count\n",
       "3     executive   2960\n",
       "4        google    600\n",
       "2  entrepreneur    186\n",
       "0    amazon.com    185\n",
       "1         e-bay     81"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_rows_distinct.term.count_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "4140"
      ],
      "text/plain": [
       "4140"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_rows_data.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "4012"
      ],
      "text/plain": [
       "4012"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_rows_distinct.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = total_rows_distinct[total_rows_distinct.term != 'executive']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>term</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>google</td>\n",
       "      <td>600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>amazon.com</td>\n",
       "      <td>185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e-bay</td>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "           term  count\n",
       "3        google    600\n",
       "2  entrepreneur    186\n",
       "0    amazon.com    185\n",
       "1         e-bay     81"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.term.count_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data[data.term == 'e-bay'].term_category.map(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>term_category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>577</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>815</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>816</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>817</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>818</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>819</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>821</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1008</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "      term_category\n",
       "24                3\n",
       "183               3\n",
       "184               3\n",
       "577               3\n",
       "815               3\n",
       "816               3\n",
       "817               3\n",
       "818               3\n",
       "819               3\n",
       "821               3\n",
       "..."
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data.term == 'e-bay'].term_category"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dshape(\"\"\"var * {\n",
       "  article_id: ?string,\n",
       "  pub_date: ?datetime,\n",
       "  section_name: ?string,\n",
       "  web_url: ?string,\n",
       "  term_category: int64,\n",
       "  term: ?string,\n",
       "  sentence: ?string\n",
       "  }\"\"\")"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dshape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "distinct(_1)[distinct(_1).term != 'executive'][3]\n"
     ]
    }
   ],
   "source": [
    "print(data[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('sentiment/be.pickle', 'rb') as be_file:\n",
    "    be = pickle.load(be_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dshape(\"\"\"var * {\n",
       "  article_id: ?string,\n",
       "  pub_date: ?datetime,\n",
       "  section_name: ?string,\n",
       "  web_url: ?string,\n",
       "  term_category: int64,\n",
       "  term: ?string,\n",
       "  sentence: ?string\n",
       "  }\"\"\")"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.dshape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ss = data.sentence.map(lambda x: be.predict([x])[0], '?string')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rr = ss.relabel({'sentence': 'sentiment'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "   sentiment\n",
       "0        pos\n",
       "1        pos\n",
       "6        pos\n",
       "7        pos\n",
       "8        pos\n",
       "9        pos\n",
       "11       pos\n",
       "12       pos\n",
       "13       pos\n",
       "14       pos\n",
       "..."
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_id</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>section_name</th>\n",
       "      <th>web_url</th>\n",
       "      <th>term_category</th>\n",
       "      <th>term</th>\n",
       "      <th>sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>52ce1f2c38f0d81799290366</td>\n",
       "      <td>2014-01-01 00:00:00</td>\n",
       "      <td>Paid Death Notices</td>\n",
       "      <td>http://query.nytimes.com/gst/fullpage.html?res...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>An entrepreneur and connoisseur she was also p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>52c4045238f0d82e35d8e45c</td>\n",
       "      <td>2014-01-01 07:00:45</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>http://boss.blogs.nytimes.com/2014/01/01/the-b...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>Colleen DeBaise — and the Story Exchange — bro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>52c4b55038f0d82d0ea32638</td>\n",
       "      <td>2014-01-02 00:00:00</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>http://www.nytimes.com/2014/01/02/business/hig...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>DETROIT — Matt Hlavin an entrepreneur in Cleve...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>52c8a15c38f0d862ec32236d</td>\n",
       "      <td>2014-01-05 00:00:00</td>\n",
       "      <td>Real Estate</td>\n",
       "      <td>http://www.nytimes.com/2014/01/05/realestate/r...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>to the thrift and simplicity of our grandfathe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>52c6e22438f0d82ae415dd89</td>\n",
       "      <td>2014-01-05 00:00:00</td>\n",
       "      <td>Books</td>\n",
       "      <td>http://www.nytimes.com/2014/01/05/books/review...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>squander this thing we call ‘the day’ as quick...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>52c6e23b38f0d82ae415dd8c</td>\n",
       "      <td>2014-01-05 00:00:00</td>\n",
       "      <td>Books</td>\n",
       "      <td>http://www.nytimes.com/2014/01/05/books/review...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>) an entrepreneur (superior service!).</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>52c6ea6e38f0d82ae415ddb2</td>\n",
       "      <td>2014-01-05 00:00:00</td>\n",
       "      <td>Real Estate</td>\n",
       "      <td>http://www.nytimes.com/2014/01/05/realestate/a...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>did exactly what I wanted in the apartment in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>52cc1e9738f0d86264f604ec</td>\n",
       "      <td>2014-01-07 10:30:22</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>http://boss.blogs.nytimes.com/2014/01/07/today...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>Entrepreneurs This entrepreneur is trying to c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>52cc414138f0d878bab47df0</td>\n",
       "      <td>2014-01-07 12:58:19</td>\n",
       "      <td>Technology</td>\n",
       "      <td>http://bits.blogs.nytimes.com/2014/01/07/wolfr...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>Stephen Wolfram a scientist and entrepreneur a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>52cc414138f0d878bab47df0</td>\n",
       "      <td>2014-01-07 12:58:19</td>\n",
       "      <td>Technology</td>\n",
       "      <td>http://bits.blogs.nytimes.com/2014/01/07/wolfr...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>unveiled at the International Consumer Electro...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>52cc5d5638f0d878bab47e50</td>\n",
       "      <td>2014-01-07 15:00:58</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>http://boss.blogs.nytimes.com/2014/01/07/entre...</td>\n",
       "      <td>1</td>\n",
       "      <td>entrepreneur</td>\n",
       "      <td>Entrepreneurial Lessons From 'The Wolf of Wall...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "                  article_id            pub_date        section_name  \\\n",
       "0   52ce1f2c38f0d81799290366 2014-01-01 00:00:00  Paid Death Notices   \n",
       "1   52c4045238f0d82e35d8e45c 2014-01-01 07:00:45        Business Day   \n",
       "6   52c4b55038f0d82d0ea32638 2014-01-02 00:00:00        Business Day   \n",
       "7   52c8a15c38f0d862ec32236d 2014-01-05 00:00:00         Real Estate   \n",
       "8   52c6e22438f0d82ae415dd89 2014-01-05 00:00:00               Books   \n",
       "9   52c6e23b38f0d82ae415dd8c 2014-01-05 00:00:00               Books   \n",
       "11  52c6ea6e38f0d82ae415ddb2 2014-01-05 00:00:00         Real Estate   \n",
       "12  52cc1e9738f0d86264f604ec 2014-01-07 10:30:22        Business Day   \n",
       "13  52cc414138f0d878bab47df0 2014-01-07 12:58:19          Technology   \n",
       "14  52cc414138f0d878bab47df0 2014-01-07 12:58:19          Technology   \n",
       "16  52cc5d5638f0d878bab47e50 2014-01-07 15:00:58        Business Day   \n",
       "\n",
       "                                              web_url  term_category  \\\n",
       "0   http://query.nytimes.com/gst/fullpage.html?res...              1   \n",
       "1   http://boss.blogs.nytimes.com/2014/01/01/the-b...              1   \n",
       "6   http://www.nytimes.com/2014/01/02/business/hig...              1   \n",
       "7   http://www.nytimes.com/2014/01/05/realestate/r...              1   \n",
       "8   http://www.nytimes.com/2014/01/05/books/review...              1   \n",
       "9   http://www.nytimes.com/2014/01/05/books/review...              1   \n",
       "11  http://www.nytimes.com/2014/01/05/realestate/a...              1   \n",
       "12  http://boss.blogs.nytimes.com/2014/01/07/today...              1   \n",
       "13  http://bits.blogs.nytimes.com/2014/01/07/wolfr...              1   \n",
       "14  http://bits.blogs.nytimes.com/2014/01/07/wolfr...              1   \n",
       "16  http://boss.blogs.nytimes.com/2014/01/07/entre...              1   \n",
       "\n",
       "            term                                           sentence  \n",
       "0   entrepreneur  An entrepreneur and connoisseur she was also p...  \n",
       "1   entrepreneur  Colleen DeBaise — and the Story Exchange — bro...  \n",
       "6   entrepreneur  DETROIT — Matt Hlavin an entrepreneur in Cleve...  \n",
       "7   entrepreneur  to the thrift and simplicity of our grandfathe...  \n",
       "8   entrepreneur  squander this thing we call ‘the day’ as quick...  \n",
       "9   entrepreneur             ) an entrepreneur (superior service!).  \n",
       "11  entrepreneur  did exactly what I wanted in the apartment in ...  \n",
       "12  entrepreneur  Entrepreneurs This entrepreneur is trying to c...  \n",
       "13  entrepreneur  Stephen Wolfram a scientist and entrepreneur a...  \n",
       "14  entrepreneur  unveiled at the International Consumer Electro...  \n",
       "..."
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import blaze"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ext = blaze.join(data, data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>pos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>neg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "   sentiment\n",
       "0        pos\n",
       "1        pos\n",
       "6        pos\n",
       "7        pos\n",
       "8        pos\n",
       "9        pos\n",
       "11       pos\n",
       "12       pos\n",
       "13       pos\n",
       "14       pos\n",
       "..."
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sdf = blaze.odo(rr, blaze.DataFrame)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sentiment    906\n",
       "dtype: int64"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sdf[sdf.sentiment == 'pos'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sentiment    146\n",
       "dtype: int64"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sdf[sdf.sentiment == 'neg'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "ddf = blaze.odo(data, blaze.DataFrame)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ddf['sent'] = sdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data2 = Data(ddf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "1052"
      ],
      "text/plain": [
       "1052"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "906"
      ],
      "text/plain": [
       "906"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2[data2.sent == 'pos'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "1052"
      ],
      "text/plain": [
       "1052"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2.term_category.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "86"
      ],
      "text/plain": [
       "86"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "100*data2[data2.sent == 'pos'].count()//data2.term_category.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-138-6ee2a1550800>, line 4)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-138-6ee2a1550800>\"\u001b[1;36m, line \u001b[1;32m4\u001b[0m\n\u001b[1;33m    )a\u001b[0m\n\u001b[1;37m     ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "by(data2.term_category,\n",
    "   prop_pos=100*data2[data2.sent == 'pos'].count()//data2.term_category.count(),\n",
    "   prop_neg=100*data2[data2.sent == 'neg'].count()//data2.term_category.count(),\n",
    ")a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>term_category</th>\n",
       "      <th>amount_neg</th>\n",
       "      <th>amount_pos</th>\n",
       "      <th>total_amount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>173</td>\n",
       "      <td>186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>106</td>\n",
       "      <td>620</td>\n",
       "      <td>726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>27</td>\n",
       "      <td>113</td>\n",
       "      <td>140</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "   term_category  amount_neg  amount_pos  total_amount\n",
       "0              1          13         173           186\n",
       "1              2         106         620           726\n",
       "2              3          27         113           140"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "by(data2.term_category,\n",
    "   total_amount=data2.sent.count(),\n",
    "   amount_pos=data2[data2.sent == 'pos'].sent.count(),\n",
    "   amount_neg=data2[data2.sent == 'neg'].sent.count()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x = by(data2.sentence, unique=data2.term.nunique() == 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "996"
      ],
      "text/plain": [
       "996"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[x.unique == True].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "14"
      ],
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[x.unique == False].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "repeat = x[x.unique == False]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "14"
      ],
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "repeat.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 543,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "'module' object is not callable",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-543-e06cfb91f969>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mblaze\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrepeat\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m: 'module' object is not callable"
     ]
    }
   ],
   "source": [
    "blaze.utils."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 530,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_id</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>section_name</th>\n",
       "      <th>web_url</th>\n",
       "      <th>term_category</th>\n",
       "      <th>term</th>\n",
       "      <th>sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3919</th>\n",
       "      <td>5303a06738f0d835dccb3278</td>\n",
       "      <td>2014-02-18 12:59:23</td>\n",
       "      <td>Technology</td>\n",
       "      <td>http://bits.blogs.nytimes.com/2014/02/18/postm...</td>\n",
       "      <td>2</td>\n",
       "      <td>google</td>\n",
       "      <td>In addition to Uber there is competition from ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3921</th>\n",
       "      <td>5303a06738f0d835dccb3278</td>\n",
       "      <td>2014-02-18 12:59:23</td>\n",
       "      <td>Technology</td>\n",
       "      <td>http://bits.blogs.nytimes.com/2014/02/18/postm...</td>\n",
       "      <td>3</td>\n",
       "      <td>e-bay</td>\n",
       "      <td>In addition to Uber there is competition from ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3922</th>\n",
       "      <td>5303a06738f0d835dccb3278</td>\n",
       "      <td>2014-02-18 12:59:23</td>\n",
       "      <td>Technology</td>\n",
       "      <td>http://bits.blogs.nytimes.com/2014/02/18/postm...</td>\n",
       "      <td>2</td>\n",
       "      <td>amazon.com</td>\n",
       "      <td>In addition to Uber there is competition from ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "                    article_id            pub_date section_name  \\\n",
       "3919  5303a06738f0d835dccb3278 2014-02-18 12:59:23   Technology   \n",
       "3921  5303a06738f0d835dccb3278 2014-02-18 12:59:23   Technology   \n",
       "3922  5303a06738f0d835dccb3278 2014-02-18 12:59:23   Technology   \n",
       "\n",
       "                                                web_url  term_category  \\\n",
       "3919  http://bits.blogs.nytimes.com/2014/02/18/postm...              2   \n",
       "3921  http://bits.blogs.nytimes.com/2014/02/18/postm...              3   \n",
       "3922  http://bits.blogs.nytimes.com/2014/02/18/postm...              2   \n",
       "\n",
       "            term                                           sentence  \n",
       "3919      google  In addition to Uber there is competition from ...  \n",
       "3921       e-bay  In addition to Uber there is competition from ...  \n",
       "3922  amazon.com  In addition to Uber there is competition from ...  "
      ]
     },
     "execution_count": 530,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.like(sentence='In addition to Uber there is competition*')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 523,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>unique</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "   unique  count\n",
       "0       1    996\n",
       "1       2     13\n",
       "2       3      1"
      ]
     },
     "execution_count": 523,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.unique.count_values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 494,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_id</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>section_name</th>\n",
       "      <th>web_url</th>\n",
       "      <th>term_category</th>\n",
       "      <th>term</th>\n",
       "      <th>sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>52cf2fa138f0d8359e12cf29</td>\n",
       "      <td>2014-01-10 00:00:00</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>http://www.nytimes.com/2014/01/10/business/med...</td>\n",
       "      <td>3</td>\n",
       "      <td>e-bay</td>\n",
       "      <td>Pierre Omidyar the founder of eBay has pledged...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>52deac6238f0d80317840823</td>\n",
       "      <td>2014-01-21 12:17:14</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>http://boss.blogs.nytimes.com/2014/01/21/today...</td>\n",
       "      <td>3</td>\n",
       "      <td>e-bay</td>\n",
       "      <td>Online Ebay is reportedly planning a new marke...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184</th>\n",
       "      <td>52eeff4238f0d82003e2538e</td>\n",
       "      <td>2014-02-03 00:00:00</td>\n",
       "      <td>Sports</td>\n",
       "      <td>http://www.nytimes.com/2014/02/03/sports/footb...</td>\n",
       "      <td>3</td>\n",
       "      <td>e-bay</td>\n",
       "      <td>As for parking passes they were sold for $150 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>576</th>\n",
       "      <td>52cd55e038f0d878bab480c9</td>\n",
       "      <td>2014-01-08 08:39:33</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>http://dealbook.nytimes.com/2014/01/08/morning...</td>\n",
       "      <td>3</td>\n",
       "      <td>e-bay</td>\n",
       "      <td>Get your own hedge fund fleece on eBay where s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>815</th>\n",
       "      <td>52dea67e38f0d808a7481088</td>\n",
       "      <td>2014-01-21 11:54:30</td>\n",
       "      <td>Business Day</td>\n",
       "      <td>http://dealbook.nytimes.com/2014/01/21/why-bit...</td>\n",
       "      <td>3</td>\n",
       "      <td>e-bay</td>\n",
       "      <td>Bitcoin shares this network effect property wi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>816</th>\n",
       "      <td>52e2c8ed38f0d87ecf265e38</td>\n",
       "      <td>2014-01-25 00:00:00</td>\n",
       "      <td>Automobiles</td>\n",
       "      <td>http://www.nytimes.com/2014/01/25/automobiles/...</td>\n",
       "      <td>3</td>\n",
       "      <td>e-bay</td>\n",
       "      <td>the Candelabra” popped up for sale on eBay thi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>817</th>\n",
       "      <td>52e99d9b38f0d84aab9e9a52</td>\n",
       "      <td>2014-01-30 00:00:00</td>\n",
       "      <td>Home &amp; Garden</td>\n",
       "      <td>http://www.nytimes.com/2014/01/30/garden/extra...</td>\n",
       "      <td>3</td>\n",
       "      <td>e-bay</td>\n",
       "      <td>I use wrought-iron tools I bought on eBay and ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>818</th>\n",
       "      <td>52fe860938f0d826f8460d7c</td>\n",
       "      <td>2014-02-16 00:00:00</td>\n",
       "      <td>Fashion &amp; Style</td>\n",
       "      <td>http://www.nytimes.com/2014/02/16/fashion/wint...</td>\n",
       "      <td>3</td>\n",
       "      <td>e-bay</td>\n",
       "      <td>I knew no suede.” Her Sorels were vintage sour...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>819</th>\n",
       "      <td>52fe860938f0d826f8460d7c</td>\n",
       "      <td>2014-02-16 00:00:00</td>\n",
       "      <td>Fashion &amp; Style</td>\n",
       "      <td>http://www.nytimes.com/2014/02/16/fashion/wint...</td>\n",
       "      <td>3</td>\n",
       "      <td>e-bay</td>\n",
       "      <td>She turned to eBay because she said the boots ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>820</th>\n",
       "      <td>530c153f38f0d82a1358d5cf</td>\n",
       "      <td>2014-02-25 00:00:00</td>\n",
       "      <td>Sports</td>\n",
       "      <td>http://www.nytimes.com/2014/02/25/sports/baske...</td>\n",
       "      <td>3</td>\n",
       "      <td>e-bay</td>\n",
       "      <td>Game-used Collins jerseys from his previous te...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1008</th>\n",
       "      <td>52ec67dc38f0d84314fee6eb</td>\n",
       "      <td>2014-02-01 00:00:00</td>\n",
       "      <td>Technology</td>\n",
       "      <td>http://www.nytimes.com/2014/02/01/technology/t...</td>\n",
       "      <td>3</td>\n",
       "      <td>e-bay</td>\n",
       "      <td>Rising resentment against elite technology fir...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "                    article_id            pub_date     section_name  \\\n",
       "24    52cf2fa138f0d8359e12cf29 2014-01-10 00:00:00     Business Day   \n",
       "183   52deac6238f0d80317840823 2014-01-21 12:17:14     Business Day   \n",
       "184   52eeff4238f0d82003e2538e 2014-02-03 00:00:00           Sports   \n",
       "576   52cd55e038f0d878bab480c9 2014-01-08 08:39:33     Business Day   \n",
       "815   52dea67e38f0d808a7481088 2014-01-21 11:54:30     Business Day   \n",
       "816   52e2c8ed38f0d87ecf265e38 2014-01-25 00:00:00      Automobiles   \n",
       "817   52e99d9b38f0d84aab9e9a52 2014-01-30 00:00:00    Home & Garden   \n",
       "818   52fe860938f0d826f8460d7c 2014-02-16 00:00:00  Fashion & Style   \n",
       "819   52fe860938f0d826f8460d7c 2014-02-16 00:00:00  Fashion & Style   \n",
       "820   530c153f38f0d82a1358d5cf 2014-02-25 00:00:00           Sports   \n",
       "1008  52ec67dc38f0d84314fee6eb 2014-02-01 00:00:00       Technology   \n",
       "\n",
       "                                                web_url  term_category   term  \\\n",
       "24    http://www.nytimes.com/2014/01/10/business/med...              3  e-bay   \n",
       "183   http://boss.blogs.nytimes.com/2014/01/21/today...              3  e-bay   \n",
       "184   http://www.nytimes.com/2014/02/03/sports/footb...              3  e-bay   \n",
       "576   http://dealbook.nytimes.com/2014/01/08/morning...              3  e-bay   \n",
       "815   http://dealbook.nytimes.com/2014/01/21/why-bit...              3  e-bay   \n",
       "816   http://www.nytimes.com/2014/01/25/automobiles/...              3  e-bay   \n",
       "817   http://www.nytimes.com/2014/01/30/garden/extra...              3  e-bay   \n",
       "818   http://www.nytimes.com/2014/02/16/fashion/wint...              3  e-bay   \n",
       "819   http://www.nytimes.com/2014/02/16/fashion/wint...              3  e-bay   \n",
       "820   http://www.nytimes.com/2014/02/25/sports/baske...              3  e-bay   \n",
       "1008  http://www.nytimes.com/2014/02/01/technology/t...              3  e-bay   \n",
       "\n",
       "                                               sentence  \n",
       "24    Pierre Omidyar the founder of eBay has pledged...  \n",
       "183   Online Ebay is reportedly planning a new marke...  \n",
       "184   As for parking passes they were sold for $150 ...  \n",
       "576   Get your own hedge fund fleece on eBay where s...  \n",
       "815   Bitcoin shares this network effect property wi...  \n",
       "816   the Candelabra” popped up for sale on eBay thi...  \n",
       "817   I use wrought-iron tools I bought on eBay and ...  \n",
       "818   I knew no suede.” Her Sorels were vintage sour...  \n",
       "819   She turned to eBay because she said the boots ...  \n",
       "820   Game-used Collins jerseys from his previous te...  \n",
       "..."
      ]
     },
     "execution_count": 494,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[data.term == 'e-bay']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "iris = Data('iris.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal_length</th>\n",
       "      <th>sepal_width</th>\n",
       "      <th>petal_length</th>\n",
       "      <th>petal_width</th>\n",
       "      <th>species</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris, setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5.4</td>\n",
       "      <td>3.9</td>\n",
       "      <td>1.7</td>\n",
       "      <td>0.4</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.4</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.3</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.4</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>4.4</td>\n",
       "      <td>2.9</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.1</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>5.4</td>\n",
       "      <td>3.7</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "    sepal_length  sepal_width  petal_length  petal_width       species\n",
       "0            5.1          3.5           1.4          0.2  Iris, setosa\n",
       "1            4.9          3.0           1.4          0.2   Iris-setosa\n",
       "2            4.7          3.2           1.3          0.2   Iris-setosa\n",
       "3            4.6          3.1           1.5          0.2   Iris-setosa\n",
       "4            5.0          3.6           1.4          0.2   Iris-setosa\n",
       "5            5.4          3.9           1.7          0.4   Iris-setosa\n",
       "6            4.6          3.4           1.4          0.3   Iris-setosa\n",
       "7            5.0          3.4           1.5          0.2   Iris-setosa\n",
       "8            4.4          2.9           1.4          0.2   Iris-setosa\n",
       "9            4.9          3.1           1.5          0.1   Iris-setosa\n",
       "..."
      ]
     },
     "execution_count": 389,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iris"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}