ft-old.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Sentence extractor for The New York Times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import logging\n",
    "import math\n",
    "import os\n",
    "from collections import defaultdict\n",
    "from datetime import timedelta, date, datetime\n",
    "from dateutil import parser\n",
    "from time import sleep, time\n",
    "\n",
    "import requests\n",
    "import yaml\n",
    "from joblib import Parallel, delayed\n",
    "from pymongo import MongoClient\n",
    "from pymongo.errors import BulkWriteError, DuplicateKeyError"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Logging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    os.remove('search_ft.log')\n",
    "except:\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "logging.getLogger().handlers = []\n",
    "logging.getLogger('requests.packages.urllib3').setLevel(logging.WARNING)\n",
    "logging.basicConfig(filename='search_ft.log', level=logging.INFO, format='%(asctime)s %(message)s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def write_log(*args, status=None):\n",
    "    record = '{} ==> {}'.format(args, status)\n",
    "    logging.info(record)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##MongoDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "client = MongoClient()\n",
    "client.drop_database('ft')\n",
    "db = client.ft"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def insert_documents(docs, q):\n",
    "    try:\n",
    "        inserted = db.articles.insert_many(docs, ordered=False)\n",
    "        total_inserted = len(inserted.inserted_ids)\n",
    "        write_log(q, status='INSERTION OK {}'.format(total_inserted))\n",
    "    except BulkWriteError as e:\n",
    "        for err in e.details['writeErrors']:\n",
    "            write_log(q, status='INSERTION EXCEPTION {}'.format(err['errmsg']))\n",
    "        write_log(q, status='INSERTION OK {}'.format(e.details['nInserted']))\n",
    "    except Exception as e:\n",
    "        write_log(q, status='INSERTION EXCEPTION {}'.format(e))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Search terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def preprocess_term(term):\n",
    "    terms = []\n",
    "    \n",
    "    curated_term = term.lower()\n",
    "    curated_term = curated_term.replace(' & ', ' ')\n",
    "    curated_term = curated_term.replace(' and ', ' ')\n",
    "    terms.append(curated_term)\n",
    "    if '-' in curated_term:\n",
    "        terms.append(curated_term.replace('-', ''))\n",
    "#         terms.append(curated_term.replace('-', ' ')) # same result\n",
    "\n",
    "    for term in terms:\n",
    "        if term.endswith('corporation'):\n",
    "            terms.append(term[:-12])\n",
    "        elif term.endswith('corp'):\n",
    "            terms.append(term[:-5])\n",
    "        elif term.endswith('company'):\n",
    "            terms.append(term[:-8])\n",
    "        elif term.endswith('inc.'):\n",
    "            terms.append(term[:-5])\n",
    "        elif term.endswith('inc'):\n",
    "            terms.append(term[:-4])\n",
    "        elif term.endswith('.com'):\n",
    "            terms.append(term[:-4])\n",
    "    \n",
    "#     terms = list(map(lambda x: '\"{}\"'.format(x), terms))\n",
    "    return terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "term_dict = {}\n",
    "for i in range(1, 4):\n",
    "    term_dict[i] = {}\n",
    "    st_file = open('search_terms_{}.txt'.format(i))\n",
    "    term_list = map(lambda x: x.strip(), st_file.readlines())\n",
    "    for term in term_list:\n",
    "        term_dict[i][term] = preprocess_term(term)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('search_terms.yml', 'w') as search_term_file:\n",
    "    search_term_file.write(yaml.dump(term_dict, default_flow_style=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('search_terms.yml') as search_term_file:\n",
    "    term_yaml = yaml.load(search_term_file.read())\n",
    "    #\n",
    "    for key in term_yaml:\n",
    "        d = term_yaml[key]\n",
    "        for k in d:\n",
    "            d[k] = list(map(lambda x: '\"{}\"'.format(x), d[k]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##NYTimes API keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# One API key for each core\n",
    "api_keys = [\n",
    "        \"mtqjgqpd63c5aky6ujz7mfe4\",\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def next_multiple(n, m):\n",
    "    # 4, 17 ==> 20\n",
    "    rest = m % n\n",
    "    return m if rest == 0 else m + n - rest\n",
    "\n",
    "def chunks(l, n_chunks):\n",
    "    l = list(l)\n",
    "    size = len(l)\n",
    "    n = next_multiple(n_chunks, size) // n_chunks\n",
    "    for i in range(0, size, n):\n",
    "        yield l[i:i + n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# [(search_term, original_term, term_type)]\n",
    "search_terms = [(t, k2, k1)\n",
    "                   for k1 in term_yaml\n",
    "                       for k2 in term_yaml[k1]\n",
    "                           for t in term_yaml[k1][k2]\n",
    "              ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TEST\n",
    "\n",
    "# api_keys = [\n",
    "#     \"3439a9084efa80c4f5fb1d290dfc1b44:11:70233981\", # my api key\n",
    "#     \"a5c709f3168b829711241b243457e9d6:13:70235641\", # the other api key\n",
    "# ]\n",
    "\n",
    "# search_terms = [\n",
    "#     ('\"entrepreneur\"', 'entrepreneur', 1),\n",
    "#     ('\"executive\"', 'executive', 1),\n",
    "#     ('\"google\"', 'google', 2),\n",
    "#     ('\"amazon\"', 'amazon.com', 2),\n",
    "#     ('\"ebay\"', 'e-bay', 3),\n",
    "#     ('\"facebook\"', 'facebook', 3),\n",
    "# ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "search_terms_by_api_key = defaultdict(list)\n",
    "for i in range(1, 4):\n",
    "    search_terms_i = list(filter(lambda x: x[2] == i, search_terms))\n",
    "    for t in zip(api_keys, chunks(search_terms_i, len(api_keys))):\n",
    "        search_terms_by_api_key[t[0]].extend(t[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Dates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def month_duration(d):\n",
    "    if d.month in [1, 3, 5, 7, 8, 10, 12]:\n",
    "        ndays = 31\n",
    "    elif d.month in [4, 6, 9, 11]:\n",
    "        ndays = 30\n",
    "    else: # d.month == 2\n",
    "        if d.year % 400 == 0 or d.year % 4 == 0 and d.year % 100 != 0: # lap-year\n",
    "            ndays = 29\n",
    "        else:\n",
    "            ndays = 28\n",
    "    return ndays\n",
    "\n",
    "def n_days(d, n_months):\n",
    "    ndays = 0\n",
    "    new_d = d\n",
    "    for _ in range(n_months):\n",
    "        m_duration = month_duration(d)\n",
    "        d += timedelta(m_duration)\n",
    "        ndays += m_duration\n",
    "    return ndays - 1\n",
    "\n",
    "def date_ranges(begin_date, end_date, n_months=1):\n",
    "    aux_date = begin_date\n",
    "    while aux_date < end_date:\n",
    "        ndays = n_days(aux_date, n_months)\n",
    "        yield (aux_date, min(aux_date + timedelta(ndays), end_date))\n",
    "        aux_date += timedelta(ndays + 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Timer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "LAST_REQUEST = time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def wait(f, *args, t=9):\n",
    "    global LAST_REQUEST\n",
    "    now = time()\n",
    "    elapsed_time = now - LAST_REQUEST\n",
    "    if elapsed_time < t:\n",
    "        sleep(t - elapsed_time)\n",
    "    LAST_REQUEST = time()\n",
    "    return f(*args)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Query:\n",
    "    def __init__(self, term, begin_date, end_date, page, api_key):\n",
    "        self.term = term\n",
    "        self.begin_date = begin_date\n",
    "        self.end_date = end_date\n",
    "        self.page = page\n",
    "        self.api_key =api_key\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return 'Q<{}, {}, {}, {}, {}>'.format(self.term, self.begin_date, self.end_date, self.page, self.api_key)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##Downloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pytz\n",
    "from datetime import timedelta, date, datetime, tzinfo\n",
    "BEGIN_DATE = datetime(1999, 1, 1)\n",
    "END_DATE = date(2014, 12, 31)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2014-12-31T00:00:00Z'"
      ]
     },
     "execution_count": 350,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datetime.combine(END_DATE, datetime.min.time()).isoformat() + 'Z'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'datetime.datetime'>\n",
      "<class 'datetime.date'>\n"
     ]
    }
   ],
   "source": [
    "print(type(BEGIN_DATE))\n",
    "print(type(END_DATE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1999-01-01 00:00:00'"
      ]
     },
     "execution_count": 348,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(BEGIN_DATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'http://api.ft.com/content/search/v1?apiKey=mtqjgqpd63c5aky6ujz7mfe4'\n",
    "r1 = requests.post(url, json={'queryString': 'executive'})\n",
    "r1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'http://api.ft.com/content/search/v1'\n",
    "r2 = requests.post(url, params={'apiKey': 'mtqjgqpd63c5aky6ujz7mfe4'}, json={'queryString': 'entrepreneur'})\n",
    "r2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 368,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'http://api.ft.com/content/search/v1'\n",
    "r3 = requests.post(url, params={'apiKey': 'mtqjgqpd63c5aky6ujz7mfe4'}, json={\n",
    "        'queryString': '\"entrepreneur\" AND initialPublishDateTime:>2014-01-01T00:00:00Z AND initialPublishDateTime:<2014-01-04T00:00:00Z',\n",
    "#         'queryString': '\"executive\"',\n",
    "        'resultContext': {\n",
    "            'aspects': [\"title\",\"lifecycle\",\"location\",\"summary\",\"editorial\"],\n",
    "            'offset': 0,\n",
    "        }\n",
    "    })\n",
    "r3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'OK'"
      ]
     },
     "execution_count": 369,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r3.reason"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'apiUrl': 'http://api.ft.com/content/items/v1/b7a2a7f2-72d6-11e3-b05b-00144feabdc0',\n",
       "  'aspectSet': 'article',\n",
       "  'editorial': {'byline': 'By Henry Mance, Media Correspondent'},\n",
       "  'id': 'b7a2a7f2-72d6-11e3-b05b-00144feabdc0',\n",
       "  'lifecycle': {'initialPublishDateTime': '2014-01-01T21:00:25Z',\n",
       "   'lastPublishDateTime': '2014-01-01T21:00:25Z'},\n",
       "  'location': {'uri': 'http://www.ft.com/cms/s/0/b7a2a7f2-72d6-11e3-b05b-00144feabdc0.html'},\n",
       "  'modelVersion': '1',\n",
       "  'summary': {'excerpt': 'Richard Desmond has asked advisers to work on a possible sale of Channel 5, the British free-to-air TV broadcaster, that'},\n",
       "  'title': {'title': 'Richard Desmond explores Channel 5 sale options'}},\n",
       " {'apiUrl': 'http://api.ft.com/content/items/v1/bde1a771-814a-3e45-8443-220537ed8ba1',\n",
       "  'aspectSet': 'blogPost',\n",
       "  'editorial': {'byline': 'Samantha Pearson'},\n",
       "  'id': 'bde1a771-814a-3e45-8443-220537ed8ba1',\n",
       "  'lifecycle': {'initialPublishDateTime': '2014-01-02T23:26:40Z',\n",
       "   'lastPublishDateTime': '2014-01-02T23:26:40Z'},\n",
       "  'location': {'uri': 'http://blogs.ft.com/beyond-brics/2014/01/02/brazil-good-riddance-2013/'},\n",
       "  'modelVersion': '1',\n",
       "  'summary': {'excerpt': 'When the clock struck midnight on New Year’s eve this week, Brazilian President Dilma Rousseff probably breathed a huge sigh'},\n",
       "  'title': {'title': 'Brazil – good riddance, 2013'}},\n",
       " {'apiUrl': 'http://api.ft.com/content/items/v1/9908e506-6ef6-11e3-9ac9-00144feabdc0',\n",
       "  'aspectSet': 'article',\n",
       "  'editorial': {},\n",
       "  'id': '9908e506-6ef6-11e3-9ac9-00144feabdc0',\n",
       "  'lifecycle': {'initialPublishDateTime': '2014-01-02T22:54:00Z',\n",
       "   'lastPublishDateTime': '2014-01-02T22:54:00Z'},\n",
       "  'location': {'uri': 'http://www.ft.com/cms/s/0/9908e506-6ef6-11e3-9ac9-00144feabdc0.html'},\n",
       "  'modelVersion': '1',\n",
       "  'summary': {'excerpt': 'From Mr Pierre Kiecolt-Wahl. Sir, I appreciated Jeremy Grant’s article “Marubeni sets sights on Asean” (December 16).'},\n",
       "  'title': {'title': 'Southeast Asia prizes its ties with mainland China'}},\n",
       " {'apiUrl': 'http://api.ft.com/content/items/v1/274f8b4a-6c26-11e3-a216-00144feabdc0',\n",
       "  'aspectSet': 'article',\n",
       "  'editorial': {'byline': 'By Nicholas Lander'},\n",
       "  'id': '274f8b4a-6c26-11e3-a216-00144feabdc0',\n",
       "  'lifecycle': {'initialPublishDateTime': '2014-01-03T15:31:32Z',\n",
       "   'lastPublishDateTime': '2014-01-03T15:31:32Z'},\n",
       "  'location': {'uri': 'http://www.ft.com/cms/s/2/274f8b4a-6c26-11e3-a216-00144feabdc0.html'},\n",
       "  'modelVersion': '1',\n",
       "  'summary': {'excerpt': 'At the end of a radio discussion on what 2014 will hold for restaurant-goers, the other panellists and I repaired to a'},\n",
       "  'title': {'title': 'Jackson + Rye, London'}},\n",
       " {'apiUrl': 'http://api.ft.com/content/items/v1/42dcef88-7469-11e3-af50-00144feabdc0',\n",
       "  'aspectSet': 'article',\n",
       "  'editorial': {'byline': 'By'},\n",
       "  'id': '42dcef88-7469-11e3-af50-00144feabdc0',\n",
       "  'lifecycle': {'initialPublishDateTime': '2014-01-03T19:55:23Z',\n",
       "   'lastPublishDateTime': '2014-01-03T19:55:23Z'},\n",
       "  'location': {'uri': 'http://www.ft.com/cms/s/0/42dcef88-7469-11e3-af50-00144feabdc0.html'},\n",
       "  'modelVersion': '1',\n",
       "  'summary': {'excerpt': 'A round up of some of the week’s most significant corporate events and news stories. Train crash sets off debate about oil'},\n",
       "  'title': {'title': 'Week in Review, January 4'}},\n",
       " {'apiUrl': 'http://api.ft.com/content/items/v1/5fe95832-777e-3ef6-ab3e-54debfb0ba6c',\n",
       "  'aspectSet': 'blogPost',\n",
       "  'editorial': {'byline': 'FT Alphaville'},\n",
       "  'id': '5fe95832-777e-3ef6-ab3e-54debfb0ba6c',\n",
       "  'lifecycle': {'initialPublishDateTime': '2014-01-02T05:31:10Z',\n",
       "   'lastPublishDateTime': '2014-01-02T05:31:10Z'},\n",
       "  'location': {'uri': 'http://ftalphaville.ft.com/2014/01/02/1730772/the-6am-london-cut-86/'},\n",
       "  'modelVersion': '1',\n",
       "  'summary': {'excerpt': 'Yup. The Cut’s back. Happy New Year everyone. Markets: Asian markets are suffering on the first day of 2014 trading on the'},\n",
       "  'title': {'title': 'The 6am London Cut'}},\n",
       " {'apiUrl': 'http://api.ft.com/content/items/v1/e2fa35f8-7116-11e3-8f92-00144feabdc0',\n",
       "  'aspectSet': 'article',\n",
       "  'editorial': {},\n",
       "  'id': 'e2fa35f8-7116-11e3-8f92-00144feabdc0',\n",
       "  'lifecycle': {'initialPublishDateTime': '2014-01-01T20:00:10Z',\n",
       "   'lastPublishDateTime': '2014-01-01T20:00:10Z'},\n",
       "  'location': {'uri': 'http://www.ft.com/cms/s/0/e2fa35f8-7116-11e3-8f92-00144feabdc0.html'},\n",
       "  'modelVersion': '1',\n",
       "  'summary': {'excerpt': 'Has George Osborne’s “plan A” been vindicated by the recovery in 2013 and should the planned pace of deficit reduction'},\n",
       "  'title': {'title': 'Question 4: Fiscal policy'}},\n",
       " {'apiUrl': 'http://api.ft.com/content/items/v1/ad2a4fba-674a-11e3-a5f9-00144feabdc0',\n",
       "  'aspectSet': 'article',\n",
       "  'editorial': {'byline': 'By Sarah Mishkin in Taipei'},\n",
       "  'id': 'ad2a4fba-674a-11e3-a5f9-00144feabdc0',\n",
       "  'lifecycle': {'initialPublishDateTime': '2014-01-02T12:10:53Z',\n",
       "   'lastPublishDateTime': '2014-01-02T12:10:53Z'},\n",
       "  'location': {'uri': 'http://www.ft.com/cms/s/0/ad2a4fba-674a-11e3-a5f9-00144feabdc0.html'},\n",
       "  'modelVersion': '1',\n",
       "  'summary': {'excerpt': 'Tech in China is sometimes thought of as a scene of copy cats and clones. Alibaba is China’s eBay, the saying goes, Baidu is'},\n",
       "  'title': {'title': 'China tech titans hold sway over start-ups'}},\n",
       " {'apiUrl': 'http://api.ft.com/content/items/v1/77f7afd0-7399-11e3-beeb-00144feabdc0',\n",
       "  'aspectSet': 'article',\n",
       "  'editorial': {'byline': 'By Kate Burgess'},\n",
       "  'id': '77f7afd0-7399-11e3-beeb-00144feabdc0',\n",
       "  'lifecycle': {'initialPublishDateTime': '2014-01-02T16:02:48Z',\n",
       "   'lastPublishDateTime': '2014-01-02T16:02:48Z'},\n",
       "  'location': {'uri': 'http://www.ft.com/cms/s/0/77f7afd0-7399-11e3-beeb-00144feabdc0.html'},\n",
       "  'modelVersion': '1',\n",
       "  'summary': {'excerpt': 'Encouraging an entrepreneurial spirit among staff is vital to keeping companies forever youthful, according to experts in'},\n",
       "  'title': {'title': 'Encouraging entrepreneurs keeps companies forever young'}},\n",
       " {'apiUrl': 'http://api.ft.com/content/items/v1/6ec25fd2-72f9-11e3-b05b-00144feabdc0',\n",
       "  'aspectSet': 'article',\n",
       "  'editorial': {},\n",
       "  'id': '6ec25fd2-72f9-11e3-b05b-00144feabdc0',\n",
       "  'lifecycle': {'initialPublishDateTime': '2014-01-02T22:58:47Z',\n",
       "   'lastPublishDateTime': '2014-01-02T22:58:47Z'},\n",
       "  'location': {'uri': 'http://www.ft.com/cms/s/0/6ec25fd2-72f9-11e3-b05b-00144feabdc0.html'},\n",
       "  'modelVersion': '1',\n",
       "  'summary': {'excerpt': 'From Mr Timothy Hadleigh. Sir, The financial services authorities have become so terrified that the average punter might'},\n",
       "  'title': {'title': 'We’re scared of our own shadow – start-ups haven’t a chance'}},\n",
       " {'apiUrl': 'http://api.ft.com/content/items/v1/a92358fc-7498-11e3-af50-00144feabdc0',\n",
       "  'aspectSet': 'article',\n",
       "  'editorial': {'byline': 'By Hannah Kuchler in San Francisco'},\n",
       "  'id': 'a92358fc-7498-11e3-af50-00144feabdc0',\n",
       "  'lifecycle': {'initialPublishDateTime': '2014-01-03T18:49:01Z',\n",
       "   'lastPublishDateTime': '2014-01-03T18:49:01Z'},\n",
       "  'location': {'uri': 'http://www.ft.com/cms/s/0/a92358fc-7498-11e3-af50-00144feabdc0.html'},\n",
       "  'modelVersion': '1',\n",
       "  'summary': {'excerpt': 'When Mark Zuckerberg gave $1bn in Facebook stock to a charity – the largest donation in the US last year – he chose to'},\n",
       "  'title': {'title': 'Mark Zuckerberg’s donation highlights Silicon Valley poverty'}}]"
      ]
     },
     "execution_count": 370,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r3.json()['results'][0]['results']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11"
      ]
     },
     "execution_count": 357,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(r3.json()['results'][0]['results'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r1.json() == r2.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "r = r2.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'queryContext': {'curations': ['ARTICLES',\n",
       "   'BLOGS',\n",
       "   'PAGES',\n",
       "   'PODCASTS',\n",
       "   'VIDEOS']},\n",
       " 'queryString': 'entrepreneur',\n",
       " 'resultContext': {'contextual': False,\n",
       "  'highlight': False,\n",
       "  'maxResults': 100,\n",
       "  'offset': 0}}"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r['query']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def search(q):\n",
    "    try:\n",
    "        base_url = 'http://api.ft.com/content/search/v1'\n",
    "        payload = {'apiKey': q.api_key}\n",
    "        data = {'queryString': q.term}\n",
    "        r = requests.post(base_url, params=payload, json=data)\n",
    "        response = r.json()\n",
    "        \n",
    "        if response['status'] == 'OK':\n",
    "            write_log(q, status='SEARCH OK {}'.format(response['response']['meta']['hits']))\n",
    "        elif response['status'] == 'ERROR':\n",
    "            write_log(q, status='SEARCH ERROR {}'.format(response['errors']))\n",
    "        else:\n",
    "            write_log(q, status='SEARCH {}'.format(response['status']))\n",
    "    except ValueError as e:\n",
    "        if str(e) == 'Expecting value: line 1 column 1 (char 0)':\n",
    "            write_log(q, status='SEARCH ERROR api-key')\n",
    "        else:\n",
    "            write_log(q, status='SEARCH EXCEPTION {}'.format(e))\n",
    "        response = {'status': 'ERROR'}\n",
    "    except Exception as e:\n",
    "        write_log(q, status='SEARCH EXCEPTION {}'.format(e))\n",
    "        response = {'status': 'ERROR'}\n",
    "    return response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_documents_by_page(q, term, total_results):\n",
    "    n_pages = math.ceil(total_results / 10)\n",
    "    for page in range(n_pages):\n",
    "        q.page = page\n",
    "        response = wait(search, q)\n",
    "        \n",
    "        if response['status'] == 'OK':\n",
    "            docs = response['response']['docs']\n",
    "            for doc in docs:\n",
    "                doc.update({'q': q.__dict__, 'original_term': term[1], 'term_type': term[2]})\n",
    "            insert_documents(docs, q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_documents_by_query(q, term):\n",
    "    response = wait(search, q)\n",
    "    \n",
    "    if response['status'] == 'OK':\n",
    "        total_results = response['response']['meta']['hits']\n",
    "        if total_results <= 1010:\n",
    "            get_documents_by_page(q, term, total_results)\n",
    "        else:\n",
    "            bd = parser.parse(q.begin_date)\n",
    "            ed = parser.parse(q.end_date)\n",
    "            half = (ed - bd) // 2\n",
    "            \n",
    "            begin_date1 = q.begin_date\n",
    "            end_date1 = (bd + timedelta(half.days)).strftime(\"%Y%m%d\")\n",
    "            q1 = Query(q.term, begin_date1, end_date1, 0, q.api_key)\n",
    "            get_documents_by_query(q1, term)\n",
    "            \n",
    "            begin_date2 = (bd + timedelta(half.days + 1)).strftime(\"%Y%m%d\")\n",
    "            end_date2 = q.end_date\n",
    "            q2 = Query(q.term, begin_date2, end_date2, 0, q.api_key)\n",
    "            get_documents_by_query(q2, term)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def download_by_date_ranges(term, api_key):\n",
    "    for r in date_ranges(BEGIN_DATE, END_DATE, 1):\n",
    "        begin_date = r[0].strftime(\"%Y%m%d\")\n",
    "        end_date = r[1].strftime(\"%Y%m%d\")\n",
    "        q = Query(term[0], begin_date, end_date, 0, api_key)\n",
    "        get_documents_by_query(q, term)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def download_by_term(term, api_key):\n",
    "    begin_date = BEGIN_DATE.strftime(\"%Y%m%d\")\n",
    "    end_date = END_DATE.strftime(\"%Y%m%d\")\n",
    "    q = Query(term[0], begin_date, end_date, 0, api_key)\n",
    "    response = wait(search, q)\n",
    "    \n",
    "    if response['status'] == 'OK':\n",
    "        total_results = response['response']['meta']['hits']\n",
    "        if total_results <= 1010:\n",
    "            get_documents_by_page(q, term, total_results)\n",
    "        else:\n",
    "            download_by_date_ranges(term, api_key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def download_by_key(terms, api_key):\n",
    "    for term in terms:\n",
    "        try:\n",
    "            download_by_term(term, api_key)\n",
    "        except Exception as e:\n",
    "            write_log(e, status='DOWNLOAD EXCEPTION {} {}'.format(term, api_key))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def downloader(search_terms_by_api_key, api_keys):\n",
    "    # Version parallel\n",
    "    Parallel(n_jobs=8)(delayed(download_by_key)(search_terms_by_api_key[api_key], api_key) for api_key in api_keys)\n",
    "\n",
    "    # Version sequencial\n",
    "#     for api_key in api_keys:\n",
    "#         download_by_key(search_terms_by_api_key[api_key], api_key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "downloader(search_terms_by_api_key, api_keys)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}