4. Comparing Full Text.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparing Full Text "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import and setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import json, requests, math, re, string, nltk\n",
    "\n",
    "# allow matplotlib to run in-line\n",
    "% matplotlib inline \n",
    "\n",
    "nltk.download(\"punkt\") # Word tokenizer\n",
    "nltk.download(\"stopwords\") # Stop words\n",
    "from nltk import word_tokenize\n",
    "\n",
    "ocUrl = 'https://open.library.ubc.ca/'\n",
    "ocApiUrl = 'https://oc-index.library.ubc.ca' # APPY URL"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set API Key"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get your own API key at: https://open.library.ubc.ca/research"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "apiKey = 'ac40e6c2cb345593ed1691e0a8b601bba398e42d85f81f893c5ab709cec63c6c'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "collection = '24'\n",
    "itemOneID = '1.0053673'\n",
    "itemTwoID = '1.0069900'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fullText = []\n",
    "\n",
    "# Get first item\n",
    "itemUrl = ocApiUrl+'/collections/'+collection+'/items/'+itemOneID+'?apiKey='+apiKey\n",
    "apiResponse = requests.get(itemUrl).json()\n",
    "item = apiResponse['data']\n",
    "itemNameOne = item['Title'][0]['value']\n",
    "\n",
    "### Clean text ###\n",
    "cleanFullText = item['FullText'][0]['value'].lower()\n",
    "pattern = re.compile('[\\W_]+')\n",
    "cleanFullText = pattern.sub(' ', cleanFullText)\n",
    "\n",
    "fullText.append(cleanFullText)\n",
    "\n",
    "### Get second item ###\n",
    "itemUrl = ocApiUrl+'/collections/'+collection+'/items/'+itemTwoID+'?apiKey='+apiKey\n",
    "apiResponse = requests.get(itemUrl).json()\n",
    "item = apiResponse['data']\n",
    "itemNameTwo = item['Title'][0]['value']\n",
    "\n",
    "# Clean text\n",
    "cleanFullText = item['FullText'][0]['value'].lower()\n",
    "pattern = re.compile('[\\W_]+')\n",
    "cleanFullText = pattern.sub(' ', cleanFullText)\n",
    "\n",
    "fullText.append(cleanFullText)\n",
    "\n",
    "print(itemNameOne)\n",
    "print(ocUrl+'collections/'+collection+'/items/'+itemOneID)\n",
    "print(\"vs\")\n",
    "print(itemNameTwo)\n",
    "print(ocUrl+'collections/'+collection+'/items/'+itemTwoID)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare character counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(\"Full text 1 character count: \"+str(len(fullText[0])))\n",
    "print(\"Full text 2 character count: \"+str(len(fullText[1])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenize full text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "words = []\n",
    "words.append(word_tokenize(fullText[0]))\n",
    "words.append(word_tokenize(fullText[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare word counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "print(\"Full text 1 word count: \"+str(len(words[0])))\n",
    "print(\"Full text 2 word count: \"+str(len(words[1])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare unique word counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(\"Full text 1 word count: \"+str(len(set(words[0]))))\n",
    "print(\"Full text 2 word count: \"+str(len(set(words[1]))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Collocations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "text = nltk.Text(words[0])\n",
    "print(\"Full text 1:\")\n",
    "text.collocations()\n",
    "print(\"--------------------\")\n",
    "print(\"Full text 2:\")\n",
    "print(\"--------------------\")\n",
    "text = nltk.Text(words[1])\n",
    "text.collocations()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Frequency Distributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from nltk import FreqDist\n",
    "allWords = words[0] + words[1]\n",
    "textOne = nltk.Text(words[0])\n",
    "textTwo = nltk.Text(words[1])\n",
    "fdistOne = FreqDist(textOne)\n",
    "fdistTwo = FreqDist(textTwo)\n",
    "fdistOne.plot(25)\n",
    "fdistTwo.plot(25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Total Dispersion Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "text.dispersion_plot([\"social\", \"gender\", \"relationship\", \"with\", \"they\", \"their\", \"there\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Comparing time it would take to read"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If top readers read at speeds of above 1000 words per minute (wpm) with near 85% comprehension, they only represent 1% of readers.\n",
    "\n",
    "Average readers are the majority and only reach around 200 wpm with a typical comprehension of 60%.\n",
    "\n",
    "[Source: http://www.readingsoft.com/ (questionable but eh ) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "topReadTimeOne = round(((len(words[0])/1000)/60), 2)\n",
    "topReadTimeTwo = round(((len(words[1])/1000)/60), 2)\n",
    "\n",
    "avgReadTimeOne = round(((len(words[0])/200)/60), 2)\n",
    "avgReadTimeTwo = round(((len(words[1])/200)/60), 2)\n",
    "\n",
    "print(\"It would take a top reader roughly \"+str(topReadTimeOne)+\" hours to read the first item\")\n",
    "print(\"It would take a top reader roughly \"+str(topReadTimeTwo)+\" hours to read the second item\")\n",
    "print(\"\\n\")\n",
    "print(\"It would take the average reader roughly \"+str(avgReadTimeOne)+\" hours to read the first item\")\n",
    "print(\"It would take the average reader roughly \"+str(avgReadTimeTwo)+\" hours to read the second item\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Which is more positive? "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sentiment Analysis is the process of determining whether a piece of writing is positive, negative or neutral. It's also known as opinion mining, deriving the opinion or attitude of a speaker. A common use case for this technology is to discover how people feel about a particular topic."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Load words into dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "adj = 'dictionaries/adj.txt'\n",
    "adv = 'dictionaries/adv.txt'\n",
    "noun = 'dictionaries/noun.txt'\n",
    "verb = 'dictionaries/verb.txt'\n",
    "\n",
    "sent = dict()\n",
    "import codecs\n",
    "\n",
    "with codecs.open(adj, \"r\",encoding='utf-8', errors='ignore') as fdata:    \n",
    "    for line in fdata:\n",
    "       (key, val) = line.split()\n",
    "       sent[key] = val\n",
    "\n",
    "with codecs.open(adv, \"r\",encoding='utf-8', errors='ignore') as fdata:    \n",
    "    for line in fdata:\n",
    "       (key, val) = line.split()\n",
    "       sent[key] = val\n",
    "        \n",
    "with codecs.open(noun, \"r\",encoding='utf-8', errors='ignore') as fdata:    \n",
    "    for line in fdata:\n",
    "       (key, val) = line.split()\n",
    "       sent[key] = val\n",
    "        \n",
    "with codecs.open(verb, \"r\",encoding='utf-8', errors='ignore') as fdata:    \n",
    "    for line in fdata:\n",
    "       (key, val) = line.split()\n",
    "       sent[key] = val"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Score each word in each text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "firstScore = 0\n",
    "for word in words[0]:\n",
    "    if word in sent:\n",
    "        firstScore += int(sent[word])\n",
    "\n",
    "secondScore = 0\n",
    "for word in words[1]:\n",
    "    if word in sent:\n",
    "        secondScore += int(sent[word])\n",
    "        \n",
    "print(itemNameOne+\" has a score of :::: \"+str(firstScore))\n",
    "print(itemNameTwo+\" has a score of :::: \"+str(secondScore))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}