1_Webscraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# WebScarping "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " # Import Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from urllib.request import urlopen\n",
    "from bs4 import BeautifulSoup\n",
    "from collections import defaultdict\n",
    "import re\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = \"https://fangj.github.io/friends/\"\n",
    "html = urlopen(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bs4.BeautifulSoup"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup = BeautifulSoup(html, 'lxml')\n",
    "type(soup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print out the text\n",
    "text = soup.get_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "season_list = text.split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "episodes =[l for l in season_list if l[0:3].isdigit()];\n",
    "pattern1 ='\\d{3}\\s'\n",
    "pattern2 = '\\d{3}-\\d{3}'\n",
    "episode_numbers = [l [0:3] if re.match(pattern1,l) else l[0:7] for l in episodes]\n",
    "episode_names = [l[4:] if re.match(pattern1,l) else l[8:] for l in episodes]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Except for season 724, Most common url formats are addressed as 3 digit or 7 digit format. we will do try and except for that one episode so that we can continue the loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "all_transcripts =   {'Episodes':[], 'Episode_Names':[],'Characters': [],'Lines': []} \n",
    "for episode in episode_numbers:\n",
    "    try:\n",
    "        # to extract url that has 3 digit as the number of the episode (e.g. 101)\n",
    "        if len(episode) == 3:\n",
    "            url1 = 'https://fangj.github.io/friends/season/0{0}.html'.format(str(episode))\n",
    "            html = urlopen(url1)\n",
    "        else:\n",
    "            # to extract url that has 7 digit as the number of the episode (e.g. 212-213)\n",
    "            url2 = 'https://fangj.github.io/friends/season/0{0}-0{1}.html'.format(episode[0:3],episode[4:])\n",
    "            html = urlopen(url2)  \n",
    "    # only one  url  of all the episodes had a diferent format than the above two in the url, ignore it\n",
    "    except Exception:\n",
    "        continue\n",
    "            \n",
    "    #parse the html\n",
    "    soup = BeautifulSoup(html, 'lxml')\n",
    "    #identify html tags and attributes associated with actors and their dialogues in the script\n",
    "    transcripts = soup.find_all('p')\n",
    "\n",
    "     # Print text only, and further filter tags to fin Charcater: Lines\n",
    "    pattern = '^\\w+[:]'   \n",
    "    transcripts_text_only = [p.text for p in transcripts if re.search(pattern,p.text) ]\n",
    "\n",
    "    #split Characters and Lines\n",
    "    Characters_this_episode = [l.split(\":\") [0] for l in transcripts_text_only]\n",
    "    Lines_this_episode = [l.split(\":\") [1] for l in transcripts_text_only]\n",
    "    \n",
    "    #store episode number and episode name for each line\n",
    "    Episode = [episode]*len(Characters_this_episode) \n",
    "    Episode_Name = [episode_names[episode_numbers.index(episode)]]*len(Characters_this_episode)\n",
    "    \n",
    "    all_transcripts['Episodes'].append(Episode)\n",
    "    all_transcripts['Episode_Names'].append(Episode_Name)\n",
    "    all_transcripts['Characters'].append(Characters_this_episode)\n",
    "    all_transcripts['Lines'].append(Lines_this_episode)\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(all_transcripts['Episodes'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#Each dictionary has list of lists as values. Flatten them so that each key has a single list\n",
    "import itertools\n",
    "all_transcripts['Episodes']= list (itertools.chain.from_iterable(  all_transcripts['Episodes']))\n",
    "all_transcripts['Episode_Names']= list (itertools.chain.from_iterable(  all_transcripts['Episode_Names']))\n",
    "all_transcripts['Characters']= list (itertools.chain.from_iterable(  all_transcripts['Characters']))\n",
    "all_transcripts['Lines']= list (itertools.chain.from_iterable(  all_transcripts['Lines']))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48944\n",
      "48944\n",
      "48944\n",
      "48944\n"
     ]
    }
   ],
   "source": [
    "print(len(all_transcripts['Episodes']))\n",
    "print(len(all_transcripts['Episode_Names']))\n",
    "print(len(all_transcripts['Characters']))\n",
    "print(len(all_transcripts['Lines']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Export dictionary of transcripts as JSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "json = json.dumps( all_transcripts)\n",
    "Transcripts = open(\"Friends_transcripts.json\",\"w\")\n",
    "Transcripts.write(json)\n",
    "Transcripts.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}