webpage.py

import os
from bs4 import BeautifulSoup as BS
import requests
from urlparse import urlparse
from urlparse import urljoin

FILE_EXTENSIONS = ["pdf", "doc", "docx", "txt", "ps", "gif", "png", "jpg", "jpeg", "mp4"] 

class WebPage(object):

  def __init__(self, url):
    self.url = urlparse(url)
    self.page = requests.get(url) 
    self.soup = BS(self.page.text, "html.parser")

  def get_assets(self):
    assets = []
    assets += self.get_js()
    assets += self.get_images()
    assets += self.get_stylesheets()
    assets += self.get_files()
    # return unique set of urls ( assets can be repeated on same page )
    return list(set(map(lambda a: a.geturl(), assets)))

  def __get_url(self, element, attribute, keep_fragments=True):
    resources = []
    resources += self.soup.find_all(element)
    url_strings = filter(lambda val: val is not None, map(lambda res: res.get(attribute), resources))
    
    # parse url as well as convert relative url to absolute url
    def parse_url(url_str):
      # TODO handle tel:11234567890, mailto:someone@somewhere.com
      absolute_url = urljoin(self.page.url, url_str) 

      if not keep_fragments:
        # get rid of fragment
        absolute_url = absolute_url.split("#")[0]

      if 'DEBUG' in os.environ and url_str != absolute_url:
        print "converted: ", url_str, " --> ", absolute_url
      return urlparse(absolute_url) 

    urls = map(parse_url, url_strings)
    return urls

  def get_js(self):
    return self.__get_url('script', 'src')

  def get_images(self):
    return self.__get_url('img', 'src')

  def get_links(self):
    return self.__get_url('link', 'href')

  def get_anchors(self, keep_fragments=True):
    return self.__get_url('a', 'href', keep_fragments)

  def get_stylesheets(self):
    stylesheets = []
    stylesheets += self.soup.find_all('link')
    def sytlesheet_filter(link):
      rel = link.get('rel')[0]      
      if rel == "stylesheet" or rel == "" or rel is None:
        return True
      return False    

    stylesheets = filter(sytlesheet_filter, stylesheets)
    url_strings = map(lambda ss: ss.get('href'), stylesheets)
    urls = map(lambda ss: urlparse(urljoin(self.page.url, ss)), url_strings)
    return urls

  def get_files(self):
    all_anchors = self.get_anchors()
    
    def filter_files(f):
      if f.geturl().split(".")[-1] in FILE_EXTENSIONS: 
        return True
      return False
  
    return filter(filter_files, all_anchors)