scrape.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd      \n",
    "import numpy as np\n",
    "import requests\n",
    "from bs4 import BeautifulSoup, element\n",
    "\n",
    "\n",
    "\n",
    "base_url = \"https://builtwith.com\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrape_fun(url_list):\n",
    "    survey_tech_list = []\n",
    "    for url in url_list:\n",
    "        current_url = base_url + '/' + url\n",
    "        \n",
    "        res = requests.get(current_url)\n",
    "        soup = BeautifulSoup(res.content, \"html.parser\")\n",
    "        divs = soup.findAll(\"div\", {\"class\": \"card-body pb-0\"})\n",
    "\n",
    "        for div in divs:\n",
    "            elt = div.findAll(\"a\")[0]\n",
    "            survey_tech_list.append({'url': url, 'tech': elt.text ,'href': elt.attrs[\"href\"] })\n",
    "\n",
    "    return survey_tech_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "NCsurvey_df = pd.read_csv('NCsurvey.csv')\n",
    "url_list = list(filter(lambda x: x==x, NCsurvey_df.iloc[2,2:].values))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "tech_list = scrape_fun(url_list[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = requests.get('https://builtwith.com/arletanc.org')\n",
    "soup = BeautifulSoup(res.content, \"html.parser\")\n",
    "divs = soup.findAll(\"div\", {\"class\": \"card-body pb-0\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Widgets'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "divs[0].findAll(\"h6\", {\"class\": \"card-title\"})[0].text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<a class=\"text-dark\" href=\"https://trends.builtwith.com/widgets/Trumba\">Trumba</a>,\n",
       " <a class=\"text-dark\" href=\"https://trends.builtwith.com/widgets/TablePress\">TablePress</a>,\n",
       " <a class=\"text-dark\" href=\"https://trends.builtwith.com/widgets/Google-Font-API\">Google Font API</a>,\n",
       " <a class=\"text-dark\" href=\"https://trends.builtwith.com/widgets/Google-Translate-Widget\">Google Translate Widget</a>,\n",
       " <a class=\"text-dark\" href=\"https://trends.builtwith.com/widgets/Jetpack\">Jetpack</a>,\n",
       " <a class=\"text-dark\" href=\"https://trends.builtwith.com/widgets/Gravatar-Profiles\">Gravatar Profiles</a>,\n",
       " <a class=\"text-dark\" href=\"https://trends.builtwith.com/widgets/Wordpress-Plugins\">Wordpress Plugins</a>,\n",
       " <a class=\"text-dark\" href=\"https://trends.builtwith.com/widgets/Shutterstock\">Shutterstock</a>,\n",
       " <a class=\"text-dark\" href=\"https://trends.builtwith.com/widgets/Google-Language-Translator\">Google Language Translator</a>,\n",
       " <a class=\"text-dark\" href=\"https://trends.builtwith.com/widgets/MailChimp-for-WordPress\">MailChimp for WordPress</a>,\n",
       " <a class=\"text-dark\" href=\"https://trends.builtwith.com/widgets/COVID-19\">COVID-19</a>,\n",
       " <a class=\"text-dark\" href=\"https://trends.builtwith.com/widgets/Simple-Social-Icons\">Simple Social Icons</a>,\n",
       " <a class=\"text-dark\" href=\"https://trends.builtwith.com/widgets/Twitter-Tweet-Button\">Twitter Tweet Button</a>]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "divs[0].findAll(\"a\", {\"class\": \"text-dark\"})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = requests.get('https://www.xml-sitemaps.com/details-arletanc.org-8eb84a758.html')\n",
    "soup = BeautifulSoup(res.content, \"html.parser\")\n",
    "long_list = soup.findAll(\"div\", {\"class\": \"long-list\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https'"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "long_list[0].findAll('td')[0].text[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "63fd5069d213b44bf678585dea6b12cceca9941eaf7f819626cde1f2670de90d"
  },
  "kernelspec": {
   "display_name": "Python 3.9.4 64-bit",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}