ESTGA_extrai_dados_da_Bib_Recomendada_por_UC.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyN4Vifr2dGIdoa6LYv1pLDh",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/fmbento/Jupyter-Notebooks_ad-hoc-solutions/blob/main/ESTGA_extrai_dados_da_Bib_Recomendada_por_UC.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!touch URLs_ESTGA_Lic_UCs.txt"
      ],
      "metadata": {
        "id": "V5O0sqi34MjX"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "eYFSkp2CuwLl"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "!pip install selenium\n",
        "!apt-get update # to update ubuntu to correctly run apt install\n",
        "!apt install -y chromium-chromedriver\n",
        "!cp /usr/lib/chromium-browser/chromedriver /usr/bin\n",
        "!pip install webdriver_manager\n",
        "\n",
        "import sys\n",
        "sys.path.insert(0,'/usr/lib/chromium-browser/chromedriver')\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Fazer paste dos links das UCs no ficheiro criado acima, enquanto a célula acima corre -- não é preciso esperar que ela termine."
      ],
      "metadata": {
        "id": "J4mJ7KKMNkvW"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from selenium import webdriver\n",
        "from selenium.webdriver.chrome.service import Service\n",
        "from webdriver_manager.chrome import ChromeDriverManager\n",
        "from selenium.webdriver.common.by import By\n",
        "from selenium.webdriver.chrome.options import Options\n",
        "from bs4 import BeautifulSoup\n",
        "\n",
        "import csv\n",
        "import time\n",
        "from google.colab import files\n",
        "\n",
        "import re\n",
        "\n",
        "# Setup chrome options\n",
        "chrome_options = Options()\n",
        "chrome_options.add_argument('--headless')\n",
        "chrome_options.add_argument('--no-sandbox')\n",
        "chrome_options.add_argument('--disable-dev-shm-usage')\n",
        "\n",
        "# Function to extract information from a single page\n",
        "def extract_info(url):\n",
        "    driver = webdriver.Chrome(options=chrome_options)\n",
        "    driver.get(url)\n",
        "\n",
        "    docente = \"\"\n",
        "    codigo = \"\"\n",
        "\n",
        "    # wait = WebDriverWait(driver, 10)  # wait for up to 10 seconds\n",
        "    time.sleep(5)  # wait for 10 seconds -- safe value, despite 2 seconds did work initially\n",
        "\n",
        "    UC = driver.find_element(By.XPATH, \"//h1\").text\n",
        "\n",
        "\n",
        "    # Parse the page source with BeautifulSoup\n",
        "    soup = BeautifulSoup(driver.page_source, 'html.parser')\n",
        "\n",
        "    # Find the <H3> tag with the desired text and get the next <a>\n",
        "    h3_tag_doc = soup.find('h3', string='Docente responsável:')\n",
        "    try:\n",
        "        docente = h3_tag_doc.find_next_sibling('a').text\n",
        "    except Exception as e:\n",
        "        print(f\"Sem Docente: {e} - \" + url)\n",
        "\n",
        "\n",
        "    # Find the <H3> tag with the desired text and get the next <div>\n",
        "    h3_tag_cod = soup.find('h3', string='Código:')\n",
        "    codigo = h3_tag_cod.find_next_sibling('div').text\n",
        "\n",
        "\n",
        "    print(docente)\n",
        "    print(codigo)\n",
        "\n",
        "\n",
        "    # Find the <H2> tag with the desired text and get the next <div>\n",
        "    h2_tag = soup.find('h2', string='Bibliografia recomendada')\n",
        "\n",
        "    bibrec = \"\"\n",
        "    try:\n",
        "        next_div = h2_tag.find_next_sibling('div')\n",
        "    except Exception as e:\n",
        "        print(f\"Sem bib Rec: {e} - \" + url)\n",
        "\n",
        "    # Extract and print the text within the <div>\n",
        "    try:\n",
        "        bibrec = next_div.text\n",
        "    except Exception as e:\n",
        "        print()\n",
        "    print(bibrec)\n",
        "    print(\"----------------------------------------------------\")\n",
        "\n",
        "    driver.quit()\n",
        "\n",
        "    return UC,docente,codigo,bibrec,url\n",
        "\n",
        "# Read URLs from file and extract information\n",
        "with open('URLs_ESTGA_Lic_UCs.txt', 'r') as url_file, open('ESTGA_UCs_BibRec.csv', 'w') as output_file:\n",
        "    csv_writer = csv.writer(output_file)\n",
        "    csv_writer.writerow([\"UC\",\"docente\",\"codigo\",\"BibRec\",\"URL\"])  # Write header\n",
        "\n",
        "    for url in url_file:\n",
        "        UC, docente, codigo, bibrec, url = extract_info(url.strip())\n",
        "        csv_writer.writerow([UC,docente,codigo,bibrec,url])\n",
        "\n",
        "\n",
        "# AutoDonwloads 'SAUNE_DirsCursos.csv'\n",
        "files.download('ESTGA_UCs_BibRec.csv')"
      ],
      "metadata": {
        "id": "OectXZAsxWgM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Zs_3RQNpbx_Z"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## E é isto... feito!\n",
        "\n",
        "Abaixo é código solto para a tentativa de fazer o parsing das referências bibliográficas -- não funciona."
      ],
      "metadata": {
        "id": "Vw-RyYPsNIC8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "       # Expressão regular para extrair os metadados da referencias bib em bibrec\n",
        "        regex = r\"^(.*), \\((\\d{4})\\)\\. (.*). (\\d+ª Edição), (.*)\\.$\"\n",
        "\n",
        "        # Extrai os metadados\n",
        "        referencias = bibrec.splitlines()\n",
        "        for ref in referencias:\n",
        "            # print(ref)\n",
        "            match = re.match(r'^(.*), \\((\\d{4})\\)\\. (.*), (.*)$', ref)\n",
        "            if match:\n",
        "                groups = match.groups()\n",
        "                authors = groups[0]\n",
        "                year = groups[1]\n",
        "                title = groups[2]\n",
        "                rest = groups[3]\n",
        "\n",
        "                edition_publisher = rest.rsplit(',', 1)\n",
        "                if len(edition_publisher) == 2:\n",
        "                    edition, publisher = edition_publisher\n",
        "                else:\n",
        "                    edition = edition_publisher[0]\n",
        "                    publisher = ''\n",
        "\n",
        "                print(f\"Autores: {authors}\")\n",
        "                print(f\"Ano: {year}\")\n",
        "                print(f\"Título: {title}\")\n",
        "                print(f\"Edição: {edition.strip()}\")\n",
        "                print(f\"Editora: {publisher.strip()}\")\n",
        "                print()"
      ],
      "metadata": {
        "id": "tXS7WU7MGuSg"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}