find_image_id.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook finds all the image IDs of an IMSLP url given "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ids(url):\n",
    "    '''\n",
    "    Given a url, returns a dictionary of pdf ids and their corresponding urls.\n",
    "    '''\n",
    "    try:\n",
    "        response = requests.get(url)\n",
    "        html = response.text\n",
    "\n",
    "        # find all strings of the form \"id='num-of-ratings-[0-9]+'\" under the \"Scores\" section and extract the number\n",
    "        html = html[html.find('id=\"Sheet_Music\"'):]\n",
    "\n",
    "\n",
    "        pdf_id = re.findall(r'>#(\\d+)</a>', html)\n",
    "        pdf_dict = {}\n",
    "        for i in pdf_id:\n",
    "            pdf_dict[i] = url\n",
    "        return pdf_dict\n",
    "    except:\n",
    "        return {}\n",
    "\n",
    "def get_multiple_ids(urls):\n",
    "    '''\n",
    "    Given a list of urls, returns a dictionary of pdf ids and their corresponding urls.\n",
    "    '''\n",
    "    pdf_dict = {}\n",
    "    for url in urls:\n",
    "        pdf_dict.update(get_ids(url))\n",
    "    return pdf_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# retrive urls\n",
    "with open(\"9_way_urls_for_all_scores.txt\", \"r\") as file:\n",
    "    urls_9 = [line.strip() for line in file.readlines()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# retrive urls\n",
    "with open(\"100_way_urls_for_all_scores.txt\", \"r\") as file:\n",
    "    urls_100 = [line.strip() for line in file.readlines()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get all the pdf ids from the urls\n",
    "pdf_dict_9 = get_multiple_ids(urls_9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "pdf_dict_100 = get_multiple_ids(urls_100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_dict_to_pickle(dictionary, filename):\n",
    "    with open(filename, 'wb') as f:\n",
    "        pickle.dump(dictionary, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_dict_to_pickle(pdf_dict_9, '9_way_pdf_dict.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_dict_to_pickle(pdf_dict_100, '100_way_pdf_dict.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.1.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}