match_id_piece.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import numpy as np\n",
    "import os\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import pandas as pd\n",
    "import zipfile\n",
    "import pickle\n",
    "import subprocess\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "import urllib.parse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIRECTORY_PATH = \"/mnt/data0/Datasets/imslp/score_scrape/results/composer\"\n",
    "\n",
    "def find_all_folders(directory_path):\n",
    "    folders = []\n",
    "    for root, dirs, files in os.walk(directory_path):\n",
    "        for dir_name in dirs:\n",
    "            folders.append(os.path.join(root, dir_name))\n",
    "    return folders\n",
    "\n",
    "def find_second_level_folders(directory_path):\n",
    "    second_level_folders = []\n",
    "    for root, dirs, files in os.walk(directory_path):\n",
    "        # Calculate the depth by counting the number of separators in the path\n",
    "        depth = root[len(directory_path):].count(os.sep)\n",
    "        if depth == 1:  # We're looking for directories at depth 1 (second level)\n",
    "            for dir_name in dirs:\n",
    "                second_level_folders.append(os.path.join(root, dir_name))\n",
    "    return second_level_folders\n",
    "\n",
    "folders = find_all_folders(DIRECTORY_PATH)\n",
    "second_level_folders = find_second_level_folders(DIRECTORY_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/mnt/data0/Datasets/imslp/score_scrape/results/composer/Schrammel,_Johann/Wien_bleibt_Wien!_/397957.pdf',\n",
       " '/mnt/data0/Datasets/imslp/score_scrape/results/composer/Schrammel,_Johann/Wien_bleibt_Wien!_/414872.pdf',\n",
       " '/mnt/data0/Datasets/imslp/score_scrape/results/composer/Schrammel,_Johann/Wien_bleibt_Wien!_/450459.pdf',\n",
       " '/mnt/data0/Datasets/imslp/score_scrape/results/composer/Schrammel,_Johann/Wien_bleibt_Wien!_/420122.pdf']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def find_files(directory_path):\n",
    "    files = []\n",
    "    for root, dirs, file_names in os.walk(directory_path):\n",
    "        for file_name in file_names:\n",
    "            if file_name.endswith('.pdf'):\n",
    "                files.append(os.path.join(root, file_name))\n",
    "    return files\n",
    "\n",
    "find_files(second_level_folders[25])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 129607/129607 [00:01<00:00, 76764.95it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "420264"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find all files in the second level folders\n",
    "all_files = []\n",
    "for folder in tqdm(second_level_folders):\n",
    "    all_files += find_files(folder)\n",
    "    \n",
    "len(all_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'220875': ('Frescobaldi, Girolamo', 'Recercar sopra Sol, mi, fa, la, sol')}"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def match_id_to_composer_and_title(path):\n",
    "    split_path = path.split('/')\n",
    "    composer = split_path[-3]\n",
    "    composer = urllib.parse.unquote(composer).replace('_', ' ')\n",
    "    \n",
    "    title = split_path[-2]\n",
    "    title = urllib.parse.unquote(title).replace('_', ' ')[:-1]\n",
    "    # check if title contains comma and number\n",
    "    if ',' in title and any(char.isdigit() for char in title):\n",
    "        title = \",\".join(title.split(\",\")[:-1])\n",
    "    \n",
    "    pdf_id = split_path[-1].split('.')[0]\n",
    "    return {pdf_id: (composer, title)}\n",
    "    \n",
    "match_id_to_composer_and_title(find_files(second_level_folders[235])[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 420264/420264 [00:01<00:00, 250745.95it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "364007"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# match all files to their composer and title\n",
    "file_to_composer_and_title = {}\n",
    "for file in tqdm(all_files):\n",
    "    file_to_composer_and_title.update(match_id_to_composer_and_title(file))\n",
    "    \n",
    "len(file_to_composer_and_title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the dictionary\n",
    "with open('/home/ctang/ttmp/MMCSR/dataset_metadata/id_composer_title.pkl', 'wb') as f:\n",
    "    pickle.dump(file_to_composer_and_title, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/home/ctang/ttmp/MMCSR/pretraining_metadata/arr_piano_metadata.pkl', 'rb') as f:\n",
    "    arr_piano_metadata = pickle.load(f)\n",
    "    \n",
    "with open('/home/ctang/ttmp/MMCSR/pretraining_metadata/harpsichord_metadata.pkl', 'rb') as f:\n",
    "    harpsichord_metadata = pickle.load(f)\n",
    "    \n",
    "with open('/home/ctang/ttmp/MMCSR/pretraining_metadata/solo_piano_metadata.pkl', 'rb') as f:\n",
    "    solo_piano_metadata = pickle.load(f)\n",
    "    \n",
    "with open('/home/ctang/ttmp/MMCSR/pretraining_metadata/keyboard_metadata.pkl', 'rb') as f:\n",
    "    keyboard_metadata = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add up all the metadata\n",
    "all_metadata = {**arr_piano_metadata, **harpsichord_metadata, **solo_piano_metadata, **keyboard_metadata}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Work Title': '8 Intermezzi',\n",
       " 'Composer': 'Sinding, Christian',\n",
       " 'Opus/Catalogue Number': 'Op.72',\n",
       " 'I-Catalogue Number': 'ICS 28',\n",
       " 'Movements/Sections': '8 pieces',\n",
       " 'Composer Time Period': 'Romantic',\n",
       " 'Piece Style': 'Romantic',\n",
       " 'Instrumentation': 'Piano',\n",
       " 'url': 'https://imslp.org/wiki/8_Intermezzi,_Op.72_(Sinding,_Christian)',\n",
       " 'Alternative Title': '',\n",
       " 'First Publication': '.\\n1905 (ca.)'}"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c,p = ('Sinding, Christian', '8 Intermezzi, Op.65')\n",
    "p = p.split(',')[0]\n",
    "all_metadata[c][p]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save all metadata to disk\n",
    "with open('/home/ctang/ttmp/MMCSR/pretraining_metadata/all_metadata.pkl', 'wb') as f:\n",
    "    pickle.dump(all_metadata, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "baselines",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}