loop_detecton.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import random\n",
    "import fluidsynth\n",
    "import pretty_midi\n",
    "import numpy as np\n",
    "\n",
    "from time import time\n",
    "from scipy import stats\n",
    "from scipy.special import boxcox, inv_boxcox\n",
    "\n",
    "import IPython.display\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import pytorch_lightning as pl\n",
    "\n",
    "from torch.utils.data import DataLoader\n",
    "from pytorch_lightning.callbacks import ModelCheckpoint\n",
    "\n",
    "from utils.data import *\n",
    "from utils.model import *\n",
    "from utils.common_utils import *\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "plt.style.use('seaborn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# init seed\n",
    "random_seed = 0\n",
    "pl.seed_everything(random_seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize available devices\n",
    "use_cuda = torch.cuda.is_available()\n",
    "device = torch.device('cuda' if use_cuda else 'cpu')\n",
    "print('using', device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = './data/wav_cov.pkl'\n",
    "with open(path, 'rb') as f:\n",
    "    data = pickle.load(f)\n",
    "\n",
    "print('the number of data :', data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shuffle and split\n",
    "random.shuffle(data)\n",
    "\n",
    "num_data = data.shape[0]\n",
    "num_train = int(num_data * 0.8)\n",
    "\n",
    "train_data = data[:num_train]\n",
    "val_data = data[num_train:]\n",
    "\n",
    "print('the number of train :', train_data.shape)\n",
    "print('the number of validation :', val_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataloader\n",
    "batch_size = 64\n",
    "train_params = {'batch_size': batch_size, 'shuffle': True, 'pin_memory': True, 'num_workers': 4}\n",
    "val_params = {'batch_size': batch_size, 'shuffle': False, 'pin_memory': True, 'num_workers': 4}\n",
    "\n",
    "train_set = DataLoader(DatasetSampler(train_data), **train_params)\n",
    "val_set = DataLoader(DatasetSampler(val_data), **val_params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Autoencoder Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model\n",
    "num_features = data.shape[1]\n",
    "AE = AutoEncoder(num_features)\n",
    "checkpoint_callback = ModelCheckpoint(monitor='val_loss',\n",
    "                                      filename='loop-detection-{epoch:02d}-{val_loss:.2f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training\n",
    "trainer = pl.Trainer(gpus=1,\n",
    "                     num_nodes=1,\n",
    "                     max_epochs=1000,\n",
    "                     deterministic=True,\n",
    "                     default_root_dir='./model',\n",
    "                     callbacks=[checkpoint_callback])\n",
    "\n",
    "trainer.fit(AE, train_set, val_set)\n",
    "\n",
    "print('best model path :', checkpoint_callback.best_model_path)\n",
    "print('final results :', trainer.logged_metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # load model\n",
    "# ckpt_path = './model/loop-detection-epoch=927-val_loss=0.02.ckpt'\n",
    "# AE = AE.load_from_checkpoint(ckpt_path, num_features=num_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Center from Autoencoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_z = []\n",
    "\n",
    "AE.eval()\n",
    "with torch.no_grad():\n",
    "    for batch_idx, x_train in enumerate(train_set):\n",
    "        z = AE.encoder(x_train)\n",
    "        train_z.append(z)\n",
    "        \n",
    "center = torch.vstack(train_z).mean(0)\n",
    "print('center shape :', center.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# center = [3.4064, -2.3389, -2.8335, -1.2972, -2.0128, -1.1937, 1.1904]\n",
    "# center = torch.as_tensor(center)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Deep SVDD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# model\n",
    "AE.train()\n",
    "SVDD_model = SVDD(AE.encoder, center.to(device))\n",
    "checkpoint_callback = ModelCheckpoint(monitor='val_loss',\n",
    "                                      filename='SVDD-{epoch:02d}-{val_loss:.2f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training\n",
    "trainer = pl.Trainer(gpus=1,\n",
    "                     num_nodes=1,\n",
    "                     max_epochs=1000,\n",
    "                     deterministic=True,\n",
    "                     default_root_dir='./model',\n",
    "                     callbacks=[checkpoint_callback])\n",
    "\n",
    "trainer.fit(SVDD_model, train_set, val_set)\n",
    "\n",
    "print('best model path :', checkpoint_callback.best_model_path)\n",
    "print('final results :', trainer.logged_metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # load SVDD\n",
    "# ckpt_path = './model/SVDD-epoch=562-val_loss=0.06.ckpt'\n",
    "# SVDD_model = SVDD_model.load_from_checkpoint(ckpt_path, encoder=AE.encoder, center=center.to(device))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dist_from_SVDD(data_set, model, center):\n",
    "    z_set = []\n",
    "    \n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for batch_idx, x in enumerate(data_set):\n",
    "            z = model(x)\n",
    "            z_set.append(z)\n",
    "            \n",
    "    z_set = torch.vstack(z_set)\n",
    "    \n",
    "    # compute distance\n",
    "    dist = z_set - center.unsqueeze(0)\n",
    "    dist = dist.square().mean(1)\n",
    "    dist = dist.cpu().detach().numpy()\n",
    "    \n",
    "    return dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dist = get_dist_from_SVDD(train_set, SVDD_model, center)\n",
    "val_dist = get_dist_from_SVDD(val_set, SVDD_model, center)\n",
    "\n",
    "print('train dist :', train_dist.shape)\n",
    "print('val dist :', val_dist.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loop Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = './data/midi_full_strict_pianoroll.pkl'\n",
    "with open(path, 'rb') as f:\n",
    "    midi = pickle.load(f)\n",
    "\n",
    "print('the number of data :', len(midi))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "midi_dist = []\n",
    "start_time = time()\n",
    "\n",
    "SVDD_model.eval()\n",
    "with torch.no_grad():\n",
    "    for i in range(len(midi)):\n",
    "        pianoroll = midi[i].todense()\n",
    "        \n",
    "        inputs = torch.from_numpy(get_xor_corr(pianoroll))\n",
    "        z = SVDD_model(inputs.float())\n",
    "\n",
    "        # compute distance\n",
    "        dist = z - center\n",
    "        dist = dist.square().mean()\n",
    "        midi_dist.append(dist.item())\n",
    "            \n",
    "        if i % 1e6 == 0:\n",
    "            print('I am on %d (%0.3f sec)' % (i, time()-start_time))\n",
    "            start_time = time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get threshold\n",
    "boxcox_train_dist = boxcox(train_dist, 0)\n",
    "box_cox_thres = np.mean(boxcox_train_dist) + np.std(boxcox_train_dist)\n",
    "thres = inv_boxcox(box_cox_thres, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot parameters\n",
    "CHAR_FONT_SIZE = 17\n",
    "NUM_FONT_SIZE = 15\n",
    "WIDTH = HEIGHT = 8\n",
    "LABEL_PAD = 13\n",
    "\n",
    "# plotting\n",
    "plt.figure(figsize=(WIDTH, HEIGHT))\n",
    "plt.hist(train_dist, facecolor='tab:blue', bins=50, alpha=0.8)\n",
    "plt.hist(val_dist, facecolor='tab:orange', bins=50, alpha=0.4)\n",
    "plt.xticks(fontsize=NUM_FONT_SIZE)\n",
    "plt.yticks(fontsize=NUM_FONT_SIZE)\n",
    "plt.yscale('log')\n",
    "plt.xlabel('Loop Score', fontsize=CHAR_FONT_SIZE, labelpad=LABEL_PAD)\n",
    "plt.ylabel('Frequency', fontsize=CHAR_FONT_SIZE, labelpad=LABEL_PAD)\n",
    "plt.legend(['wav_train', 'wav_val'], fontsize=CHAR_FONT_SIZE)\n",
    "plt.savefig('./images/distance.png', dpi=1000, bbox_inches='tight', pad_inches=0)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plotting\n",
    "plt.figure(figsize=(WIDTH, HEIGHT))\n",
    "plt.hist(midi_dist, bins=50, alpha=0.8)\n",
    "plt.xticks(fontsize=NUM_FONT_SIZE)\n",
    "plt.yticks(fontsize=NUM_FONT_SIZE)\n",
    "plt.yscale('log')\n",
    "plt.xlabel('Loop Score', fontsize=CHAR_FONT_SIZE, labelpad=LABEL_PAD)\n",
    "plt.ylabel('Frequency', fontsize=CHAR_FONT_SIZE, labelpad=LABEL_PAD)\n",
    "plt.legend(['midi'], fontsize=CHAR_FONT_SIZE)\n",
    "# plt.axvline(x=thres, color='tab:red', linestyle='--', linewidth=3.5)\n",
    "plt.savefig('./images/midi_distance.png', dpi=1000, bbox_inches='tight', pad_inches=0)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get Loop from Loop Detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "loop_data = []\n",
    "start_time = time()\n",
    "\n",
    "SVDD_model.eval()\n",
    "with torch.no_grad():\n",
    "    for i in range(len(midi)):\n",
    "        pianoroll = midi[i].todense()\n",
    "        \n",
    "        inputs = torch.from_numpy(get_xor_corr(pianoroll))\n",
    "        z = SVDD_model(inputs.float())\n",
    "\n",
    "        # compute distance\n",
    "        dist = z - center\n",
    "        dist = dist.square().mean().item()\n",
    "\n",
    "        if dist < thres:\n",
    "            loop_data.append(pianoroll)\n",
    "            \n",
    "        if i % 1e6 == 0:\n",
    "            print('I am on %d (%0.3f sec)' % (i, time()-start_time))\n",
    "            start_time = time()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# play demo\n",
    "# random.shuffle(loop_list)\n",
    "for loop in loop_data[0:]:\n",
    "    pm = play_pianoroll(loop, fs=8)\n",
    "    IPython.display.display(IPython.display.Audio(pm.fluidsynth(fs=16000), rate=16000))\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVDD_model.eval()\n",
    "with torch.no_grad():\n",
    "    inputs = torch.from_numpy(get_xor_corr(loop))\n",
    "    z = SVDD_model(inputs.float())\n",
    "\n",
    "    # compute distance\n",
    "    dist = z - center\n",
    "    dist = dist.square().mean().item()\n",
    "    \n",
    "    print(dist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save pickle\n",
    "save_path = './data/midi_detected_strict_pianoroll.pkl'\n",
    "with open(save_path, 'wb') as f:\n",
    "    print('File saved!')\n",
    "    pickle.dump(loop_data, f, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    \n",
    "print('the number of data :', loop_data.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loop Detector Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# play demo\n",
    "while True:\n",
    "    rand_idx = np.random.randint(len(midi))\n",
    "    loop = midi[rand_idx]\n",
    "    \n",
    "    if loop[0, 0] == 1 and loop[0, 6] == 1:\n",
    "        break\n",
    "        \n",
    "pm = play_pianoroll(loop, fs=8)\n",
    "IPython.display.display(IPython.display.Audio(pm.fluidsynth(fs=16000), rate=16000))\n",
    "print(rand_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loop_idx = [5383096, 3978016, 618411, 5186352, 5261371, 5528929, 2975564, 2652515, 1356022, 890982,\n",
    "            3017643, 2272088, 5086286, 725082, 350768, 2283479, 5011746, 3861869, 4733529, 7228,\n",
    "            4402904, 511009, 3499391, 39603, 4082163, 1896876, 3159032, 4534427, 2248818, 4270337,\n",
    "            5648465, 4283338, 516120, 2020602, 1943102, 2959300, 1957977, 5270308, 993345, 2741232,\n",
    "            3634553, 1172096, 4643747, 2389123, 1983880, 2211931, 3795814, 2613799, 5065649, 4723833,\n",
    "            4018700, 578129, 578129, 578129, 1167437, 5043236, 686352, 3127576, 5410844, 4623258,\n",
    "            3977773, 5133501, 3917701, 4792378, 596408, 275077, 5264031, 5574783, 4011830, 2127268,\n",
    "            773880, 5658692, 4213914, 5511406, 1526841, 5292300, 1035078, 836215, 2335737, 282704,\n",
    "            1590806, 2724512, 148811, 1990322, 4261188, 51099, 4190463, 1708669, 920486, 3178536,\n",
    "            1309008, 4529539, 3151077, 3170320, 83849, 5052372, 4776581, 4355603, 1066846, 2585470]\n",
    "\n",
    "print('the number of loop_idx :', len(loop_idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rand_idx = np.random.randint(len(midi), size=400).tolist()\n",
    "eval_idx = loop_idx + rand_idx\n",
    "\n",
    "print('the number of eval_idx :', len(eval_idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_dist = []\n",
    "start_time = time()\n",
    "\n",
    "SVDD_model.eval()\n",
    "with torch.no_grad():\n",
    "    for idx in eval_idx:\n",
    "        pianoroll = midi[idx].todense()\n",
    "        \n",
    "        inputs = torch.from_numpy(get_xor_corr(pianoroll))\n",
    "        z = SVDD_model(inputs.float())\n",
    "\n",
    "        # compute distance\n",
    "        dist = z - center\n",
    "        dist = dist.square().mean()\n",
    "        eval_dist.append(dist.item())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# statistical test\n",
    "for i in range(5):\n",
    "    print(stats.ttest_ind(eval_dist[:100], eval_dist[100*i:100*(i+1)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot parameters\n",
    "CHAR_FONT_SIZE = 17\n",
    "NUM_FONT_SIZE = 14\n",
    "WIDTH = 10; HEIGHT = 5\n",
    "LABEL_PAD = 13\n",
    "\n",
    "# plotting\n",
    "plt.figure(figsize=(WIDTH, HEIGHT))\n",
    "plt.boxplot([eval_dist[:100], eval_dist[100:200], eval_dist[200:300], eval_dist[300:400], eval_dist[400:500]], \n",
    "            vert=False)\n",
    "plt.yticks([1, 2, 3, 4, 5], ['Loop Set', 'Random Set 1', 'Random Set 2', 'Random Set 3', 'Random Set 4'], fontsize=NUM_FONT_SIZE)\n",
    "plt.xticks(fontsize=NUM_FONT_SIZE)\n",
    "plt.xlabel('Loop Score', fontsize=CHAR_FONT_SIZE, labelpad=LABEL_PAD)\n",
    "plt.ylabel('Groups', fontsize=CHAR_FONT_SIZE, labelpad=LABEL_PAD)\n",
    "plt.savefig('./images/loop_score.png', dpi=1000, bbox_inches='tight', pad_inches=0)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}