Person_Identification_Siamese_Network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 208
    },
    "colab_type": "code",
    "id": "EXkkRp3v0cyb",
    "outputId": "f58078a6-299b-40ea-c849-1db0896b25ab"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: librosa in /usr/local/lib/python3.6/dist-packages (0.6.3)\n",
      "Requirement already satisfied: audioread>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from librosa) (2.1.8)\n",
      "Requirement already satisfied: scikit-learn!=0.19.0,>=0.14.0 in /usr/local/lib/python3.6/dist-packages (from librosa) (0.21.3)\n",
      "Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from librosa) (1.3.3)\n",
      "Requirement already satisfied: six>=1.3 in /usr/local/lib/python3.6/dist-packages (from librosa) (1.12.0)\n",
      "Requirement already satisfied: decorator>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from librosa) (4.4.1)\n",
      "Requirement already satisfied: numba>=0.38.0 in /usr/local/lib/python3.6/dist-packages (from librosa) (0.40.1)\n",
      "Requirement already satisfied: resampy>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from librosa) (0.2.2)\n",
      "Requirement already satisfied: joblib>=0.12 in /usr/local/lib/python3.6/dist-packages (from librosa) (0.14.0)\n",
      "Requirement already satisfied: numpy>=1.8.0 in /usr/local/lib/python3.6/dist-packages (from librosa) (1.17.4)\n",
      "Requirement already satisfied: llvmlite>=0.25.0dev0 in /usr/local/lib/python3.6/dist-packages (from numba>=0.38.0->librosa) (0.30.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install librosa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sYQqTRFIrteh"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn, optim\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd.variable import Variable\n",
    "\n",
    "from IPython import display\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "from torch.utils.data.dataset import Dataset \n",
    "from torch.utils.data import DataLoader\n",
    "import librosa\n",
    "\n",
    "from IPython import display\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "ipGxP-rKeC1B",
    "outputId": "8b63ec8f-e57c-4cf4-9b2a-51ee92fcf07f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')\n",
    "\n",
    "train_file_pickle = \"/content/drive/My Drive/Deep Learning Systems/Assignment4Part1Data/hw4_trs.pkl\"\n",
    "test_file_pickle = \"/content/drive/My Drive/Deep Learning Systems/Assignment4Part1Data/hw4_tes.pkl\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ct4jv-lV0yvZ"
   },
   "outputs": [],
   "source": [
    "from itertools import combinations, product\n",
    "from random import sample\n",
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "def getstft(raw):\n",
    "    stft=librosa.stft(raw, n_fft=1024, hop_length=512)\n",
    "    return torch.tensor(np.abs(stft)).float().t()\n",
    "\n",
    "def getstftdatafrompickle(filepath):\n",
    "    data_array = None\n",
    "    \n",
    "    with open(filepath, 'rb') as f:\n",
    "        data_array = pickle.load(f)\n",
    "    stft_arr = []\n",
    "    for item in data_array:\n",
    "        stft_arr.append(getstft(item))\n",
    "    return stft_arr\n",
    "\n",
    "def generate_combinations(numspeak, sample_count):\n",
    "    # This part of combitions genereration was discussed with a friend\n",
    "     positive_pairs = torch.zeros((numspeak, sample_count, 3))\n",
    "     negative_pairs = torch.zeros((numspeak, sample_count, 3))\n",
    "     for speaker in range(numspeak):\n",
    "         fullset = set(range(numspeak*10))\n",
    "         positiveset = set((range(speaker*10, speaker*10+10)))\n",
    "         negativeset = set(fullset-positiveset)\n",
    "         pos_comb = torch.tensor(sample(list(combinations(list(positiveset),2)), sample_count))\n",
    "         neg_comb = torch.tensor(sample(list(product(list(positiveset), list(negativeset))), sample_count))\n",
    "         positive_labels = torch.ones(sample_count, 1, dtype=torch.long)\n",
    "         negative_labels = torch.zeros(sample_count,1, dtype=torch.long)\n",
    "         positive_pairs[speaker] = torch.cat((pos_comb, positive_labels), 1)\n",
    "         negative_pairs[speaker] = torch.cat((neg_comb, negative_labels), 1)\n",
    "     return positive_pairs, negative_pairs\n",
    "\n",
    "class SiameseDataset(Dataset):\n",
    "    def __init__(self, positive_comb, negative_comb, datafile):\n",
    "        pos_pair, neg_pair = generate_combinations(50, 45)\n",
    "        all_pairs = torch.cat((pos_pair, neg_pair), 0)\n",
    "        all_pairs = all_pairs.reshape((2*50*45, 3))\n",
    "        self.samples_indices = all_pairs.int()\n",
    "        self.stft_arr = getstftdatafrompickle(datafile)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.samples_indices)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        #returns first utterance, second utterance and label\n",
    "        return self.stft_arr[self.samples_indices[idx][0]], self.stft_arr[self.samples_indices[idx][1]], self.samples_indices[idx][2]\n",
    "\n",
    "class SiameseDatasetTest(Dataset):\n",
    "    def __init__(self, positive_comb, negative_comb, datafile):\n",
    "        pos_pair, neg_pair = generate_combinations(20, 45)\n",
    "        all_pairs = torch.cat((pos_pair, neg_pair), 0)\n",
    "        all_pairs = all_pairs.reshape((2*20*45, 3))\n",
    "        self.samples_indices = all_pairs.int()\n",
    "        self.stft_arr = getstftdatafrompickle(datafile)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.samples_indices)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        #returns first utterance, second utterance and label\n",
    "        return self.stft_arr[self.samples_indices[idx][0]], self.stft_arr[self.samples_indices[idx][1]], self.samples_indices[idx][2]\n",
    "\n",
    "def get_batch_dot_product(embedding_1_batch, embedding_2_batch):\n",
    "    op = torch.bmm(\n",
    "        embedding_1_batch.view(embedding_1_batch.shape[0], 1, embedding_1_batch.shape[1]),\n",
    "        embedding_2_batch.view(embedding_2_batch.shape[0], embedding_2_batch.shape[1], 1)\n",
    "    )\n",
    "    return op.reshape(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 86
    },
    "colab_type": "code",
    "id": "DJ2TtL3Clf52",
    "outputId": "034ee81d-49da-4925-e14a-4d0ef0a97611"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Net(\n",
      "  (lstm): LSTM(513, 513, num_layers=2, batch_first=True)\n",
      "  (lin1): Linear(in_features=513, out_features=100, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "positive_comb, negative_comb = generate_combinations(50, 45)\n",
    "ds = SiameseDataset(positive_comb, negative_comb, train_file_pickle) \n",
    "train_loader = DataLoader(ds, batch_size=45, shuffle=True, num_workers=2)\n",
    "\n",
    "class Net(nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        self.lstm = nn.LSTM(513, 513, 2, batch_first=True)\n",
    "        self.lin1 = nn.Linear(513, 100)\n",
    "\n",
    "\n",
    "    def forward(self, x):\n",
    "        x,_ = self.lstm(x)\n",
    "        x = x[:, -1, :]\n",
    "        x = self.lin1(x)\n",
    "        return x\n",
    "\n",
    "net = Net()\n",
    "print(net)\n",
    "\n",
    "learning_rate = 5e-4\n",
    "criterion = torch.nn.BCEWithLogitsLoss()\n",
    "optimizer = optim.Adam(net.parameters(), lr=learning_rate)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 104
    },
    "colab_type": "code",
    "id": "RXyC_EjbgJKu",
    "outputId": "fdaffeba-d775-4483-9fb1-48b653ec38f0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0, Loss: 0.0004076989134773612\n",
      "Epoch: 1, Loss: 3.168747207382694e-05\n",
      "Epoch: 2, Loss: 0.0001128827134380117\n",
      "Epoch: 3, Loss: 2.2173251636559144e-05\n",
      "Epoch: 4, Loss: 0.00013234572543296963\n"
     ]
    }
   ],
   "source": [
    "# for epoch in range(40):\n",
    "for epoch in range(5):\n",
    "    for index, batch in enumerate(train_loader):\n",
    "        uttr1, uttr2, label = batch\n",
    "        optimizer.zero_grad()\n",
    "        embedding1 = net(uttr1)\n",
    "        embedding2 = net(uttr2)\n",
    "        op = get_batch_dot_product(embedding1, embedding2)\n",
    "        loss = criterion(op, label.float())\n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        if index%256 == 0:\n",
    "            print(\"Epoch: {}, Loss: {}\".format(epoch, loss.data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "BrseNkyN15le"
   },
   "source": [
    "Though it looks like I have done 5 epochs, i actually did 45 epochs.\n",
    "\n",
    "40 Before this and 5 More to get the network to converge a bit more. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xPtxvrwhL-3t"
   },
   "outputs": [],
   "source": [
    "accuracy_arr = []\n",
    "for acc_test in range(10):\n",
    "    #generate combitions for test data\n",
    "    positive_comb, negative_comb = generate_combinations(20, 45)\n",
    "    #get data from test pickle file and combitions\n",
    "    dstest = SiameseDatasetTest(positive_comb, negative_comb, test_file_pickle)\n",
    "    test_loader = DataLoader(dstest, batch_size=512, shuffle=False, num_workers=2)\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for batch in test_loader:\n",
    "            uttr1, uttr2, label = batch\n",
    "            embedding1 = net(uttr1)\n",
    "            embedding2 = net(uttr2)\n",
    "            op = get_batch_dot_product(embedding1, embedding2)\n",
    "            predictions = torch.sigmoid(op)\n",
    "\n",
    "            for index, val in enumerate(predictions):\n",
    "                #predict 1 if the sigmoid of dot product is more than 0.5\n",
    "                predict = 1 if val > 0.5 else 0\n",
    "                if predict == label[index]:\n",
    "                    correct+=1\n",
    "                total+=1\n",
    "    accuracy_arr.append(correct*100/total)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 86
    },
    "colab_type": "code",
    "id": "XMXpV5tLXRbh",
    "outputId": "443a5be3-ded8-4f8b-990f-2d428e2ef8b8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[68.38888889 67.83333333 68.88888889 67.88888889 66.72222222 69.33333333\n",
      " 68.27777778 67.16666667 68.5        69.05555556]\n",
      "Average Accuracy: 68.20555555555555\n",
      "Max Accuracy: 69.33333333333333, Min Accuracy: 66.72222222222223\n"
     ]
    }
   ],
   "source": [
    "accuracy_arr = np.asarray(accuracy_arr)\n",
    "print(accuracy_arr)\n",
    "print(\"Average Accuracy: {}\".format(np.average(accuracy_arr)))\n",
    "print(\"Max Accuracy: {}, Min Accuracy: {}\".format(np.max(accuracy_arr), np.min(accuracy_arr)))"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "DLAssignment4part1",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}