FinalTask1B.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "FinalTask1B.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FY-kroxEJ1ih",
        "colab_type": "text"
      },
      "source": [
        "### Cloning git repo"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1OB_PCI9LMXZ",
        "colab_type": "code",
        "outputId": "0df2d971-b2bc-4bc7-9161-1d71ef808fcd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "!git clone -l -s https://github.com/WING-NUS/scisumm-corpus.git\n",
        "%cd scisumm-corpus/data/Training-Set-2019/Task1/From-Training-Set-2018/\n",
        "X = !ls"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fatal: destination path 'scisumm-corpus' already exists and is not an empty directory.\n",
            "/content/scisumm-corpus1/data/Training-Set-2019/Task1/From-Training-Set-2018/scisumm-corpus/data/Training-Set-2019/Task1/From-Training-Set-2018\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eRPoxzY94Kgn",
        "colab_type": "text"
      },
      "source": [
        "## Loading Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nYR9_ysZLbPM",
        "colab_type": "code",
        "outputId": "8e6a19f5-50b4-4f31-9bed-d51dd842f8d0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        }
      },
      "source": [
        "import re\n",
        "files = []\n",
        "print(X)\n",
        "for file in X:\n",
        "  Y = re.split(r'[ ,\\t]', file)\n",
        "  files += [y for y in Y if y!=\"\"]\n",
        "\n",
        "print(len(files))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['C00-2123  C90-2039  D10-1083  I05-5011\\tN04-1038  P05-1004  P98-1081  W08-2222', 'C02-1025  C94-2154  E03-1020  J00-3003\\tN06-2049  P05-1053  P98-2143  W09-0621', 'C04-1089  C98-1097  E09-2008  J96-3004\\tN09-1001  P06-2124  W03-0410  W11-0815', 'C08-1098  D09-1023  H05-1115  J98-2005\\tN09-1025  P07-1040  W04-0213  W95-0104', 'C10-1045  D10-1058  H89-2014  N01-1011\\tP00-1025  P98-1046  W06-3909  X96-1048']\n",
            "40\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bq8BBbPg4Vz7",
        "colab_type": "text"
      },
      "source": [
        "### Feature Extraction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "If7frK9HL_P5",
        "colab_type": "code",
        "outputId": "c7c960a4-bdf4-40de-d773-5a8e06327f48",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "from collections import defaultdict\n",
        "import numpy as np\n",
        "DataDict = defaultdict(lambda: {\"C_text\": str, \"R_text\": str , \"Discourse_facet\" : str})\n",
        "c = 0\n",
        "featureDict = {}\n",
        "for rfile in files:\n",
        "  try :\n",
        "    open_file = open(rfile+\"/annotation/\"+rfile+\".ann.txt\", 'r')\n",
        "  except:\n",
        "    open_file = open(rfile+\"/annotation/\"+rfile+\".annv3.txt\", 'r')\n",
        "  citances = open_file.readlines()\n",
        "  citanceNumber = 0\n",
        "  for citance in citances:\n",
        "    citanceNumber = citanceNumber + 1\n",
        "    if len(citance) > 1:\n",
        "      data_file = citance.split(\" | \")\n",
        "      if(len(data_file) == 11):\n",
        "        C_text = (data_file[6].split(\":\")[1])\n",
        "        R_text = (data_file[8].split(\":\")[1])\n",
        "        DataDict[data_file[1].split(\":\")[1]][\"C_text\"] = \" \".join(C_text)\n",
        "        DataDict[data_file[1].split(\":\")[1]][\"R_text\"] = \" \".join(R_text)\n",
        "        sentencepos = []\n",
        "        sectionpos = []\n",
        "        # Five facets \n",
        "        #  Aim, Hypothesis, Implication, Results, Method \n",
        "        mat = []\n",
        "        C_textb = C_text.split(\"</S>\")\n",
        "        for entry in C_textb:\n",
        "          if entry.find(\">\") != -1 :\n",
        "            temp = entry.split(\">\")[0]  \n",
        "            temp2 = temp.split(\"=\")[1]\n",
        "            temp3 = temp.split(\"=\")[2]\n",
        "            sentencepos.append((int)(temp2.split(\"\\\"\")[1]))\n",
        "            sectionpos.append((int)(temp3.split(\"\\\"\")[1]))\n",
        "        if len(sentencepos)<2  :\n",
        "          sentencepos.append(0)\n",
        "          sectionpos.append(0)\n",
        "        for i in range(2) :\n",
        "          vec = []\n",
        "          vec.append(sentencepos[i])\n",
        "          vec.append(sectionpos[i])\n",
        "          mat.append(vec)\n",
        "        mat = np.array(mat)\n",
        "        mat = np.reshape(mat,(4,))\n",
        "        featureDict[(rfile,citanceNumber)]=mat\n",
        "\n",
        "        if(c==0):\n",
        "          c=1;\n",
        "print(featureDict)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{('C00-2123', 1): array([39, 19, 40, 20]), ('C00-2123', 4): array([8, 8, 0, 0]), ('C00-2123', 7): array([43,  1,  0,  0]), ('C00-2123', 10): array([80, 38,  0,  0]), ('C00-2123', 13): array([22, 22,  0,  0]), ('C00-2123', 15): array([21, 21, 22, 22]), ('C00-2123', 18): array([113,  16,   0,   0]), ('C00-2123', 21): array([115,  73, 117,  75]), ('C00-2123', 24): array([35, 35,  0,  0]), ('C00-2123', 27): array([282,  48,   0,   0]), ('C00-2123', 30): array([16, 16,  0,  0]), ('C00-2123', 33): array([127,  34,   0,   0]), ('C00-2123', 36): array([113,  22,   0,   0]), ('C00-2123', 39): array([120,  29,   0,   0]), ('C00-2123', 42): array([5, 5, 0, 0]), ('C00-2123', 45): array([110,  26,   0,   0]), ('C00-2123', 48): array([47, 23,  0,  0]), ('C00-2123', 51): array([61, 19, 62, 20]), ('C90-2039', 1): array([48, 28,  0,  0]), ('C90-2039', 4): array([102,  67, 103,  68]), ('C90-2039', 7): array([49,  9,  0,  0]), ('C90-2039', 10): array([62, 22,  0,  0]), ('C90-2039', 13): array([49, 29,  0,  0]), ('C90-2039', 16): array([21, 21,  0,  0]), ('C90-2039', 19): array([16, 16,  0,  0]), ('C90-2039', 22): array([230,   9,   0,   0]), ('C90-2039', 25): array([56, 11,  0,  0]), ('C90-2039', 28): array([69, 24,  0,  0]), ('C90-2039', 31): array([37, 37,  0,  0]), ('C90-2039', 34): array([116, 116, 117, 117]), ('C90-2039', 37): array([136, 136,   0,   0]), ('D10-1083', 1): array([263,  44, 264,  45]), ('D10-1083', 4): array([10, 10,  0,  0]), ('D10-1083', 7): array([16, 16, 17, 17]), ('D10-1083', 10): array([32, 32,  0,  0]), ('D10-1083', 13): array([90, 57, 91, 58]), ('D10-1083', 16): array([129,  35, 130,  36]), ('D10-1083', 19): array([74, 33,  0,  0]), ('D10-1083', 22): array([213,  29,   0,   0]), ('D10-1083', 25): array([13, 13,  0,  0]), ('D10-1083', 28): array([27, 27,  0,  0]), ('D10-1083', 31): array([14, 14,  0,  0]), ('D10-1083', 34): array([40, 20, 41, 21]), ('D10-1083', 37): array([152,  33, 153,  34]), ('D10-1083', 40): array([55, 27,  0,  0]), ('D10-1083', 43): array([102,  55,   0,   0]), ('D10-1083', 46): array([7, 7, 8, 8]), ('I05-5011', 1): array([18, 18, 19, 19]), ('I05-5011', 4): array([12, 12,  0,  0]), ('I05-5011', 7): array([70, 25,  0,  0]), ('I05-5011', 10): array([27,  1,  0,  0]), ('I05-5011', 13): array([53, 24,  0,  0]), ('I05-5011', 16): array([429, 302,   0,   0]), ('I05-5011', 19): array([65, 24,  0,  0]), ('I05-5011', 22): array([49, 13,  0,  0]), ('I05-5011', 25): array([57, 21,  0,  0]), ('I05-5011', 28): array([67, 31,  0,  0]), ('I05-5011', 31): array([11, 11,  0,  0]), ('I05-5011', 34): array([218,  82,   0,   0]), ('I05-5011', 37): array([9, 9, 0, 0]), ('I05-5011', 40): array([20, 20,  0,  0]), ('I05-5011', 43): array([134,  15,   0,   0]), ('I05-5011', 46): array([14, 14, 15, 15]), ('I05-5011', 49): array([41, 11,  0,  0]), ('N04-1038', 1): array([7, 7, 0, 0]), ('N04-1038', 4): array([35, 11, 36, 12]), ('N04-1038', 7): array([109,   5,   0,   0]), ('N04-1038', 10): array([207,  10, 208,  11]), ('N04-1038', 13): array([202,  71,   0,   0]), ('N04-1038', 16): array([124,   9,   0,   0]), ('N04-1038', 19): array([177,  62, 178,  63]), ('N04-1038', 22): array([46,  9, 47, 10]), ('N04-1038', 25): array([11, 11,  0,  0]), ('N04-1038', 28): array([35,  5, 38,  8]), ('N04-1038', 31): array([17, 17,  0,  0]), ('N04-1038', 34): array([223, 160,   0,   0]), ('N04-1038', 37): array([72, 42,  0,  0]), ('N04-1038', 40): array([129,  39,   0,   0]), ('N04-1038', 43): array([63, 13,  0,  0]), ('N04-1038', 46): array([46, 28, 47, 29]), ('N04-1038', 49): array([33, 13,  0,  0]), ('N04-1038', 52): array([181,  10,   0,   0]), ('N04-1038', 55): array([10, 10,  0,  0]), ('N04-1038', 58): array([22, 22, 23, 23]), ('N04-1038', 61): array([36,  3, 37,  4]), ('N04-1038', 64): array([50, 17, 51, 18]), ('P05-1004', 1): array([74, 44,  0,  0]), ('P05-1004', 4): array([229,  72,   0,   0]), ('P05-1004', 7): array([446,  30,   0,   0]), ('P05-1004', 10): array([26, 26,  0,  0]), ('P05-1004', 13): array([189,  11,   0,   0]), ('P05-1004', 16): array([83,  3,  0,  0]), ('P05-1004', 19): array([15, 15,  0,  0]), ('P05-1004', 22): array([16, 16,  0,  0]), ('P05-1004', 25): array([16, 16,  0,  0]), ('P05-1004', 28): array([6, 6, 0, 0]), ('P05-1004', 31): array([50,  2,  0,  0]), ('P05-1004', 34): array([234,  15,   0,   0]), ('P05-1004', 37): array([94, 12,  0,  0]), ('P98-1081', 1): array([130, 116,   0,   0]), ('P98-1081', 4): array([134, 120, 135, 121]), ('P98-1081', 7): array([105,  48, 106,  49]), ('P98-1081', 10): array([117,  73, 118,  74]), ('P98-1081', 13): array([35, 13,  0,  0]), ('P98-1081', 16): array([42, 17, 43, 18]), ('P98-1081', 19): array([10, 10, 11, 11]), ('P98-1081', 22): array([81, 19, 82, 20]), ('P98-1081', 25): array([90, 28,  0,  0]), ('P98-1081', 28): array([111,   3,   0,   0]), ('P98-1081', 31): array([26, 23,  0,  0]), ('P98-1081', 34): array([18, 15, 19, 16]), ('P98-1081', 37): array([32, 29,  0,  0]), ('P98-1081', 40): array([40, 40, 41, 41]), ('P98-1081', 43): array([183,  33, 184,  34]), ('P98-1081', 46): array([570,   7, 571,   8]), ('P98-1081', 49): array([102,  52, 103,  53]), ('P98-1081', 52): array([487,  81, 488,  82]), ('P98-1081', 55): array([176,  26,   0,   0]), ('P98-1081', 58): array([309, 159,   0,   0]), ('P98-1081', 61): array([398,  83,   0,   0]), ('W08-2222', 1): array([69, 11,  0,  0]), ('W08-2222', 4): array([43, 26,  0,  0]), ('W08-2222', 7): array([12, 12,  0,  0]), ('W08-2222', 10): array([42,  5,  0,  0]), ('W08-2222', 13): array([56, 25,  0,  0]), ('W08-2222', 16): array([124,  40,   0,   0]), ('W08-2222', 19): array([7, 7, 0, 0]), ('W08-2222', 22): array([35,  7,  0,  0]), ('C02-1025', 1): array([115,  27,   0,   0]), ('C02-1025', 4): array([65, 25,  0,  0]), ('C02-1025', 7): array([88,  6,  0,  0]), ('C02-1025', 10): array([33, 33,  0,  0]), ('C02-1025', 13): array([51,  4,  0,  0]), ('C02-1025', 16): array([161,  86,   0,   0]), ('C02-1025', 19): array([52, 29,  0,  0]), ('C02-1025', 22): array([174,   9,   0,   0]), ('C02-1025', 25): array([27,  4,  0,  0]), ('C02-1025', 28): array([12, 12,  0,  0]), ('C02-1025', 31): array([32,  8,  0,  0]), ('C02-1025', 34): array([44, 20,  0,  0]), ('C02-1025', 37): array([62, 38,  0,  0]), ('C02-1025', 40): array([46, 22,  0,  0]), ('C02-1025', 43): array([44,  7,  0,  0]), ('C02-1025', 46): array([8, 8, 0, 0]), ('C02-1025', 49): array([147,  30,   0,   0]), ('C02-1025', 52): array([11, 11,  0,  0]), ('C94-2154', 1): array([113, 113,   0,   0]), ('C94-2154', 5): array([119,  17, 120,  18]), ('C94-2154', 8): array([7, 7, 0, 0]), ('C94-2154', 11): array([61, 61, 62, 62]), ('E03-1020', 1): array([12, 12,  0,  0]), ('E03-1020', 4): array([18,  2, 19,  3]), ('E03-1020', 7): array([199,   9, 200,  10]), ('E03-1020', 10): array([35, 16, 36, 17]), ('E03-1020', 13): array([21, 21,  0,  0]), ('E03-1020', 16): array([27,  5,  0,  0]), ('E03-1020', 19): array([157,   1,   0,   0]), ('E03-1020', 22): array([160,   4,   0,   0]), ('E03-1020', 25): array([165,   9,   0,   0]), ('E03-1020', 28): array([73, 32,  0,  0]), ('E03-1020', 31): array([10, 10,  0,  0]), ('E03-1020', 34): array([80, 10,  0,  0]), ('E03-1020', 37): array([176,  50, 178,  52]), ('J00-3003', 1): array([13,  2, 14,  3]), ('J00-3003', 4): array([12, 12,  0,  0]), ('J00-3003', 7): array([18, 18,  0,  0]), ('J00-3003', 10): array([185,   6,   0,   0]), ('J00-3003', 13): array([1, 1, 2, 2]), ('J00-3003', 16): array([18, 18,  0,  0]), ('J00-3003', 19): array([17,  1, 18,  2]), ('J00-3003', 22): array([95, 14, 96, 15]), ('J00-3003', 25): array([64, 18, 65, 19]), ('J00-3003', 28): array([6, 6, 0, 0]), ('N06-2049', 1): array([273, 109, 274, 110]), ('N06-2049', 4): array([19, 19,  0,  0]), ('N06-2049', 7): array([20, 20,  0,  0]), ('N06-2049', 10): array([36, 22,  0,  0]), ('N06-2049', 13): array([13, 13,  0,  0]), ('N06-2049', 16): array([51,  7,  0,  0]), ('N06-2049', 19): array([277, 168,   0,   0]), ('N06-2049', 22): array([134,  17,   0,   0]), ('N06-2049', 25): array([93,  9, 94, 10]), ('N06-2049', 28): array([172,  72, 173,  73]), ('N06-2049', 31): array([188,  41, 189,  42]), ('N06-2049', 34): array([14,  2,  0,  0]), ('N06-2049', 40): array([55, 21, 56, 22]), ('N06-2049', 43): array([160,  19,   0,   0]), ('N06-2049', 46): array([11, 11,  0,  0]), ('N06-2049', 49): array([11, 11,  0,  0]), ('N06-2049', 52): array([25, 11,  0,  0]), ('N06-2049', 55): array([30, 16,  0,  0]), ('N06-2049', 58): array([21, 14, 22, 15]), ('P05-1053', 1): array([15, 15,  0,  0]), ('P05-1053', 7): array([42,  5,  0,  0]), ('P05-1053', 10): array([45,  8,  0,  0]), ('P05-1053', 13): array([14, 14,  0,  0]), ('P05-1053', 16): array([14, 14,  0,  0]), ('P05-1053', 22): array([166,  29, 167,  30]), ('P05-1053', 25): array([176,  39, 177,  40]), ('P05-1053', 28): array([53,  5,  0,  0]), ('P05-1053', 31): array([81, 19,  0,  0]), ('P05-1053', 34): array([100,  36,   0,   0]), ('P05-1053', 37): array([59, 29,  0,  0]), ('P05-1053', 40): array([54, 34, 55, 35]), ('P05-1053', 43): array([109,  34,   0,   0]), ('P05-1053', 46): array([224,   3, 225,   4]), ('P05-1053', 49): array([10, 10,  0,  0]), ('P05-1053', 52): array([9, 9, 0, 0]), ('P05-1053', 55): array([31,  7, 32,  8]), ('P05-1053', 58): array([35, 11, 36, 12]), ('P05-1053', 61): array([38, 14, 39, 15]), ('P05-1053', 64): array([135,   6, 136,   7]), ('P05-1053', 67): array([137,   8,   0,   0]), ('P05-1053', 70): array([12, 12,  0,  0]), ('P05-1053', 73): array([33,  1, 34,  2]), ('P05-1053', 76): array([122,  69,   0,   0]), ('P05-1053', 79): array([130,  77,   0,   0]), ('P05-1053', 82): array([7, 7, 0, 0]), ('P05-1053', 85): array([107,  11, 108,  12]), ('P05-1053', 88): array([112,  16, 113,  17]), ('P05-1053', 91): array([13, 13,  0,  0]), ('P05-1053', 94): array([18, 18,  0,  0]), ('P05-1053', 97): array([45, 18,  0,  0]), ('P05-1053', 100): array([128,  60,   0,   0]), ('P05-1053', 103): array([206,  17, 207,  18]), ('P05-1053', 106): array([39,  5,  0,  0]), ('P05-1053', 109): array([29,  9, 30, 10]), ('P05-1053', 112): array([39, 39,  0,  0]), ('P05-1053', 115): array([50, 10, 51, 11]), ('P05-1053', 118): array([23,  1, 24,  2]), ('P05-1053', 121): array([7, 7, 0, 0]), ('P05-1053', 124): array([10, 10,  0,  0]), ('P05-1053', 127): array([80, 44, 81, 45]), ('P05-1053', 130): array([193, 157, 194, 158]), ('P05-1053', 133): array([11, 11,  0,  0]), ('P05-1053', 136): array([58, 19,  0,  0]), ('P05-1053', 139): array([34,  7,  0,  0]), ('P05-1053', 142): array([47,  4,  0,  0]), ('P05-1053', 145): array([167,  10,   0,   0]), ('P05-1053', 148): array([9, 9, 0, 0]), ('P05-1053', 151): array([17,  2,  0,  0]), ('P05-1053', 154): array([174, 110,   0,   0]), ('P05-1053', 157): array([39, 39,  0,  0]), ('P05-1053', 160): array([136,  51,   0,   0]), ('P05-1053', 163): array([203,  11,   0,   0]), ('P05-1053', 166): array([10, 10,  0,  0]), ('P98-2143', 1): array([28,  4, 29,  5]), ('P98-2143', 4): array([13,  3,  0,  0]), ('P98-2143', 7): array([65,  3,  0,  0]), ('P98-2143', 10): array([71,  9,  0,  0]), ('P98-2143', 13): array([79, 17,  0,  0]), ('P98-2143', 16): array([60, 22, 61, 23]), ('P98-2143', 19): array([19, 19,  0,  0]), ('P98-2143', 22): array([101,  44,   0,   0]), ('P98-2143', 25): array([52,  6,  0,  0]), ('P98-2143', 28): array([91,  8,  0,  0]), ('P98-2143', 31): array([59, 21,  0,  0]), ('P98-2143', 34): array([61, 23, 62, 24]), ('P98-2143', 37): array([63, 25,  0,  0]), ('P98-2143', 40): array([15, 15,  0,  0]), ('P98-2143', 43): array([22, 22, 23, 23]), ('P98-2143', 46): array([41,  2,  0,  0]), ('P98-2143', 49): array([5, 5, 0, 0]), ('P98-2143', 52): array([318,  14,   0,   0]), ('P98-2143', 55): array([45, 33,  0,  0]), ('P98-2143', 58): array([6, 6, 0, 0]), ('P98-2143', 61): array([7, 7, 8, 8]), ('P98-2143', 64): array([ 9,  9, 10, 10]), ('P98-2143', 67): array([63, 63,  0,  0]), ('P98-2143', 70): array([45, 32,  0,  0]), ('P98-2143', 73): array([27, 27,  0,  0]), ('P98-2143', 76): array([178,  18,   0,   0]), ('P98-2143', 79): array([209,  28,   0,   0]), ('P98-2143', 82): array([65, 31,  0,  0]), ('P98-2143', 85): array([82, 48,  0,  0]), ('P98-2143', 88): array([8, 8, 0, 0]), ('P98-2143', 91): array([13, 13,  0,  0]), ('P98-2143', 94): array([191,   5,   0,   0]), ('P98-2143', 97): array([8, 8, 0, 0]), ('P98-2143', 100): array([17, 17,  0,  0]), ('P98-2143', 103): array([97, 23,  0,  0]), ('P98-2143', 106): array([12, 12,  0,  0]), ('P98-2143', 109): array([5, 5, 0, 0]), ('P98-2143', 112): array([15,  2,  0,  0]), ('P98-2143', 115): array([4, 4, 5, 5]), ('P98-2143', 118): array([82,  6, 83,  7]), ('P98-2143', 121): array([222, 165,   0,   0]), ('P98-2143', 124): array([102,   7,   0,   0]), ('P98-2143', 127): array([11, 11,  0,  0]), ('P98-2143', 130): array([48, 32,  0,  0]), ('P98-2143', 133): array([6, 6, 0, 0]), ('P98-2143', 136): array([59, 22,  0,  0]), ('P98-2143', 139): array([60, 23,  0,  0]), ('P98-2143', 142): array([128,   4, 129,   5]), ('P98-2143', 145): array([131,   7,   0,   0]), ('W09-0621', 1): array([83, 12,  0,  0]), ('W09-0621', 2): array([102,   5,   0,   0]), ('W09-0621', 3): array([43, 15,  0,  0]), ('W09-0621', 4): array([253,   9,   0,   0]), ('W09-0621', 5): array([288,  44,   0,   0]), ('W09-0621', 6): array([289,  45,   0,   0]), ('W09-0621', 7): array([349, 105,   0,   0]), ('W09-0621', 8): array([362, 118,   0,   0]), ('W09-0621', 9): array([368, 124,   0,   0]), ('W09-0621', 10): array([33, 10,  0,  0]), ('W09-0621', 11): array([34,  7,  0,  0]), ('W09-0621', 12): array([73, 16,  0,  0]), ('C04-1089', 1): array([78, 38, 79, 39]), ('C04-1089', 4): array([15,  1,  0,  0]), ('C04-1089', 7): array([53,  8, 54,  9]), ('C04-1089', 10): array([38,  7,  0,  0]), ('C04-1089', 13): array([68, 37,  0,  0]), ('C04-1089', 16): array([15, 15,  0,  0]), ('C04-1089', 19): array([50, 10,  0,  0]), ('C04-1089', 22): array([147,   1,   0,   0]), ('C04-1089', 25): array([208,  17,   0,   0]), ('C04-1089', 28): array([163,   3, 164,   4]), ('C04-1089', 31): array([7, 7, 0, 0]), ('C04-1089', 34): array([8, 8, 0, 0]), ('C04-1089', 37): array([38, 16,  0,  0]), ('C04-1089', 40): array([23,  1,  0,  0]), ('C04-1089', 43): array([66, 18,  0,  0]), ('C98-1097', 1): array([15, 15,  0,  0]), ('C98-1097', 2): array([136,  91,   0,   0]), ('C98-1097', 3): array([31,  1, 32,  2]), ('C98-1097', 4): array([17, 17,  0,  0]), ('C98-1097', 5): array([30,  9, 31, 10]), ('C98-1097', 6): array([89,  5, 90,  6]), ('C98-1097', 7): array([97, 13, 98, 14]), ('C98-1097', 8): array([203,  21, 204,  22]), ('C98-1097', 9): array([216,   8,   0,   0]), ('C98-1097', 10): array([27,  4,  0,  0]), ('C98-1097', 11): array([10, 10,  0,  0]), ('C98-1097', 12): array([109,  27, 110,  28]), ('E09-2008', 1): array([161,   4,   0,   0]), ('E09-2008', 4): array([73, 11,  0,  0]), ('E09-2008', 7): array([43, 15,  0,  0]), ('E09-2008', 10): array([124,  44,   0,   0]), ('E09-2008', 13): array([49, 22,  0,  0]), ('E09-2008', 16): array([22, 22,  0,  0]), ('E09-2008', 19): array([80, 43,  0,  0]), ('E09-2008', 22): array([77, 46,  0,  0]), ('J96-3004', 1): array([141,  10, 142,  11]), ('J96-3004', 4): array([80, 25,  0,  0]), ('J96-3004', 7): array([58, 39,  0,  0]), ('J96-3004', 10): array([124, 105, 127, 108]), ('J96-3004', 13): array([20, 20, 21, 21]), ('J96-3004', 16): array([107,  48,   0,   0]), ('J96-3004', 19): array([107,   4,   0,   0]), ('J96-3004', 22): array([6, 6, 7, 7]), ('J96-3004', 25): array([42, 42,  0,  0]), ('J96-3004', 28): array([96, 41, 97, 42]), ('J96-3004', 31): array([53, 53,  0,  0]), ('J96-3004', 34): array([113,   9,   0,   0]), ('J96-3004', 37): array([211,  32,   0,   0]), ('J96-3004', 40): array([321,   7,   0,   0]), ('J96-3004', 43): array([88, 24, 89, 25]), ('J96-3004', 46): array([125,  61, 126,  62]), ('J96-3004', 49): array([131,  67, 132,  68]), ('J96-3004', 52): array([489, 153, 490, 154]), ('J96-3004', 55): array([123,  14,   0,   0]), ('J96-3004', 61): array([9, 9, 0, 0]), ('J96-3004', 64): array([612,  54, 613,  55]), ('J96-3004', 67): array([621,  63, 622,  64]), ('J96-3004', 70): array([6, 6, 0, 0]), ('J96-3004', 73): array([41, 18, 42, 19]), ('J96-3004', 76): array([122,  27,   0,   0]), ('J96-3004', 79): array([154,  59, 155,  60]), ('J96-3004', 82): array([42, 10, 43, 11]), ('J96-3004', 85): array([7, 7, 0, 0]), ('J96-3004', 88): array([113,  21,   0,   0]), ('J96-3004', 91): array([70, 42,  0,  0]), ('J96-3004', 94): array([105,  53,   0,   0]), ('J96-3004', 97): array([91, 31,  0,  0]), ('J96-3004', 100): array([12, 12,  0,  0]), ('J96-3004', 103): array([144,  11, 145,  12]), ('J96-3004', 106): array([149,   2, 150,   3]), ('J96-3004', 109): array([5, 5, 6, 6]), ('J96-3004', 112): array([8, 8, 0, 0]), ('J96-3004', 115): array([10, 10,  0,  0]), ('J96-3004', 118): array([178, 108,   0,   0]), ('J96-3004', 121): array([29, 29,  0,  0]), ('J96-3004', 124): array([10, 10,  0,  0]), ('J96-3004', 127): array([40, 11, 41, 12]), ('J96-3004', 130): array([13, 13,  0,  0]), ('J96-3004', 133): array([5, 5, 0, 0]), ('J96-3004', 136): array([17, 17,  0,  0]), ('J96-3004', 139): array([180,   4,   0,   0]), ('J96-3004', 142): array([186,  10, 187,  11]), ('J96-3004', 145): array([3, 3, 0, 0]), ('J96-3004', 148): array([83,  9,  0,  0]), ('J96-3004', 151): array([118,  13,   0,   0]), ('J96-3004', 154): array([16,  2,  0,  0]), ('J96-3004', 157): array([16, 16,  0,  0]), ('J96-3004', 160): array([174,   7,   0,   0]), ('J96-3004', 163): array([41,  5,  0,  0]), ('J96-3004', 166): array([204,   9,   0,   0]), ('J96-3004', 169): array([11, 11, 12, 12]), ('J96-3004', 172): array([155,  30, 156,  31]), ('J96-3004', 175): array([26, 26,  0,  0]), ('J96-3004', 178): array([69,  5,  0,  0]), ('J96-3004', 181): array([73,  9,  0,  0]), ('J96-3004', 184): array([86, 22,  0,  0]), ('J96-3004', 187): array([121,  15,   0,   0]), ('J96-3004', 190): array([10, 10, 11, 11]), ('N09-1001', 1): array([146,  86, 147,  87]), ('N09-1001', 2): array([22, 22,  0,  0]), ('N09-1001', 3): array([22, 22,  0,  0]), ('N09-1001', 4): array([35,  5,  0,  0]), ('N09-1001', 5): array([121,  25,   0,   0]), ('N09-1001', 6): array([147,  36,   0,   0]), ('N09-1001', 7): array([146,  96,   0,   0]), ('N09-1001', 8): array([10, 10,  0,  0]), ('N09-1001', 9): array([180,   2,   0,   0]), ('N09-1001', 10): array([70, 30,  0,  0]), ('N09-1001', 11): array([33,  4,  0,  0]), ('N09-1001', 12): array([70, 16,  0,  0]), ('N09-1001', 13): array([231,   1,   0,   0]), ('P06-2124', 1): array([48,  1, 49,  2]), ('P06-2124', 4): array([52,  5, 53,  6]), ('P06-2124', 7): array([28, 28, 29, 29]), ('P06-2124', 10): array([84, 22,  0,  0]), ('P06-2124', 13): array([86, 24,  0,  0]), ('P06-2124', 16): array([11, 11,  0,  0]), ('P06-2124', 19): array([17, 17,  0,  0]), ('P06-2124', 22): array([149,  13,   0,   0]), ('P06-2124', 25): array([160,  24,   0,   0]), ('P06-2124', 28): array([8, 8, 0, 0]), ('P06-2124', 31): array([40,  1,  0,  0]), ('P06-2124', 34): array([30, 30,  0,  0]), ('P06-2124', 37): array([27,  4,  0,  0]), ('P06-2124', 40): array([13, 13, 14, 14]), ('P06-2124', 43): array([62, 39,  0,  0]), ('W03-0410', 1): array([9, 9, 0, 0]), ('W03-0410', 4): array([25, 25,  0,  0]), ('W03-0410', 7): array([70, 43,  0,  0]), ('W03-0410', 10): array([17, 17,  0,  0]), ('W03-0410', 13): array([38,  5,  0,  0]), ('W03-0410', 16): array([40,  7,  0,  0]), ('W03-0410', 19): array([54,  2,  0,  0]), ('W03-0410', 22): array([82, 30, 83, 31]), ('W03-0410', 25): array([167,  16,   0,   0]), ('W03-0410', 28): array([168,  17,   0,   0]), ('W03-0410', 31): array([169,  18,   0,   0]), ('W03-0410', 34): array([397, 325,   0,   0]), ('W03-0410', 37): array([586,  35, 587,  36]), ('W03-0410', 40): array([124,  67,   0,   0]), ('W03-0410', 43): array([143,  86,   0,   0]), ('W03-0410', 46): array([11, 11,  0,  0]), ('W03-0410', 49): array([39, 16,  0,  0]), ('W03-0410', 52): array([7, 7, 8, 8]), ('W03-0410', 55): array([13, 13,  0,  0]), ('W03-0410', 58): array([118,  21, 119,  22]), ('W03-0410', 61): array([80,  6,  0,  0]), ('W11-0815', 1): array([7, 7, 0, 0]), ('W11-0815', 2): array([625, 267,   0,   0]), ('W11-0815', 3): array([15, 15,  0,  0]), ('W11-0815', 4): array([16, 16,  0,  0]), ('W11-0815', 5): array([5, 5, 0, 0]), ('W11-0815', 6): array([31, 31,  0,  0]), ('W11-0815', 7): array([15, 15,  0,  0]), ('W11-0815', 8): array([16, 16,  0,  0]), ('W11-0815', 9): array([4, 4, 0, 0]), ('W11-0815', 10): array([7, 7, 0, 0]), ('C08-1098', 1): array([122,   9,   0,   0]), ('C08-1098', 4): array([208,  99,   0,   0]), ('C08-1098', 7): array([228, 119,   0,   0]), ('C08-1098', 10): array([212,  20,   0,   0]), ('C08-1098', 13): array([176, 139,   0,   0]), ('C08-1098', 16): array([121,   7,   0,   0]), ('C08-1098', 19): array([159,  71,   0,   0]), ('C08-1098', 22): array([42, 31,  0,  0]), ('C08-1098', 25): array([74, 63,  0,  0]), ('C08-1098', 28): array([81,  6,  0,  0]), ('C08-1098', 31): array([32, 18,  0,  0]), ('C08-1098', 34): array([41,  4,  0,  0]), ('C08-1098', 37): array([49,  4,  0,  0]), ('C08-1098', 40): array([101,  92, 102,  93]), ('C08-1098', 43): array([26, 15,  0,  0]), ('C08-1098', 46): array([89,  8,  0,  0]), ('C08-1098', 49): array([127, 113,   0,   0]), ('C08-1098', 52): array([78, 16,  0,  0]), ('C08-1098', 55): array([35,  6,  0,  0]), ('C08-1098', 58): array([37,  4,  0,  0]), ('C08-1098', 61): array([85,  8,  0,  0]), ('C08-1098', 64): array([84, 73,  0,  0]), ('C08-1098', 67): array([134,  14,   0,   0]), ('C08-1098', 70): array([35,  6,  0,  0]), ('C08-1098', 73): array([58, 11,  0,  0]), ('C08-1098', 76): array([83,  2,  0,  0]), ('C08-1098', 79): array([87, 65,  0,  0]), ('C08-1098', 82): array([111,  54,   0,   0]), ('D09-1023', 1): array([78, 14,  0,  0]), ('D09-1023', 2): array([1, 1, 0, 0]), ('D09-1023', 3): array([16, 16,  0,  0]), ('D09-1023', 4): array([23,  1,  0,  0]), ('D09-1023', 5): array([41, 17,  0,  0]), ('D09-1023', 6): array([146,   6,   0,   0]), ('D09-1023', 7): array([79, 18,  0,  0]), ('D09-1023', 8): array([139,  78,   0,   0]), ('D09-1023', 9): array([205,  24,   0,   0]), ('D09-1023', 10): array([17, 17,  0,  0]), ('D09-1023', 11): array([37,  3,  0,  0]), ('D09-1023', 12): array([6, 6, 0, 0]), ('H05-1115', 1): array([60,  6,  0,  0]), ('H05-1115', 4): array([85, 31,  0,  0]), ('H05-1115', 7): array([43, 14,  0,  0]), ('H05-1115', 10): array([41,  7,  0,  0]), ('H05-1115', 13): array([83, 83,  0,  0]), ('H05-1115', 16): array([113,  31,   0,   0]), ('H05-1115', 19): array([24,  3, 25,  4]), ('H05-1115', 22): array([94, 73,  0,  0]), ('H05-1115', 25): array([11, 11,  0,  0]), ('H05-1115', 28): array([22,  5,  0,  0]), ('H05-1115', 31): array([104,  36, 108,  40]), ('J98-2005', 1): array([72, 37,  0,  0]), ('J98-2005', 4): array([56, 19,  0,  0]), ('J98-2005', 7): array([79, 32,  0,  0]), ('J98-2005', 10): array([23, 23,  0,  0]), ('N09-1025', 1): array([22, 22,  0,  0]), ('N09-1025', 2): array([6, 6, 0, 0]), ('N09-1025', 3): array([9, 9, 0, 0]), ('N09-1025', 4): array([25, 25,  0,  0]), ('N09-1025', 5): array([159,  11,   0,   0]), ('N09-1025', 6): array([239,  91,   0,   0]), ('N09-1025', 7): array([13, 13,  0,  0]), ('N09-1025', 8): array([77,  2, 78,  3]), ('N09-1025', 9): array([207,  55,   0,   0]), ('N09-1025', 10): array([210,  58,   0,   0]), ('N09-1025', 11): array([227,  16,   0,   0]), ('N09-1025', 12): array([102,   7,   0,   0]), ('N09-1025', 13): array([147,  64,   0,   0]), ('N09-1025', 14): array([36, 16,  0,  0]), ('N09-1025', 15): array([102,  56,   0,   0]), ('N09-1025', 16): array([115,   9,   0,   0]), ('N09-1025', 17): array([120,  14,   0,   0]), ('N09-1025', 18): array([51, 23,  0,  0]), ('N09-1025', 19): array([27, 27,  0,  0]), ('N09-1025', 20): array([15, 15,  0,  0]), ('N09-1025', 21): array([16, 16,  0,  0]), ('N09-1025', 22): array([218, 109,   0,   0]), ('N09-1025', 23): array([245, 136,   0,   0]), ('N09-1025', 24): array([252,   6,   0,   0]), ('N09-1025', 25): array([73,  6,  0,  0]), ('N09-1025', 26): array([79, 79,  0,  0]), ('N09-1025', 27): array([86, 86,  0,  0]), ('N09-1025', 28): array([48, 25,  0,  0]), ('N09-1025', 29): array([209,  25,   0,   0]), ('N09-1025', 30): array([53, 30,  0,  0]), ('N09-1025', 31): array([92, 25,  0,  0]), ('N09-1025', 32): array([97, 30,  0,  0]), ('P07-1040', 1): array([16, 16, 17, 17]), ('P07-1040', 2): array([32,  7,  0,  0]), ('P07-1040', 3): array([88, 35, 89, 36]), ('P07-1040', 4): array([93, 40, 94, 41]), ('P07-1040', 5): array([7, 7, 0, 0]), ('P07-1040', 6): array([133,   3, 134,   4]), ('P07-1040', 7): array([147,  17, 148,  18]), ('P07-1040', 8): array([24, 24,  0,  0]), ('P07-1040', 9): array([2, 2, 3, 3]), ('P07-1040', 10): array([23, 23, 24, 24]), ('P07-1040', 11): array([28, 28, 29, 29]), ('P07-1040', 12): array([69,  1, 70,  2]), ('P07-1040', 13): array([78, 10, 79, 11]), ('P07-1040', 14): array([46, 16, 47, 17]), ('P07-1040', 15): array([127,  18,   0,   0]), ('P07-1040', 16): array([24, 24,  0,  0]), ('P07-1040', 17): array([19, 19, 20, 20]), ('P07-1040', 18): array([12, 12, 13, 13]), ('P07-1040', 19): array([41, 13,  0,  0]), ('P07-1040', 20): array([13, 13,  0,  0]), ('P07-1040', 21): array([116,   8,   0,   0]), ('P07-1040', 22): array([7, 7, 0, 0]), ('P07-1040', 23): array([17, 17,  0,  0]), ('P07-1040', 24): array([39, 17,  0,  0]), ('P07-1040', 25): array([87, 21,  0,  0]), ('P07-1040', 26): array([35,  9, 36, 10]), ('W04-0213', 1): array([13, 13,  0,  0]), ('W04-0213', 4): array([32, 14,  0,  0]), ('W04-0213', 7): array([47,  2,  0,  0]), ('W04-0213', 10): array([4, 4, 0, 0]), ('W04-0213', 13): array([22,  2,  0,  0]), ('W04-0213', 16): array([3, 3, 4, 4]), ('W04-0213', 19): array([15,  1,  0,  0]), ('W04-0213', 22): array([72,  9,  0,  0]), ('W04-0213', 25): array([170,   1,   0,   0]), ('W04-0213', 28): array([29, 10, 30, 11]), ('W04-0213', 31): array([40,  1,  0,  0]), ('W04-0213', 34): array([42, 11, 43, 12]), ('W04-0213', 37): array([98, 35,  0,  0]), ('W04-0213', 40): array([161,  39,   0,   0]), ('W04-0213', 43): array([31,  5,  0,  0]), ('W95-0104', 1): array([23, 23,  0,  0]), ('W95-0104', 4): array([29,  7,  0,  0]), ('W95-0104', 7): array([125,   1,   0,   0]), ('W95-0104', 10): array([46, 34,  0,  0]), ('W95-0104', 13): array([71, 36,  0,  0]), ('W95-0104', 16): array([40, 11,  0,  0]), ('W95-0104', 19): array([164,  24,   0,   0]), ('W95-0104', 22): array([33, 33, 34, 34]), ('W95-0104', 25): array([20,  3, 21,  4]), ('W95-0104', 28): array([108,  80,   0,   0]), ('W95-0104', 31): array([23, 23, 24, 24]), ('W95-0104', 34): array([56, 23,  0,  0]), ('W95-0104', 37): array([89, 56,  0,  0]), ('W95-0104', 40): array([98,  8, 99,  9]), ('W95-0104', 43): array([102,  12,   0,   0]), ('W95-0104', 46): array([103,  13,   0,   0]), ('W95-0104', 49): array([114,  24,   0,   0]), ('W95-0104', 52): array([116,  26,   0,   0]), ('W95-0104', 55): array([11, 11,  0,  0]), ('W95-0104', 58): array([19,  4, 20,  5]), ('W95-0104', 61): array([25, 25,  0,  0]), ('W95-0104', 64): array([38, 38,  0,  0]), ('W95-0104', 67): array([78,  3,  0,  0]), ('W95-0104', 70): array([79,  4, 80,  5]), ('W95-0104', 73): array([95, 13,  0,  0]), ('W95-0104', 76): array([143,  61,   0,   0]), ('W95-0104', 79): array([23,  6,  0,  0]), ('W95-0104', 82): array([113,  18,   0,   0]), ('W95-0104', 85): array([10, 10,  0,  0]), ('W95-0104', 88): array([9, 9, 0, 0]), ('W95-0104', 91): array([180,  30,   0,   0]), ('W95-0104', 94): array([13, 13,  0,  0]), ('W95-0104', 97): array([84, 60,  0,  0]), ('W95-0104', 100): array([146,   9,   0,   0]), ('W95-0104', 103): array([33,  7,  0,  0]), ('W95-0104', 106): array([28, 28,  0,  0]), ('W95-0104', 109): array([4, 4, 0, 0]), ('C10-1045', 1): array([39, 13,  0,  0]), ('C10-1045', 4): array([72, 72,  0,  0]), ('C10-1045', 7): array([148,  74, 149,  75]), ('C10-1045', 10): array([479,  16, 480,  17]), ('C10-1045', 13): array([510,  47, 511,  48]), ('C10-1045', 16): array([670,  13,   0,   0]), ('C10-1045', 19): array([672,  15,   0,   0]), ('C10-1045', 22): array([195,  12,   0,   0]), ('C10-1045', 25): array([196,  13,   0,   0]), ('C10-1045', 28): array([665, 131,   0,   0]), ('C10-1045', 31): array([190,  29,   0,   0]), ('C10-1045', 34): array([193,  32,   0,   0]), ('C10-1045', 37): array([316,  36,   0,   0]), ('C10-1045', 40): array([397,  13,   0,   0]), ('C10-1045', 43): array([406,  22,   0,   0]), ('C10-1045', 46): array([109,  36,   0,   0]), ('C10-1045', 49): array([34, 12,  0,  0]), ('C10-1045', 52): array([7, 7, 0, 0]), ('C10-1045', 55): array([73,  1,  0,  0]), ('C10-1045', 58): array([109,  37,   0,   0]), ('C10-1045', 61): array([18, 18,  0,  0]), ('C10-1045', 64): array([196,  23,   0,   0]), ('C10-1045', 67): array([12, 12,  0,  0]), ('C10-1045', 70): array([34, 12, 35, 13]), ('C10-1045', 73): array([38, 16, 39, 17]), ('C10-1045', 76): array([115,  46,   0,   0]), ('C10-1045', 79): array([234,  86,   0,   0]), ('C10-1045', 82): array([244,  96,   0,   0]), ('C10-1045', 85): array([260,  60,   0,   0]), ('C10-1045', 88): array([262,  62,   0,   0]), ('C10-1045', 91): array([218,  18,   0,   0]), ('D10-1058', 1): array([48, 27,  0,  0]), ('D10-1058', 2): array([24, 24,  0,  0]), ('D10-1058', 3): array([33,  4,  0,  0]), ('D10-1058', 4): array([43, 14,  0,  0]), ('D10-1058', 5): array([82,  3,  0,  0]), ('D10-1058', 6): array([99,  7,  0,  0]), ('D10-1058', 7): array([45, 45,  0,  0]), ('D10-1058', 8): array([79,  2,  0,  0]), ('D10-1058', 9): array([103,  24, 104,  25]), ('D10-1058', 10): array([131,  13,   0,   0]), ('D10-1058', 11): array([232,   9,   0,   0]), ('D10-1058', 12): array([24, 24,  0,  0]), ('D10-1058', 13): array([38,  9,  0,  0]), ('D10-1058', 14): array([39, 10,  0,  0]), ('D10-1058', 15): array([86,  3,  0,  0]), ('D10-1058', 16): array([104,   7,   0,   0]), ('D10-1058', 17): array([60,  3,  0,  0]), ('D10-1058', 18): array([130,  22,   0,   0]), ('H89-2014', 1): array([178, 178,   0,   0]), ('H89-2014', 4): array([47, 47,  0,  0]), ('H89-2014', 7): array([108, 108,   0,   0]), ('H89-2014', 10): array([40, 40, 41, 41]), ('H89-2014', 13): array([21, 21,  0,  0]), ('H89-2014', 16): array([60, 60,  0,  0]), ('H89-2014', 19): array([85, 55,  0,  0]), ('H89-2014', 22): array([18, 18,  0,  0]), ('H89-2014', 25): array([9, 9, 0, 0]), ('H89-2014', 28): array([124,  19,   0,   0]), ('N01-1011', 1): array([131,  33, 132,  34]), ('N01-1011', 4): array([43, 23,  0,  0]), ('N01-1011', 7): array([136,  45,   0,   0]), ('N01-1011', 10): array([49,  9,  0,  0]), ('N01-1011', 13): array([148,  54,   0,   0]), ('N01-1011', 16): array([103,  17,   0,   0]), ('P00-1025', 1): array([104,  39,   0,   0]), ('P00-1025', 2): array([52, 52,  0,  0]), ('P00-1025', 3): array([200,  43, 201,  44]), ('P00-1025', 4): array([104,  39,   0,   0]), ('P00-1025', 5): array([31, 31,  0,  0]), ('P00-1025', 6): array([32, 32,  0,  0]), ('P00-1025', 7): array([67,  1,  0,  0]), ('P00-1025', 8): array([86, 59,  0,  0]), ('P00-1025', 9): array([17, 17,  0,  0]), ('P00-1025', 10): array([134,   1,   0,   0]), ('P00-1025', 11): array([15, 15,  0,  0]), ('P00-1025', 12): array([207,  74,   0,   0]), ('P98-1046', 1): array([36, 22, 37, 23]), ('P98-1046', 4): array([38, 24, 39, 25]), ('P98-1046', 7): array([211, 162,   0,   0]), ('P98-1046', 10): array([18,  4, 19,  5]), ('P98-1046', 13): array([46,  4,  0,  0]), ('P98-1046', 16): array([25, 12,  0,  0]), ('P98-1046', 19): array([660,   3, 661,   4]), ('P98-1046', 22): array([58, 58, 59, 59]), ('P98-1046', 25): array([68, 20,  0,  0]), ('P98-1046', 28): array([30, 11, 31, 12]), ('P98-1046', 40): array([18,  4,  0,  0]), ('P98-1046', 43): array([48,  8, 49,  9]), ('P98-1046', 46): array([8, 4, 0, 0]), ('P98-1046', 52): array([83, 19,  0,  0]), ('P98-1046', 55): array([6, 2, 0, 0]), ('P98-1046', 58): array([45, 16, 46, 17]), ('P98-1046', 61): array([203,  23,   0,   0]), ('W06-3909', 1): array([93, 16,  0,  0]), ('W06-3909', 2): array([119,  42, 120,  43]), ('W06-3909', 3): array([127,   7,   0,   0]), ('W06-3909', 4): array([17, 17,  0,  0]), ('W06-3909', 5): array([82, 26,  0,  0]), ('W06-3909', 6): array([104,  66,   0,   0]), ('W06-3909', 7): array([105,  67,   0,   0]), ('W06-3909', 8): array([596, 596,   0,   0]), ('W06-3909', 9): array([601, 601,   0,   0]), ('W06-3909', 10): array([18, 18,  0,  0]), ('W06-3909', 11): array([27,  2,  0,  0]), ('W06-3909', 12): array([49, 14,  0,  0]), ('X96-1048', 1): array([3, 3, 0, 0]), ('X96-1048', 4): array([12, 12, 20, 20]), ('X96-1048', 7): array([71, 42,  0,  0]), ('X96-1048', 10): array([29, 29, 30, 30]), ('X96-1048', 13): array([10, 10,  0,  0]), ('X96-1048', 16): array([49, 18, 50, 19]), ('X96-1048', 19): array([34,  3,  0,  0]), ('X96-1048', 22): array([141, 141, 142, 142]), ('X96-1048', 25): array([17, 17,  0,  0]), ('X96-1048', 28): array([3, 3, 0, 0])}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3ENaqxD5tpaH",
        "colab_type": "text"
      },
      "source": [
        "## Models trained using only positional features"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g65u2CRcuaAX",
        "colab_type": "code",
        "outputId": "56069fa7-e82d-4130-d85f-916cab84b57d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "X = []\n",
        "y = []\n",
        "cnt = 0\n",
        "for rfile in files:\n",
        "  try :\n",
        "    open_file = open(rfile+\"/annotation/\"+rfile+\".ann.txt\", 'r')\n",
        "  except:\n",
        "    open_file = open(rfile+\"/annotation/\"+rfile+\".annv3.txt\", 'r')\n",
        "  citances = open_file.readlines()\n",
        "  for citance in citances:\n",
        "    if len(citance) > 1:\n",
        "      data_file = citance.split(\" | \")\n",
        "      if(len(data_file) == 11):\n",
        "        R_text = (data_file[8].split(\":\")[1])\n",
        "        Facet = (data_file[9].split(\":\")[1])\n",
        "        result = R_text.find('sid') \n",
        "        temp = (R_text[result:result+10])\n",
        "        t1 = temp.split('\"')[1]\n",
        "        # print(t1)\n",
        "        result = R_text.find('ssid') \n",
        "        temp = (R_text[result:result+11])\n",
        "        t2 = 0\n",
        "        try:\n",
        "          t2 = temp.split('\"')[1]\n",
        "        except:\n",
        "          t2 = 0\n",
        "        # print(t2)\n",
        "        t = [int(t1), int(t2)]\n",
        "        # print(t)\n",
        "        X.append(t)\n",
        "        g = 0\n",
        "        if (Facet.find('Result') != -1):\n",
        "          g = 2\n",
        "        elif (Facet.find('Aim') != -1):\n",
        "          g = 3\n",
        "        elif (Facet.find('Hypothesis') != -1):\n",
        "          g = 4\n",
        "        elif (Facet.find('Implication') != -1):\n",
        "          g = 5\n",
        "        elif (Facet.find('Method') != -1):\n",
        "          g = 1\n",
        "        \n",
        "        y.append(g)\n",
        "print(y)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 5, 1, 1, 3, 5, 3, 5, 1, 1, 1, 5, 2, 1, 1, 5, 3, 1, 2, 2, 1, 1, 1, 2, 2, 1, 1, 1, 3, 1, 2, 1, 1, 3, 1, 5, 1, 1, 1, 1, 1, 1, 1, 5, 3, 5, 5, 5, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 3, 5, 1, 1, 3, 1, 1, 1, 1, 1, 1, 3, 5, 3, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 3, 2, 1, 2, 1, 1, 1, 1, 1, 2, 5, 1, 1, 2, 1, 2, 2, 1, 1, 1, 1, 1, 5, 5, 1, 1, 2, 2, 2, 1, 5, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 4, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 5, 1, 1, 3, 1, 3, 1, 1, 1, 3, 3, 1, 2, 5, 1, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 5, 2, 1, 1, 1, 5, 1, 2, 1, 5, 2, 1, 2, 1, 1, 1, 5, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 5, 1, 2, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1, 1, 1, 1, 1, 2, 1, 1, 5, 2, 3, 1, 1, 4, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 1, 5, 2, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 5, 1, 1, 1, 1, 1, 2, 1, 1, 5, 1, 1, 1, 1, 2, 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 3, 3, 3, 3, 1, 1, 3, 1, 3, 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 1, 1, 1, 2, 1, 2, 1, 2, 1, 5, 5, 1, 2, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 5, 1, 2, 5, 1, 1, 1, 5, 5, 1, 1, 2, 2, 1, 1, 1, 1, 1, 2, 1, 2, 2, 1, 5, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 3, 1, 1, 1, 2, 1, 1, 1, 1, 1, 4, 1, 2, 2, 5, 4, 5, 1, 5, 3, 3, 3, 2, 1, 1, 5, 2, 1, 1, 1, 1, 5, 1, 1, 1, 1, 1, 1, 2, 1, 4, 3, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 2, 5, 1, 2, 1, 1, 1, 1, 3, 1, 1, 1, 1, 2, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 3, 1, 3, 3, 3, 1, 3, 1, 3, 3, 3, 3, 1, 3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 1, 1, 3, 1, 2, 4, 1, 1, 2, 3, 1, 1, 2, 1, 1, 1, 2, 2, 2, 1, 1, 1, 2, 2, 1, 1, 3, 1, 3, 4, 1, 3, 2, 3, 1, 3, 3, 3, 3, 1, 1, 2, 1, 2, 1, 2, 3, 2, 1, 1, 2, 2, 2, 3, 3, 3, 2, 2, 2, 1, 3, 1, 3, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 5, 1, 1, 1, 1, 2, 2, 5, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 4, 1, 4, 4, 5, 5, 4, 1, 4, 5, 4, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 2, 1, 2, 1, 2, 2, 1, 5]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y5Dx5pUEP2KU",
        "colab_type": "text"
      },
      "source": [
        "## Plot of number of citance in each discourse facet"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uIKJzZaYQO2z",
        "colab_type": "code",
        "outputId": "05a681ce-8caf-46ea-8ca3-342f52b886ce",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "from sklearn.metrics import precision_recall_fscore_support\n",
        "\n",
        "\n",
        "# print(cd2)\n",
        "newdict = {'Method':0,'Result':0,'Aim':0,'Hypothesis':0,'Implication':0}\n",
        "for i in y:  \n",
        "    if (i == 1):\n",
        "      newdict['Method'] += 1\n",
        "    elif (i == 2 ):\n",
        "      newdict['Result'] += 1\n",
        "    elif (i == 3):\n",
        "      newdict['Aim'] += 1\n",
        "    elif (i == 4):\n",
        "      newdict['Hypothesis'] += 1\n",
        "    elif (i == 5):\n",
        "      newdict['Implication'] += 1\n",
        "\n",
        "plt.figure()\n",
        "plt.bar(newdict.keys(), newdict.values(),0.5, color='#C1062B')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<BarContainer object of 5 artists>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAATR0lEQVR4nO3df7SdVX3n8fcHooAoRCDNokmmsZg1DMupkWYoWnQxMDqITsMsQWXRipSZLDu0au0v2ulqteOa4nJmsNpZTDNCQWtViqUgdVRWgOKPogT5kSBtTVFLMiCRQpRSmQLf+ePZF869Jrn35p6bm2zfr7XuOvvZzz7Ps599z/ncfZ5znnNTVUiS+nLAQndAkjR+hrskdchwl6QOGe6S1CHDXZI6tGihOwBw1FFH1cqVKxe6G5K0X7ntttu+XVVLdrZunwj3lStXsnHjxoXuhiTtV5J8c1frPC0jSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkd2ieuUJ2Lmw960UJ3AYBXPL55obsgSU9z5i5JHTLcJalDMwr3JN9IsinJHUk2trojklyf5Gvt9vmtPknen2RLkruSHD+fByBJ+n6zmbn/66paXVVr2vKFwIaqWgVsaMsArwZWtZ91wCXj6qwkaWbmclpmLXBFK18BnDFS/6Ea3AIsTnL0HPYjSZqlmYZ7AZ9NcluSda1uaVXd38oPAEtbeRlw38h9t7a6SZKsS7Ixycbt27fvQdclSbsy049CnlRV25L8EHB9kr8aXVlVlaRms+OqWg+sB1izZs2s7itJ2r0Zzdyralu7fRC4GjgB+NbE6ZZ2+2Brvg1YMXL35a1OkrSXTBvuSQ5N8ryJMvAqYDNwLXBua3YucE0rXwu8qX1q5kRgx8jpG0nSXjCT0zJLgauTTLT/46r6dJJbgSuTnA98E3h9a/8p4HRgC/AYcN7Yey1J2q1pw72q7gVevJP6h4BTd1JfwAVj6Z0kaY94haokdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR2acbgnOTDJ7Umua8svSPKlJFuSfDzJs1v9QW15S1u/cn66LknaldnM3N8G3DOy/B7g4qp6IfAwcH6rPx94uNVf3NpJkvaiGYV7kuXAa4APtuUApwBXtSZXAGe08tq2TFt/amsvSdpLZjpzfx/wq8BTbflI4JGqeqItbwWWtfIy4D6Atn5Haz9JknVJNibZuH379j3sviRpZ6YN9ySvBR6sqtvGueOqWl9Va6pqzZIlS8a5aUn6gbdoBm1+EvipJKcDBwOHAb8HLE6yqM3OlwPbWvttwApga5JFwOHAQ2PvuSRpl6aduVfVr1fV8qpaCbwRuKGqzgFuBM5szc4Frmnla9sybf0NVVVj7bUkabfm8jn3XwPekWQLwzn1S1v9pcCRrf4dwIVz66IkabZmclrmaVV1E3BTK98LnLCTNt8DzhpD3yRJe8grVCWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6tC04Z7k4CRfTnJnkruTvKvVvyDJl5JsSfLxJM9u9Qe15S1t/cr5PQRJ0lQzmbk/DpxSVS8GVgOnJTkReA9wcVW9EHgYOL+1Px94uNVf3NpJkvaiacO9Bo+2xWe1nwJOAa5q9VcAZ7Ty2rZMW39qkoytx5Kkac3onHuSA5PcATwIXA/8LfBIVT3RmmwFlrXyMuA+gLZ+B3DkTra5LsnGJBu3b98+t6OQJE0yo3CvqierajWwHDgBOHauO66q9VW1pqrWLFmyZK6bkySNmNWnZarqEeBG4KXA4iSL2qrlwLZW3gasAGjrDwceGktvJUkzMpNPyyxJsriVDwFeCdzDEPJntmbnAte08rVtmbb+hqqqcXZakrR7i6ZvwtHAFUkOZPhjcGVVXZfkq8DHkrwbuB24tLW/FPhwki3A3wNvnId+S5J2Y9pwr6q7gJfspP5ehvPvU+u/B5w1lt5JkvaIV6hKUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktShacM9yYokNyb5apK7k7yt1R+R5PokX2u3z2/1SfL+JFuS3JXk+Pk+CEnSZDOZuT8B/FJVHQecCFyQ5DjgQmBDVa0CNrRlgFcDq9rPOuCSsfdakrRb04Z7Vd1fVV9p5e8C9wDLgLXAFa3ZFcAZrbwW+FANbgEWJzl67D2XJO3SrM65J1kJvAT4ErC0qu5vqx4AlrbyMuC+kbttbXVTt7UuycYkG7dv3z7LbkuSdmfG4Z7kucAngLdX1XdG11VVATWbHVfV+qpaU1VrlixZMpu7SpKmMaNwT/IshmD/SFX9aav+1sTplnb7YKvfBqwYufvyVidJ2ktm8mmZAJcC91TV/xhZdS1wbiufC1wzUv+m9qmZE4EdI6dvJEl7waIZtPlJ4GeATUnuaHW/AVwEXJnkfOCbwOvbuk8BpwNbgMeA88baY0nStKYN96r6PJBdrD51J+0LuGCO/ZIkzYFXqEpShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1KFpwz3JZUkeTLJ5pO6IJNcn+Vq7fX6rT5L3J9mS5K4kx89n5yVJOzeTmfvlwGlT6i4ENlTVKmBDWwZ4NbCq/awDLhlPNyVJs7FougZVdXOSlVOq1wInt/IVwE3Ar7X6D1VVAbckWZzk6Kq6f1wd1u7dfNCLFroLALzi8c3TN5I0b/b0nPvSkcB+AFjaysuA+0babW11kqS9aM5vqLZZes32fknWJdmYZOP27dvn2g1J0og9DfdvJTkaoN0+2Oq3AStG2i1vdd+nqtZX1ZqqWrNkyZI97IYkaWf2NNyvBc5t5XOBa0bq39Q+NXMisMPz7ZK09037hmqSjzK8eXpUkq3AbwMXAVcmOR/4JvD61vxTwOnAFuAx4Lx56LMkaRoz+bTM2btYdepO2hZwwVw7JUmaG69QlaQOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOjTtd8tI+7N94T9T+V+ptBCcuUtSh5y5S/qBtC+8qoP5e2XnzF2SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CH/WYf0A6L3f06hyZy5S1KH5iXck5yW5K+TbEly4XzsQ5K0a2MP9yQHAv8TeDVwHHB2kuPGvR9J0q7Nx8z9BGBLVd1bVf8P+Biwdh72I0nahfl4Q3UZcN/I8lbgJ6Y2SrIOWNcWH03y1/PQl5k6Cvj2nLaQjKcn+wbH4xmOxWSOx2QLPR4/sqsVC/ZpmapaD6xfqP2PSrKxqtYsdD/2FY7HMxyLyRyPyfbl8ZiP0zLbgBUjy8tbnSRpL5mPcL8VWJXkBUmeDbwRuHYe9iNJ2oWxn5apqieS/DzwGeBA4LKqunvc+xmzfeL00D7E8XiGYzGZ4zHZPjseqaqF7oMkacy8QlWSOmS4S1KH9vtwT1JJ/mhkeVGS7Umum+Z+q5OcPrL8ziS/PId+zOn+45bkySR3JNmc5JNJFo95+99IclSSxUn+0zi3vbclOaM9jo5tyz+c5KqF7tdMJXl0yvKbk/z+mPdxcpKXjSxfnuTMMWz3g+O+gn3qeMxhOydP5EiSn9rTr1JJ8htTlr84jv5NZ78Pd+AfgBclOaQtv5KZffRyNXD6tK32X/9YVaur6kXA3wMXzNN+FgP7dbgDZwOfb7dU1f+tqjkHV2dOBl42XaPZqqr/UFVfHfd2x62qrq2qi/bw7pPCvarGPo4700O4A3wKeE0rnw18dGJFkkOTXJbky0luT7K2fUTzd4A3tNntG1rz45LclOTeJG8d2cY72gx4c5K3j9T/5yR/k+TzwD+f96Pcc3/JcOUwSY5J8ukktyX53Mhs9ax2fHcmubnVTZoBJrkuyclTtn0RcEwbx/funcMZnyTPBU4Czmf42C5JVibZ3MpvTvJnSa5vr1Z+vj0ebk9yS5IjFrD7u5XkeUm+nuRZbfmwieX2OP+9kVd3J7Q2R7Tjvasd348lWQm8BfjF1v7lbRevSPLF9nw5c2S/v5Lk1raNd7W6Q5P8eXt8bZ54zrV+rElyYHs1sDnJpiS/OIbjPznJXyS5pvXxoiTntCzYlOSY1u7yJP8rycb2fH7tTrb19HMhydIkV7djuXPiFU0bt9uS3J3hCnySXAQc0sbtI63u0XabJO8dOeY3jPT7piRXJfmrJB9J9uAy1qrar3+AR4EfA64CDgbuYJhlXNfW/1fgp1t5MfA3wKHAm4HfH9nOO4EvAgcxXFL8EPAs4MeBTe0+zwXuBl4yUv8c4DBgC/DLCz0eo+PSbg8E/gQ4rS1vAFa18k8AN7TyJmDZxDi126ljdB1wcit/o43TSmDzQh/vHMbpHODSVv5i+70+fUxtDLYAzwOWADuAt7R1FwNv3weO4cn2uJ/4+buJ3xvwh8AZrbwO+O+tfBPwv1v5FSPH+wHgt1v5FOCOVn7n6OMbuLw9rg5g+ILALa3+VQwfD0xbd13b/usm9tfaHT7SjzVt3K8fWb94DI/9k4FHgKMZntfbgHe1dW8D3jdyLJ9u/V3F8JUpBzM5R55+LgAfn/i9Mzy/Jo7liHZ7CLAZOHK0Pzvp3+uA69s2lrbf29FtvzsYLgA9gGFydtJsx6GLmXtV3cXwhDybYRY/6lXAhUnuYHggHQz8s11s6s+r6vGq+jbwIMOAnwRcXVX/UFWPAn8KvLz9XF1Vj1XVd9j3LtQ6pB3zAwzHcX2bpb4M+JO27g8YHkwAXwAuT/IfGR5sPyjOZvhyO9rt2Ttpc2NVfbeqtjM86T7Z6jcxPO4W2sQpuNVVtRr4rZF1HwTOa+XzGMJ+wkcBqupm4LAM78ucBHy41d8AHJnksF3s98+q6qkaTqssbXWvaj+3A18BjmUIzE3AK5O8J8nLq2rHlG3dC/xokg8kOQ34zizHYFdurar7q+px4G+Bz7b6qb+7K9uxfK315djdbPMU4BKAqnpy5FjemuRO4BaGq/RXTdO3k4CPtm18C/gL4F+1dV+uqq1V9RTDH+yVu9jGLvX0n5iuBf4bw1+9I0fqA7yuqiZ9MVmS7/syM+DxkfKT7N/j849VtTrJcxguKLuAYYbySAuASarqLW1MXgPcluTHgSeYfOru4Pnv9t7TTqmcAvzLJMXwR60YvrJ61Ojj4qmR5afYxx8jVfWFdprpZODAqhr9N0hTL3KZ7UUvo+OSkdvfrao/mNo4yfEM73O9O8mGqvqdkX4+nOTFwL9lOAX0euBnZ9mf6fq4u9/dnMaije+/AV5aVY8luYm5PV/mnEVdzNybyxhecm2aUv8Z4BcmzlkleUmr/y7DS+3pfA44I8lzkhwK/PtWd3OrPyTJ84B/N46DGLeqegx4K/BLwGPA15OcBU+f83txKx9TVV+qqt8CtjPMPL4BrE5yQJIVDF/nPNVMx3FfdCbw4ar6kapaWVUrgK8z+buRevAh4I+ZPGsHmDjHexKwo81AP8dwqmoisL7dXpnO9Pf8GeBn26tEkixL8kNJfhh4rKr+CHgvcPzonZIcBRxQVZ8AfnPq+r3grPY4Pwb4UWB331K7Afg5GP5/RZLDgcOBh1uwHwucONL+n9Le95jicwzv+x2YZAnD6asvj+NgYB+fdcxGVW0F3r+TVf8FeB9wV5IDGJ68rwVu5JnTNb+7m+1+JcnlPDPoH6yq2wGSfBy4k+EUzq1jOpSxq6rbk9zFcMrhHOCSJL/J8J7CxxiO4b1JVjHMvDa0OhjG66vAPQwvs6du+6EkX8jwBuT/qapfmfcDGp+zgfdMqfsE8OsL0Jf59BHg3Yx80KD5XpLbGR4HE7PkdwKXtcfLY8C5rf6TwFVJ1gK/sKsdVdVnk/wL4C/bfOpR4KeBFzI8xp4C/okWjiOWAX/YnqOw938Hf8fwHD+M4T2V7+3mPcy3AeuTnM8wq/45hnP2b0lyD8MfhltG2q9nyJ+vVNU5I/VXAy9leK4V8KtV9UD74zBnfv2A1LkMn2RZW1U/M1J3E8MbpBsXrGP7iDZ5u66q9ptrG2aim5m7pO+X5AMM//Ky52s6tBPO3CWpQz29oSpJagx3SeqQ4S5JHTLcJalDhrskdej/Aw82ky0m31aJAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XwZSjyHdVEYl",
        "colab_type": "code",
        "outputId": "62158dc0-ad9a-456c-bb54-0df5e0d426e4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "\n",
        "# print(cd2)\n",
        "newdict = {'Method':0,'Non-method':0}\n",
        "for i in y:  \n",
        "    if (i == 1):\n",
        "      newdict['Method'] += 1\n",
        "    else:\n",
        "      newdict['Non-method'] += 1\n",
        "\n",
        "plt.figure()\n",
        "plt.bar(newdict.keys(), newdict.values(),0.4, color='#C1062B')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<BarContainer object of 2 artists>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAO9ElEQVR4nO3cf6yeZX3H8fdnVEDwR5V2DWvrSmI307hZXMMwOOMkGsFpmb+Jk4416T9MYY5MNpcNp1t0mcORLSSNMIrbVHQilTEdKRjmD9BTwAIy9YxIaIP0iIgiE4d898dz1TzUc3qe0/OcHnrt/UqePNd9Xdd939+n9P5w93p+pKqQJPXl5xa7AEnS+BnuktQhw12SOmS4S1KHDHdJ6tCSxS4AYNmyZbVmzZrFLkOSDis7d+78TlUtn27sSRHua9asYWJiYrHLkKTDSpJ7ZhpzWUaSOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjr0pPiG6nzceNTzF7uEbr3k0TsWuwRJB8k7d0nqkOEuSR0aKdyTfCvJ7UluSzLR+p6d5Lok32zPz2r9SXJxkskku5K8cCFfgCTpZ83lzv03q2p9VW1o2xcAO6pqLbCjbQOcBqxtjy3AJeMqVpI0mvksy2wEtrX2NuCMof4rauAmYGmS4+dxHknSHI0a7gX8R5KdSba0vhVVdV9rfxtY0dorgXuH9t3d+p4gyZYkE0kmpqamDqJ0SdJMRv0o5Iurak+SnweuS/Jfw4NVVUlqLieuqq3AVoANGzbMaV9J0oGNdOdeVXva817gKuAk4P59yy3teW+bvgdYPbT7qtYnSTpEZg33JMcmefq+NvAK4A5gO7CpTdsEXN3a24Gz2qdmTgYeGlq+kSQdAqMsy6wArkqyb/6/VNVnknwFuDLJZuAe4I1t/rXA6cAk8Ahw9tirliQd0KzhXlV3Ay+Ypv8B4NRp+gs4ZyzVSZIOit9QlaQOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQyOHe5Ijktya5Jq2fUKSm5NMJvlYkiNb/1Fte7KNr1mY0iVJM5nLnfu5wF1D2+8HLqqq5wIPAptb/2bgwdZ/UZsnSTqERgr3JKuAVwEfatsBXgZ8ok3ZBpzR2hvbNm381DZfknSIjHrn/kHgj4DH2/ZxwPeq6rG2vRtY2dorgXsB2vhDbf4TJNmSZCLJxNTU1EGWL0mazqzhnuS3gL1VtXOcJ66qrVW1oao2LF++fJyHlqT/95aMMOcU4DVJTgeOBp4B/B2wNMmSdne+CtjT5u8BVgO7kywBngk8MPbKJUkzmvXOvar+uKpWVdUa4M3A9VX1FuAG4PVt2ibg6tbe3rZp49dXVY21aknSAc3nc+7vBN6RZJLBmvqlrf9S4LjW/w7ggvmVKEmaq1GWZX6qqj4HfK617wZOmmbOj4A3jKE2SdJB8huqktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1aNZwT3J0ki8n+WqSO5O8u/WfkOTmJJNJPpbkyNZ/VNuebONrFvYlSJL2N8qd+6PAy6rqBcB64JVJTgbeD1xUVc8FHgQ2t/mbgQdb/0VtniTpEJo13Gvg4bb5lPYo4GXAJ1r/NuCM1t7YtmnjpybJ2CqWJM1qpDX3JEckuQ3YC1wH/Dfwvap6rE3ZDaxs7ZXAvQBt/CHguGmOuSXJRJKJqamp+b0KSdITjBTuVfWTqloPrAJOAp433xNX1daq2lBVG5YvXz7fw0mShszp0zJV9T3gBuBFwNIkS9rQKmBPa+8BVgO08WcCD4ylWknSSEb5tMzyJEtb+6nAy4G7GIT869u0TcDVrb29bdPGr6+qGmfRkqQDWzL7FI4HtiU5gsH/DK6sqmuSfA34aJL3ArcCl7b5lwIfTjIJfBd48wLULUk6gFnDvap2ASdO0383g/X3/ft/BLxhLNVJkg6K31CVpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdWrLYBUh68rvxqOcvdgndesmjdyzIcb1zl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHVo1nBPsjrJDUm+luTOJOe2/mcnuS7JN9vzs1p/klycZDLJriQvXOgXIUl6olHu3B8D/rCq1gEnA+ckWQdcAOyoqrXAjrYNcBqwtj22AJeMvWpJ0gHNGu5VdV9V3dLaPwDuAlYCG4Ftbdo24IzW3ghcUQM3AUuTHD/2yiVJM5rTmnuSNcCJwM3Aiqq6rw19G1jR2iuBe4d229369j/WliQTSSampqbmWLYk6UBGDvckTwP+FTivqr4/PFZVBdRcTlxVW6tqQ1VtWL58+Vx2lSTNYqRwT/IUBsH+z1X1ydZ9/77llva8t/XvAVYP7b6q9UmSDpFRPi0T4FLgrqr626Gh7cCm1t4EXD3Uf1b71MzJwENDyzeSpENglN9zPwV4K3B7ktta358A7wOuTLIZuAd4Yxu7FjgdmAQeAc4ea8WSpFnNGu5V9XkgMwyfOs38As6ZZ12SpHnwG6qS1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHVo1nBPclmSvUnuGOp7dpLrknyzPT+r9SfJxUkmk+xK8sKFLF6SNL1R7twvB165X98FwI6qWgvsaNsApwFr22MLcMl4ypQkzcWs4V5VNwLf3a97I7CttbcBZwz1X1EDNwFLkxw/rmIlSaM52DX3FVV1X2t/G1jR2iuBe4fm7W59kqRDaN5vqFZVATXX/ZJsSTKRZGJqamq+ZUiShhxsuN+/b7mlPe9t/XuA1UPzVrW+n1FVW6tqQ1VtWL58+UGWIUmazsGG+3ZgU2tvAq4e6j+rfWrmZOChoeUbSdIhsmS2CUk+ArwUWJZkN/DnwPuAK5NsBu4B3timXwucDkwCjwBnL0DNkqRZzBruVXXmDEOnTjO3gHPmW5QkaX78hqokdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQwsS7klemeTrSSaTXLAQ55AkzWzs4Z7kCOAfgNOAdcCZSdaN+zySpJktxJ37ScBkVd1dVT8GPgpsXIDzSJJmsGQBjrkSuHdoezfw6/tPSrIF2NI2H07y9QWo5cloGfCdxS5iJMliVyAdjMPnGoP5Xme/ONPAQoT7SKpqK7B1sc6/WJJMVNWGxa5D6pXX2MBCLMvsAVYPba9qfZKkQ2Qhwv0rwNokJyQ5EngzsH0BziNJmsHYl2Wq6rEkvw98FjgCuKyq7hz3eQ5j/++WoqRDzGsMSFUtdg2SpDHzG6qS1CHDXZI6ZLiPKEkl+aeh7SVJppJcM8t+65OcPrR9YZLz51HHvPaXDpV2zXxgaPv8JBcuYkk/leS8JMcMbT88z+PNa/+FYLiP7ofA85M8tW2/nNE+4rkeOH3WWVJ/HgVem2TZYhcyjfOAY2addRgz3OfmWuBVrX0m8JF9A0mOTXJZki8nuTXJxvZR0L8A3pTktiRvatPXJflckruTvH3oGO9Ickd7nDfU/64k30jyeeCXF/xVSuPxGINPrvzB/gNJ1iS5PsmuJDuSPKf1X57k4iRfbNfH66c7cJt3SZKb2ryXtuvvriSXD817RZIvJbklyceTPK1dc78A3JDkhqG5f5nkq+2YK2ap84R23NuTvHeMf2bjU1U+RngADwO/CnwCOBq4DXgpcE0b/yvgd1p7KfAN4Fjgd4G/HzrOhcAXgaMYfE36AeApwK8Bt7d9ngbcCZw41H8M8AxgEjh/sf88fPiY7dGumWcA3wKeCZwPXNjGPg1sau3fAz7V2pcDH2dw47mOwe9UTXfsyxn8blUY/HbV94FfafvtZPAv5mXAjcCxbZ93An/W2t8Clg0dr4BXt/ZfA386S53bgbNa+xzg4cX+897/sWg/P3A4qqpdSdYwuGu/dr/hVwCvGVoPPxp4zgyH+reqehR4NMleYAXwYuCqqvohQJJPAr/B4C/rVVX1SOv3C2E6bFTV95NcAbwd+J+hoRcBr23tDzMI1H0+VVWPA1/bdwc9g09XVSW5Hbi/qm4HSHInsIbBt+PXAV/I4PdbjgS+NMOxfgzse/9sJ4Nl1wPVeQrwuqH+9x+gzkVhuM/dduBvGNy1HzfUH+B1VfWEH0BL8jM/msZgLXKfn+B/B/Xtg8AtwD+OOH/4+ggMlkxoS6JVtX6/eY/vt8/jDK6pnwDXVdWZI5zzf6vdhjP6Nfmk/pKQa+5zdxnw7n13CUM+C7wt7RYhyYmt/wfA00c47n8CZyQ5JsmxwG+3vhtb/1OTPB149ThehHSoVNV3gSuBzUPdX2Tw0yQAb2Hwd/1Ax3hXVa0fCvZR3ASckuS58NP3xX6pjY16Xc5U5xf263/SMdznqKp2V9XF0wy9h8Ha+a72z8L3tP4bGLyBOvyG6nTHvYXBOuKXgZuBD1XVra3/Y8BXgX9n8Ns90uHmAwzWwPd5G3B2kl3AW4Fzx33Cqppi8J7XR9p5vgQ8rw1vBT4z/IbqDGaq81zgnLYktHLctY+DPz8gSR3yzl2SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA79HyBr5lihOMPdAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "meny8V0aaBuC",
        "colab_type": "text"
      },
      "source": [
        "## Binary Classifiers\n",
        "These classifiers will separate method_citation from non-methods one."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jfZq_swTaSnl",
        "colab_type": "code",
        "outputId": "6d079750-e2a0-4ae5-dc7a-17925b6a2283",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "y_multi_class = y\n",
        "y = []\n",
        "for i in y_multi_class:\n",
        "  if(i != 1):\n",
        "    y.append(0)\n",
        "  else:\n",
        "    y.append(1)\n",
        "print(y)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "45GCjlbl4llK",
        "colab_type": "text"
      },
      "source": [
        "1) Logistic regression"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EpKD7mma6Soh",
        "colab_type": "code",
        "outputId": "b45644ec-75a5-4217-d4cb-b5d31c0b210e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        }
      },
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.metrics import average_precision_score\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import accuracy_score\n",
        "import warnings\n",
        "import numpy as np\n",
        "from imblearn.metrics import classification_report_imbalanced\n",
        "X = np.array(X)\n",
        "y = np.array(y)\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(\n",
        "    X, y, test_size=0.20, random_state=42)\n",
        "\n",
        "print(X_train.shape,y_test.shape)\n",
        "clf = LogisticRegression(random_state=0).fit(X_train, y_train)\n",
        "y_pred = clf.predict(X_test)\n",
        "print(accuracy_score(y_test, y_pred, normalize=False))\n",
        "print(precision_recall_fscore_support(y_test, y_pred, average='micro'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(596, 2) (149,)\n",
            "106\n",
            "(0.7114093959731543, 0.7114093959731543, 0.7114093959731543, None)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.metrics.classification module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.metrics. Anything that cannot be imported from sklearn.metrics is now part of the private API.\n",
            "  warnings.warn(message, FutureWarning)\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f4YREXDJ4sAr",
        "colab_type": "text"
      },
      "source": [
        "2)SVM"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VRBXUlzT6x5-",
        "colab_type": "code",
        "outputId": "e7561af6-deae-4139-f0a8-ed72cfc053fd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "from sklearn.svm import LinearSVC\n",
        "weights = {0:1, 1:1.0}\n",
        "clf = LinearSVC(random_state=0, tol=1e-5, max_iter = 1000000,class_weight = weights)\n",
        "X_train, X_test, y_train, y_test = train_test_split(\n",
        "    X, y, test_size=0.20, random_state=42)\n",
        "\n",
        "print(X_train.shape,y_test.shape)\n",
        "clf.fit(X_train, y_train)\n",
        "print(accuracy_score(y_test, y_pred, normalize=False))\n",
        "y_pred = clf.predict(X_test)\n",
        "print(precision_recall_fscore_support(y_test, y_pred, average='micro'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(596, 2) (149,)\n",
            "106\n",
            "(0.6912751677852349, 0.6912751677852349, 0.6912751677852349, None)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VsX5GQZF434j",
        "colab_type": "text"
      },
      "source": [
        "3)Random Forest"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kOviMdHp7zrK",
        "colab_type": "code",
        "outputId": "cfafd0f4-5dca-48ab-fe7a-79d98cddc881",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "binary_model = RandomForestClassifier(n_estimators=1000)\n",
        "binary_model.fit(X_train , y_train)\n",
        "y_pred = binary_model.predict(X_test)\n",
        "target_names = [0, 1]\n",
        "print(binary_model.score(X_test,y_test))\n",
        "print(precision_recall_fscore_support(y_test, y_pred, average='micro'))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.7315436241610739\n",
            "(0.7315436241610739, 0.7315436241610739, 0.7315436241610739, None)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DLaigC6keV0b",
        "colab_type": "code",
        "outputId": "7bd2e4cd-5127-4bb7-f2a4-a958c27a2a19",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from numpy import mean\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.model_selection import RepeatedStratifiedKFold\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "\n",
        "# define model\n",
        "weights = {0:2, 1:1}\n",
        "model = LogisticRegression(solver='lbfgs', class_weight=weights)\n",
        "model.fit(X_train,y_train)\n",
        "model.score(X_test,y_test)\n",
        "y_pred = model.predict(X_test)\n",
        "print(precision_recall_fscore_support(y_test, y_pred, average='micro'))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(0.6577181208053692, 0.6577181208053692, 0.6577181208053692, None)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rko5wqgEfstN",
        "colab_type": "code",
        "outputId": "f0597f13-6352-43b7-de3e-31e006c8bca5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "i_class0 = np.where(y_train == 0)[0]\n",
        "i_class1 = np.where(y_train == 1)[0]\n",
        "n_class0 = len(i_class0)\n",
        "n_class1 = len(i_class1)\n",
        "print(n_class0,n_class1)\n",
        "# For every observation of class 0, randomly sample from class 1 without replacement\n",
        "i_class1_downsampled = np.random.choice(i_class1, size=n_class0, replace=False)\n",
        "downsampled = np.concatenate((i_class1_downsampled ,i_class0))\n",
        "X_train_downsampled = np.take(X_train,downsampled,axis = 0)\n",
        "y_train_downsampled = np.take(y_train,downsampled)\n",
        "\n",
        "# print(X_train_downsampled)\n",
        "\n",
        "clf = LogisticRegression(random_state=0).fit(X_train_downsampled, y_train_downsampled)\n",
        "# clf.predict(X_test)\n",
        "clf.score(X_test,y_test)\n",
        "y_pred = clf.predict(X_test)\n",
        "print(precision_recall_fscore_support(y_test, y_pred, average='micro'))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "187 409\n",
            "(0.5906040268456376, 0.5906040268456376, 0.5906040268456376, None)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d5tBDqb7cOGq",
        "colab_type": "text"
      },
      "source": [
        "## Multi-Class Classifiers\n",
        "After classifying into method and non-method. We apply multi-class classifier on non-method citances to further classify into Aim, Results, Hypothesis, Implication."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7SaeMjADh-wu",
        "colab_type": "code",
        "outputId": "2e2fdf37-cbab-4c1d-9ad8-b884c9db317d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "Xm = []\n",
        "ym = []\n",
        "c = 0\n",
        "for i in y_multi_class:\n",
        "  if(i!=1):\n",
        "    Xm.append(X[c])\n",
        "    ym.append(y_multi_class[c])\n",
        "  c+= 1\n",
        "  \n",
        "print(ym)\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[5, 5, 3, 5, 3, 5, 5, 2, 5, 3, 2, 2, 2, 2, 3, 2, 3, 5, 5, 3, 5, 5, 5, 3, 3, 5, 3, 3, 5, 3, 2, 2, 3, 2, 2, 2, 5, 2, 2, 2, 5, 5, 2, 2, 2, 5, 2, 2, 4, 2, 5, 3, 3, 3, 3, 2, 5, 2, 2, 2, 2, 5, 2, 5, 2, 5, 2, 2, 5, 2, 2, 5, 2, 2, 2, 2, 2, 5, 2, 3, 4, 2, 5, 5, 2, 2, 2, 2, 5, 2, 5, 2, 5, 5, 3, 3, 3, 3, 3, 3, 5, 5, 2, 2, 2, 5, 5, 2, 2, 2, 3, 5, 2, 5, 5, 5, 2, 2, 2, 2, 2, 5, 2, 2, 3, 2, 4, 2, 2, 5, 4, 5, 5, 3, 3, 3, 2, 5, 2, 5, 2, 4, 3, 2, 2, 2, 2, 2, 2, 2, 5, 2, 3, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 3, 2, 4, 2, 3, 2, 2, 2, 2, 2, 2, 3, 3, 4, 3, 2, 3, 3, 3, 3, 3, 2, 2, 2, 3, 2, 2, 2, 2, 3, 3, 3, 2, 2, 2, 3, 3, 2, 2, 3, 5, 2, 2, 5, 2, 2, 2, 4, 4, 4, 5, 5, 4, 4, 5, 4, 5, 2, 2, 2, 2, 5]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Mweze5C59rY",
        "colab_type": "code",
        "outputId": "2b509980-411c-42b0-f548-89498270f3fd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(X_train.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(596, 2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gFahHXY8end_",
        "colab_type": "text"
      },
      "source": [
        "### Training Multi-class models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0xS355yhdgBl",
        "colab_type": "code",
        "outputId": "0b8dcf23-c062-41af-952c-a275a8b569e7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "Xm = np.array(Xm)\n",
        "ym = np.array(ym)\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(\n",
        "    Xm, ym, test_size=0.20, random_state=42)\n",
        "\n",
        "print(X_train.shape,y_test.shape)\n",
        "clf = LogisticRegression(random_state=0).fit(X_train, y_train)\n",
        "y_pred = clf.predict(X_test)\n",
        "print(accuracy_score(y_test, y_pred))\n",
        "print(precision_recall_fscore_support(y_test, y_pred, average='micro'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(184, 2) (46,)\n",
            "0.4782608695652174\n",
            "(0.4782608695652174, 0.4782608695652174, 0.4782608695652174, None)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZkScwJ0oeRcO",
        "colab_type": "code",
        "outputId": "99e0c754-f59f-446d-b002-e114da49b5ff",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "multi_model = RandomForestClassifier(n_estimators=1000)\n",
        "multi_model.fit(X_train , y_train)\n",
        "y_pred = multi_model.predict(X_test)\n",
        "print(accuracy_score(y_test, y_pred))\n",
        "print(precision_recall_fscore_support(y_test, y_pred, average='micro'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.5434782608695652\n",
            "(0.5434782608695652, 0.5434782608695652, 0.5434782608695652, None)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ika-uJ66ev88",
        "colab_type": "text"
      },
      "source": [
        "## Testing on Binary and multi-class classifiers\n",
        "(Only Positional Features)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GcGxHEtghd0R",
        "colab_type": "text"
      },
      "source": [
        "First we apply the binary classifier"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NCkJstmNebsw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# We have trained a binary random forest classifier\n",
        "X = np.array(X)\n",
        "y = np.array(y_multi_class)\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(\n",
        "    X, y, test_size=0.20, random_state=42)\n",
        "y_pred = binary_model.predict(X_test)\n",
        "\n",
        "# print(X.shape,y.shape,y_pred.shape)\n",
        "# print(binary_model.score(X_test,y_test))\n",
        "# print(precision_recall_fscore_support(y_test, y_pred, average='micro'))\n",
        "# print(y_test)\n",
        "# print(classification_report_imbalanced(y_test, y_pred))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E5nlE4R_hjzY",
        "colab_type": "text"
      },
      "source": [
        "Then we apply the multi-class classifier on top of it"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wzclUjD9gGp8",
        "colab_type": "code",
        "outputId": "568c3384-a320-4313-acf5-a5f22b2f2667",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84
        }
      },
      "source": [
        "Xm_test = []\n",
        "ym_test = []\n",
        "c = 0\n",
        "for i in y_pred:\n",
        "  if(i!=1):\n",
        "    Xm_test.append(X_test[c])\n",
        "    ym_test.append(y_test[c])\n",
        "  c+=1\n",
        "print(ym_test)\n",
        "\n",
        "y_pred = multi_model.predict(Xm_test)\n",
        "print(multi_model.score(Xm_test,ym_test))\n",
        "print(accuracy_score(ym_test, y_pred))\n",
        "print(precision_recall_fscore_support(ym_test, y_pred, average='micro'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1, 2, 3, 2, 2, 1, 2, 3, 5, 4, 3, 1, 1, 1, 1, 2, 1, 1, 1, 1, 3, 2, 5, 1, 5, 3, 1, 1, 1, 2, 2, 1, 3, 2, 1, 1, 3]\n",
            "0.43243243243243246\n",
            "0.43243243243243246\n",
            "(0.43243243243243246, 0.43243243243243246, 0.43243243243243246, None)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "arTDkIS-WiLR",
        "colab_type": "text"
      },
      "source": [
        "## BERT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pMBjPGUcj7hh",
        "colab_type": "code",
        "outputId": "68bf5bef-f691-4e90-fbdc-b0952f1c814a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 339
        }
      },
      "source": [
        "!pip install pytorch-pretrained-bert"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: pytorch-pretrained-bert in /usr/local/lib/python3.6/dist-packages (0.6.2)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (2019.12.20)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (4.38.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (1.18.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (2.23.0)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (1.12.47)\n",
            "Requirement already satisfied: torch>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from pytorch-pretrained-bert) (1.5.0+cu101)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-pretrained-bert) (2020.4.5.1)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-pretrained-bert) (2.9)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-pretrained-bert) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-pretrained-bert) (3.0.4)\n",
            "Requirement already satisfied: botocore<1.16.0,>=1.15.47 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-pretrained-bert) (1.15.47)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-pretrained-bert) (0.3.3)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-pretrained-bert) (0.9.5)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch>=0.4.1->pytorch-pretrained-bert) (0.16.0)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.47->boto3->pytorch-pretrained-bert) (2.8.1)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.47->boto3->pytorch-pretrained-bert) (0.15.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.16.0,>=1.15.47->boto3->pytorch-pretrained-bert) (1.12.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NUXlBrDGWnjz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM\n",
        "\n",
        "# OPTIONAL: if you want to have more information on what's happening, activate the logger as follows\n",
        "import logging\n",
        "#logging.basicConfig(level=logging.INFO)\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "% matplotlib inline\n",
        "\n",
        "# Load pre-trained model tokenizer (vocabulary)\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lsxJl9AtaTJA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# for rfile in files:\n",
        "#   try :\n",
        "#     open_file = open(rfile+\"/annotation/\"+rfile+\".ann.txt\", 'r')\n",
        "#   except:\n",
        "#     open_file = open(rfile+\"/annotation/\"+rfile+\".annv3.txt\", 'r')\n",
        "#   citances = open_file.readlines()\n",
        "#   for citance in citances:\n",
        "#     if len(citance) > 1:\n",
        "#       data_file = citance.split(\" | \")\n",
        "#       if(len(data_file) == 11):\n",
        "#         R_text = (data_file[8].split(\":\")[1])\n",
        "#         R = R_text.split('>')[1]\n",
        "#         R = R.split('<')[0]\n",
        "#         marked_text = \"[CLS] \" + R + \" [SEP]\"\n",
        "#         tokenized_text = tokenizer.tokenize(marked_text)\n",
        "#         indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)\n",
        "\n",
        "\n",
        "#         segments_ids = [1] * len(tokenized_text)\n",
        "\n",
        "#         # print (segments_ids)  \n",
        "#         tokens_tensor = torch.tensor([indexed_tokens])\n",
        "#         segments_tensors = torch.tensor([segments_ids])\n",
        "\n",
        "#         # Load pre-trained model (weights)\n",
        "#         model = BertModel.from_pretrained('bert-base-uncased')\n",
        "\n",
        "#         # Put the model in \"evaluation\" mode, meaning feed-forward operation.\n",
        "#         model.eval()\n",
        "\n",
        "#         with torch.no_grad():\n",
        "#           encoded_layers, _ = model(tokens_tensor, segments_tensors)\n",
        "#         token_embeddings = torch.stack(encoded_layers, dim=0)\n",
        "#         token_embeddings = torch.squeeze(token_embeddings, dim=1)\n",
        "#         token_embeddings = token_embeddings.permute(1,0,2)\n",
        "#         token_vecs = encoded_layers[11][0]\n",
        "#         sentence_embedding = torch.mean(token_vecs, dim=0)\n",
        "#         print (\"Our final sentence embedding vector of shape:\", sentence_embedding.size())\n",
        "\n",
        "        "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jDqsjuX8YJp6",
        "colab_type": "code",
        "outputId": "07087c86-7f06-4bfe-d8f0-9c3f44935de4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 524
        }
      },
      "source": [
        "!pip install sentence-transformers\n",
        "from sentence_transformers import SentenceTransformer\n",
        "bert_model = SentenceTransformer('bert-base-nli-mean-tokens')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: sentence-transformers in /usr/local/lib/python3.6/dist-packages (0.2.6.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (1.18.3)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (1.4.1)\n",
            "Requirement already satisfied: transformers>=2.8.0 in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (2.8.0)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (3.2.5)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (0.22.2.post1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (4.38.0)\n",
            "Requirement already satisfied: torch>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (1.5.0+cu101)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers>=2.8.0->sentence-transformers) (0.7)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers>=2.8.0->sentence-transformers) (1.12.47)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers>=2.8.0->sentence-transformers) (2019.12.20)\n",
            "Requirement already satisfied: tokenizers==0.5.2 in /usr/local/lib/python3.6/dist-packages (from transformers>=2.8.0->sentence-transformers) (0.5.2)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (from transformers>=2.8.0->sentence-transformers) (0.1.86)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers>=2.8.0->sentence-transformers) (3.0.12)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers>=2.8.0->sentence-transformers) (2.23.0)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers>=2.8.0->sentence-transformers) (0.0.41)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from nltk->sentence-transformers) (1.12.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sentence-transformers) (0.14.1)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch>=1.0.1->sentence-transformers) (0.16.0)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.8.0->sentence-transformers) (0.9.5)\n",
            "Requirement already satisfied: botocore<1.16.0,>=1.15.47 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.8.0->sentence-transformers) (1.15.47)\n",
            "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.8.0->sentence-transformers) (0.3.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.8.0->sentence-transformers) (2020.4.5.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.8.0->sentence-transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.8.0->sentence-transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.8.0->sentence-transformers) (2.9)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers>=2.8.0->sentence-transformers) (7.1.2)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.47->boto3->transformers>=2.8.0->sentence-transformers) (0.15.2)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.47->boto3->transformers>=2.8.0->sentence-transformers) (2.8.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P-XWu6Q8ebY1",
        "colab_type": "code",
        "outputId": "5ba01b29-7b17-4459-f89e-7c1e41f3701d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import numpy as np\n",
        "en = bert_model.encode([\"Here is the sentence I want embeddings for.\"])\n",
        "np.shape(en)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1, 768)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bp5t8Wt8t4JK",
        "colab_type": "text"
      },
      "source": [
        "## BERT sentence embedding with Positional encoding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t74fsH6_erGU",
        "colab_type": "code",
        "outputId": "986a17a5-aa7f-4bbc-e1a3-8ce6c237b3b3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "X = []\n",
        "for rfile in files:\n",
        "  try :\n",
        "    open_file = open(rfile+\"/annotation/\"+rfile+\".ann.txt\", 'r')\n",
        "  except:\n",
        "    open_file = open(rfile+\"/annotation/\"+rfile+\".annv3.txt\", 'r')\n",
        "  citances = open_file.readlines()\n",
        "  for citance in citances:\n",
        "    if len(citance) > 1:\n",
        "      data_file = citance.split(\" | \")\n",
        "      if(len(data_file) == 11):\n",
        "        R_text = (data_file[8].split(\":\")[1])\n",
        "        R = R_text.split('>')[1]\n",
        "        R = R.split('<')[0]\n",
        "        en = bert_model.encode([R])\n",
        "        en = np.array(en)\n",
        "        R_text = (data_file[8].split(\":\")[1])\n",
        "        Facet = (data_file[9].split(\":\")[1])\n",
        "        result = R_text.find('sid') \n",
        "        temp = (R_text[result:result+10])\n",
        "        t1 = temp.split('\"')[1]\n",
        "        # print(t1)\n",
        "        result = R_text.find('ssid') \n",
        "        temp = (R_text[result:result+11])\n",
        "        t2 = 0\n",
        "        try:\n",
        "          t2 = temp.split('\"')[1]\n",
        "        except:\n",
        "          t2 = 0\n",
        "        # print(t2)\n",
        "        t = [int(t1), int(t2)]\n",
        "        # print(t)\n",
        "        en = en.T\n",
        "        t = np.array(t)\n",
        "        w = np.append(en,t)\n",
        "\n",
        "        X.append(w)\n",
        "\n",
        "print(len(X))\n",
        "        "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "745\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2CPjQ0vdewT5",
        "colab_type": "code",
        "outputId": "10cce37d-c4a4-4fb7-b235-c9026822af5a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "X = np.array(X)\n",
        "print(X.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(745, 770)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J7rxt1Jsp7U1",
        "colab_type": "text"
      },
      "source": [
        "### Training models over new features\n",
        "(Positional Features + BERT sentence embedding)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kdEDZOXMp_0N",
        "colab_type": "text"
      },
      "source": [
        "1) Logistic Regression"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f0dW94smk9Vl",
        "colab_type": "code",
        "outputId": "6d8c2d26-0bc4-4e8c-d93a-a0f2ad9866fd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.metrics import average_precision_score\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "X = np.array(X)\n",
        "y = np.array(y)\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(\n",
        "    X, y, test_size=0.20, random_state=42)\n",
        "\n",
        "print(X_train.shape,y_test.shape)\n",
        "clf = LogisticRegression(random_state=0,max_iter=100000).fit(X_train, y_train)\n",
        "y_pred =  clf.predict(X_test)\n",
        "print(clf.score(X_test,y_test))\n",
        "print(precision_recall_fscore_support(y_test, y_pred, average='micro'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(596, 770) (149,)\n",
            "0.7046979865771812\n",
            "(0.7046979865771812, 0.7046979865771812, 0.7046979865771812, None)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bqHP3i0qnn_P",
        "colab_type": "code",
        "outputId": "2b5d8789-8080-4471-dcce-564d0c18afd7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "y_multi_class = y\n",
        "y = []\n",
        "for i in y_multi_class:\n",
        "  if(i != 1):\n",
        "    y.append(0)\n",
        "  else:\n",
        "    y.append(1)\n",
        "print(y)\n",
        "X = np.array(X)\n",
        "y = np.array(y)\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(\n",
        "    X, y, test_size=0.20, random_state=42)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aV2i1NyCqMBp",
        "colab_type": "text"
      },
      "source": [
        "Training Binary Random Forest Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FO6MIe6AnthN",
        "colab_type": "code",
        "outputId": "055b0f11-5914-48b6-c82d-125e5122d2b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "binary_model = RandomForestClassifier(n_estimators=1000)\n",
        "binary_model.fit(X_train , y_train)\n",
        "y_pred = binary_model.predict(X_test)\n",
        "print(binary_model.score(X_test,y_test))\n",
        "print(precision_recall_fscore_support(y_test, y_pred, average='micro'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.7583892617449665\n",
            "(0.7583892617449665, 0.7583892617449665, 0.7583892617449665, None)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yqSAVUoppLnN",
        "colab_type": "code",
        "outputId": "2b8971b8-c996-4ee5-ff0a-ce985f7162f3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "Xm = []\n",
        "ym = []\n",
        "c = 0\n",
        "for i in y_multi_class:\n",
        "  if(i!=1):\n",
        "    Xm.append(X[c])\n",
        "    ym.append(y_multi_class[c])\n",
        "  c+= 1\n",
        "  \n",
        "print(len(ym))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "230\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Rh3YoRoGqSCx",
        "colab_type": "text"
      },
      "source": [
        "Training Multi class model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0_wngeqHpl0U",
        "colab_type": "code",
        "outputId": "c9e4b08a-cd11-4e2d-af6a-ac1523ab4f94",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "Xm = np.array(Xm)\n",
        "ym = np.array(ym)\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(\n",
        "    Xm, ym, test_size=0.20, random_state=42)\n",
        "\n",
        "multi_model = RandomForestClassifier(n_estimators=1000)\n",
        "multi_model.fit(X_train , y_train)\n",
        "y_pred = multi_model.predict(X_test)\n",
        "print(y_pred)\n",
        "print(multi_model.score(X_test,y_test))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[4 2 5 2 2 3 3 2 2 4 3 5 3 3 4 5 5 2 3 3 2 3 3 3 3 5 2 2 2 2 3 2 2 5 2 2 3\n",
            " 2 2 5 5 3 2 2 5 2]\n",
            "0.7391304347826086\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ddUxLJ-tstiI",
        "colab_type": "text"
      },
      "source": [
        "## Testing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XC7_pBvWORKa",
        "colab_type": "text"
      },
      "source": [
        "### Binary Classifier"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3lv89qQ2prHf",
        "colab_type": "code",
        "outputId": "9d10edd9-fa21-4750-e390-4c2855bbf268",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "X = np.array(X)\n",
        "y = np.array(y_multi_class)\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(\n",
        "    X, y, test_size=0.20, random_state=42)\n",
        "y_pred = binary_model.predict(X_test)\n",
        "\n",
        "print(X.shape,y.shape,y_pred.shape)\n",
        "print(binary_model.score(X_test,y_test))\n",
        "print(precision_recall_fscore_support(y_test, y_pred, average='micro'))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(745, 770) (745,) (149,)\n",
            "0.6241610738255033\n",
            "(0.6241610738255033, 0.6241610738255033, 0.6241610738255033, None)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d8Z88awROZZ6",
        "colab_type": "text"
      },
      "source": [
        "### Multi-class classifier"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OJ1z-BSPpuj_",
        "colab_type": "code",
        "outputId": "9d6a59ba-6675-4aee-80d3-e6b283ab2ce4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "Xm_test = []\n",
        "ym_test = []\n",
        "c = 0\n",
        "for i in y_pred:\n",
        "  if(i!=1):\n",
        "    Xm_test.append(X_test[c])\n",
        "    ym_test.append(y_test[c])\n",
        "  c+=1\n",
        "print(ym_test)\n",
        "\n",
        "print(multi_model.predict(Xm_test))\n",
        "print(multi_model.score(Xm_test,ym_test))\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[4, 2, 2, 2, 1, 3, 2, 3, 1, 5, 3, 1, 1, 1, 1, 1, 3, 1, 2, 1, 1, 3, 1, 2, 5, 3, 3, 1, 3, 2, 1, 5, 3]\n",
            "[2 2 2 3 3 3 5 3 2 2 3 2 5 1 5 5 3 5 4 3 2 3 2 2 5 3 3 5 3 3 5 5 3]\n",
            "0.45454545454545453\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CaEumquVtFx3",
        "colab_type": "code",
        "outputId": "14d3c9b7-bb99-416f-85e9-24274147a5cf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "y_true = ym_test\n",
        "y_pred = multi_model.predict(Xm_test)\n",
        "print(y_pred)\n",
        "print(precision_recall_fscore_support(y_true, y_pred, average='micro'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[2 2 2 3 3 3 5 3 2 2 3 2 5 1 5 5 3 5 4 3 2 3 2 2 5 3 3 5 3 3 5 5 3]\n",
            "(0.45454545454545453, 0.45454545454545453, 0.45454545454545453, None)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UMBqVqPrI96Z",
        "colab_type": "text"
      },
      "source": [
        "## Expirements using resampling and ensemble methods"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aPzccHXZKsfS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = np.array(X)\n",
        "y = np.array(y)\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(\n",
        "    X, y, test_size=0.20, random_state=42)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9o19vkwpO5ot",
        "colab_type": "text"
      },
      "source": [
        "### RUSBoostClassifier"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DDR32mKfPOiT",
        "colab_type": "code",
        "outputId": "d12f4642-cc99-41f2-b21f-bb111853fe04",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from imblearn.ensemble import RUSBoostClassifier\n",
        "from sklearn.metrics import balanced_accuracy_score\n",
        "from warnings import simplefilter\n",
        "# ignore all future warnings\n",
        "simplefilter(action='ignore', category=FutureWarning)\n",
        "rusboost = RUSBoostClassifier(n_estimators=200, algorithm='SAMME.R',\n",
        "                              random_state=0)\n",
        "rusboost.fit(X_train, y_train)  \n",
        "\n",
        "y_pred = rusboost.predict(X_test)\n",
        "print(balanced_accuracy_score(y_test, y_pred))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.2809251903343264\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-a3UUfRCPDDF",
        "colab_type": "text"
      },
      "source": [
        "### SMOTE (oversampling)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LkNu2-ua_SbG",
        "colab_type": "code",
        "outputId": "7770c565-b681-49ec-e352-af95b194bd9f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "from collections import Counter\n",
        "from sklearn.datasets import make_classification\n",
        "from imblearn.over_sampling import SMOTE\n",
        "from imblearn.under_sampling import RandomUnderSampler\n",
        "from imblearn.pipeline import Pipeline\n",
        "from matplotlib import pyplot\n",
        "from numpy import where\n",
        "counter = Counter(y_train)\n",
        "print(counter)\n",
        "over = SMOTE()\n",
        "# under = RandomUnderSampler(sampling_strategy=0.5)\n",
        "# steps = [('o', over), ('u', under)]\n",
        "# pipeline = Pipeline(steps=steps)\n",
        "# transform the dataset\n",
        "X_train, y_train = over.fit_resample(X_train, y_train)\n",
        "# summarize the new class distribution\n",
        "counter = Counter(y_train)\n",
        "print(counter)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Counter({1: 409, 2: 86, 3: 46, 5: 46, 4: 9})\n",
            "Counter({1: 409, 3: 409, 5: 409, 2: 409, 4: 409})\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XvgbE0NjP8W0",
        "colab_type": "text"
      },
      "source": [
        "### SMOTETomek"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nUIYeFM-QNz2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# from imblearn.combine import SMOTETomek # doctest: +NORMALIZE_WHITESPACE\n",
        "# print('Original dataset shape %s' % Counter(y_train))\n",
        "\n",
        "# smt = SMOTETomek(random_state=42)\n",
        "# X_train, y_train = smt.fit_resample(X_train, y_train)\n",
        "# print('Resampled dataset shape %s' % Counter(y_train))\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LWwRjSzUGvxr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(len(X_train))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JSPH5nQnQ0vB",
        "colab_type": "text"
      },
      "source": [
        "Our best result, F1 score of about 0.7"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J1aElxmSIjMs",
        "colab_type": "code",
        "outputId": "3a05b8c3-3a5c-4029-b4ac-668c8563e659",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import precision_score\n",
        "multi_model = RandomForestClassifier(n_estimators=1000)\n",
        "multi_model.fit(X_train , y_train)\n",
        "# print(y_train)\n",
        "y_pred = multi_model.predict(X_test)\n",
        "print(multi_model.score(X_test,y_test))\n",
        "# print(precision_score(y_test, y_pred, average='micro'))\n",
        "print(f1_score(y_test, y_pred, average='micro'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.697986577181208\n",
            "0.697986577181208\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a8wofih8KUbp",
        "colab_type": "code",
        "outputId": "c629ec87-c1d2-4c1f-9a38-614491a13210",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        }
      },
      "source": [
        "y_pred = binary_model.predict(X_test)\n",
        "\n",
        "print(X.shape,y.shape,y_pred.shape)\n",
        "print(binary_model.score(X_test,y_test))\n",
        "print(precision_recall_fscore_support(y_test, y_pred, average='micro'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(745, 770) (745,) (149,)\n",
            "0.6241610738255033\n",
            "(0.6241610738255033, 0.6241610738255033, 0.6241610738255033, None)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_aDI2TjFPwpi",
        "colab_type": "text"
      },
      "source": [
        "### RUSBoostClassifier after oversampling"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jp5oCAn8Me8C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from imblearn.ensemble import RUSBoostClassifier\n",
        "rusboost = RUSBoostClassifier(n_estimators=200, algorithm='SAMME.R',\n",
        "                              random_state=0)\n",
        "rusboost.fit(X_train, y_train)  \n",
        "\n",
        "y_pred = rusboost.predict(X_test)\n",
        "balanced_accuracy_score(y_test, y_pred)  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-HZSlYABP0JE",
        "colab_type": "text"
      },
      "source": [
        "### EasyEnsembleClassifier"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aUT2QXSDVrxc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from imblearn.ensemble import EasyEnsembleClassifier\n",
        "eec = EasyEnsembleClassifier(random_state=0)\n",
        "eec.fit(X_train, y_train) \n",
        "\n",
        "y_pred = eec.predict(X_test)\n",
        "balanced_accuracy_score(y_test, y_pred) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kdBHBtwGZsyd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}