Test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] File b'C:/Users/Daniel/Documents/Masterarbeit/mup_valid_articles.csv' does not exist: b'C:/Users/Daniel/Documents/Masterarbeit/mup_valid_articles.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-346fde2b3ed7>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mpy2neo\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mfuzzywuzzy\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"C:/Users/Daniel/Documents/Masterarbeit/mup_valid_articles.csv\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"utf-8\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mvalidation_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"C:/Users/Daniel/Documents/Masterarbeit/entity_dataframe_lowercase.csv\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"utf-8\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mrelation_validation_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"C:/Users/Daniel/Documents/Masterarbeit/relation_dataframe_lowercase.csv\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mencoding\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"utf-8\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    700\u001b[0m                     skip_blank_lines=skip_blank_lines)\n\u001b[1;32m    701\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 702\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    703\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    704\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    427\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    428\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 429\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    430\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    431\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    893\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 895\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    896\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1120\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'c'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1121\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'c'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1122\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1123\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1124\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'python'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   1851\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'usecols'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1852\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1853\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1854\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1855\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] File b'C:/Users/Daniel/Documents/Masterarbeit/mup_valid_articles.csv' does not exist: b'C:/Users/Daniel/Documents/Masterarbeit/mup_valid_articles.csv'"
     ]
    }
   ],
   "source": [
    "# load the necessary data\n",
    "import pandas as pd\n",
    "from py2neo import *\n",
    "from fuzzywuzzy import *\n",
    "data = pd.read_csv(\"C:/Users/Daniel/Documents/Masterarbeit/mup_valid_articles.csv\", encoding=\"utf-8\")\n",
    "validation_data = pd.read_csv(\"C:/Users/Daniel/Documents/Masterarbeit/entity_dataframe_lowercase.csv\",encoding=\"utf-8\")\n",
    "relation_validation_data = pd.read_csv(\"C:/Users/Daniel/Documents/Masterarbeit/relation_dataframe_lowercase.csv\",encoding=\"utf-8\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the necessary data\n",
    "import pandas as pd\n",
    "data = pd.read_csv(\"/home/ubuntu/ma_project/mup_valid_articles.csv\", encoding=\"utf-8\")\n",
    "validation_data = pd.read_csv(\"/home/ubuntu/ma_project/entity_dataframe_lowercase.csv\",encoding=\"utf-8\")\n",
    "relation_validation_data = pd.read_csv(\"/home/ubuntu/ma_project/relation_dataframe_lowercase.csv\",encoding=\"utf-8\")\n",
    "initial_nodes = pd.read_csv(\"/home/ubuntu/ma_project/MUP_nodes.csv\",sep = \";\", encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the functionalities\n",
    "from entitynetwork.enititynetwork_pipeline.process_article import *\n",
    "from entitynetwork.database_creation.fill_database import *\n",
    "from entitynetwork.relation_extraction.relation_extraction import *\n",
    "from allennlp.predictors.predictor import Predictor\n",
    "#from entitynetwork.validation_tool.relation_validation import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Establishe DB Connection\n",
    "global g\n",
    "g = connect_database(db_name = \"testimprovements\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "constituency_predictor_1 = Predictor.from_path(\"https://storage.googleapis.com/allennlp-public-models/elmo-constituency-parser-2020.02.10.tar.gz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Article0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ubuntu/CSS_Termpaper/EntityNetwork/entitynetwork/relation_extraction/relation_extraction.py:58: RuntimeWarning: Mean of empty slice\n",
      "  sentiment = np.nanmean(prod)\n",
      "Your label namespace was 'pos'. We recommend you use a namespace ending with 'labels' or 'tags', so we don't add UNK and PAD tokens by default to your vocabulary.  See documentation for `non_padded_namespaces` parameter in Vocabulary.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Relation Extraction: 0.28818821907043457\n",
      "constituency took:  5.527460813522339\n",
      "constituency took:  6.0933918952941895\n",
      "constituency took:  6.302850246429443\n",
      "constituency took:  7.775000810623169\n",
      "Filter Relations: 8.01040530204773\n",
      "__ArticleNode: 0.0005345344543457031\n",
      "__Publuischer: 9.131431579589844e-05\n",
      "__Author: 0.00011587142944335938\n",
      "__push article_node: 0.007681131362915039\n",
      "__push author_node: 0.005246639251708984\n",
      "__push publisher_node: 0.005173921585083008\n",
      "__commit_relationships: 0.4185183048248291\n",
      "__calculate_sub_graph_metrics: 0.16561317443847656\n",
      "Commit to db: 0.6045520305633545\n",
      "Current Article1\n",
      "Relation Extraction: 0.194932222366333\n",
      "constituency took:  2.6370859146118164\n",
      "constituency took:  2.8131701946258545\n",
      "constituency took:  3.7082788944244385\n",
      "constituency took:  4.077693462371826\n",
      "Filter Relations: 4.11555290222168\n",
      "__ArticleNode: 0.0007109642028808594\n",
      "__Publuischer: 0.00015854835510253906\n",
      "__Author: 0.00014638900756835938\n",
      "__push article_node: 0.017940759658813477\n",
      "__push author_node: 0.0068111419677734375\n",
      "__push publisher_node: 0.006310701370239258\n",
      "__commit_relationships: 0.2523667812347412\n",
      "__calculate_sub_graph_metrics: 0.1748654842376709\n",
      "Commit to db: 0.4608311653137207\n",
      "Current Article2\n",
      "Relation Extraction: 0.715644359588623\n",
      "constituency took:  21.402291774749756\n",
      "constituency took:  21.710904121398926\n",
      "constituency took:  28.05314040184021\n",
      "constituency took:  31.698363780975342\n",
      "Filter Relations: 31.98875331878662\n",
      "__ArticleNode: 0.0014774799346923828\n",
      "__Publuischer: 0.0001246929168701172\n",
      "__Author: 0.00040531158447265625\n",
      "__push article_node: 0.015728235244750977\n",
      "__push author_node: 0.0057315826416015625\n",
      "__push publisher_node: 0.011635541915893555\n",
      "__commit_relationships: 0.5631039142608643\n",
      "__calculate_sub_graph_metrics: 0.1159827709197998\n",
      "Commit to db: 0.7156202793121338\n",
      "Current Article3\n",
      "Relation Extraction: 0.3900270462036133\n",
      "constituency took:  12.967928886413574\n",
      "constituency took:  16.835938692092896\n",
      "constituency took:  18.944806814193726\n",
      "constituency took:  20.007978200912476\n",
      "Filter Relations: 20.263845443725586\n",
      "__ArticleNode: 0.0007042884826660156\n",
      "__Publuischer: 0.00019097328186035156\n",
      "__Author: 0.00011467933654785156\n",
      "__push article_node: 0.008374691009521484\n",
      "__push author_node: 0.004956483840942383\n",
      "__push publisher_node: 0.00439763069152832\n",
      "__commit_relationships: 0.3499882221221924\n",
      "__calculate_sub_graph_metrics: 0.190230131149292\n",
      "Commit to db: 0.5600135326385498\n",
      "Current Article4\n",
      "Relation Extraction: 0.31493616104125977\n",
      "constituency took:  9.433844566345215\n",
      "constituency took:  12.74056887626648\n",
      "constituency took:  12.899600505828857\n",
      "constituency took:  13.45006012916565\n",
      "Filter Relations: 13.563348293304443\n",
      "__ArticleNode: 0.0005841255187988281\n",
      "__Publuischer: 0.000141143798828125\n",
      "__Author: 0.00010609626770019531\n",
      "__push article_node: 0.010016918182373047\n",
      "__push author_node: 0.0065610408782958984\n",
      "__push publisher_node: 0.007885932922363281\n",
      "__commit_relationships: 0.4106593132019043\n",
      "__calculate_sub_graph_metrics: 0.15792584419250488\n",
      "Commit to db: 0.5950770378112793\n",
      "time for first quater articles:  83.09338140487671\n"
     ]
    }
   ],
   "source": [
    "initial_entities_dict, initial_entities_df = create_synonym_dict(g, None)\n",
    "global_start = time.time()\n",
    "for i in range(5):\n",
    "    print(\"Current Article\" + str(i))\n",
    "    #ent_val = validation_data[validation_data[\"text_id\"]==data[\"id\"][i]]\n",
    "    #rel_val = relation_validation_data[relation_validation_data[\"text_id\"]==data[\"id\"][i]]\n",
    "    article = Article(data[\"text\"][i], data[\"author_extracted\"][i], data[\"date\"][i], data[\"source\"][i], data[\"id\"][i],\n",
    "               download_ner_model=True, n_shift=0,graph=g,first_round=False ,commit=True,\n",
    "               initial_entities = initial_entities_df,initial_syn_dict = initial_entities_dict,\n",
    "                validate=False, filter_by_constituency=True,use_coref = False,\n",
    "                     path_constituency_parser=\"/home/ubuntu/ma_project/elmo-constituency-parser-2020.02.10.tar.gz\",\n",
    "                     path_coref_parser=\"/home/ubuntu/ma_project/coref-spanbert-large-2020.02.27.tar.gz\",\n",
    "                     constituency_predictor = constituency_predictor_1)\n",
    "    article.process_articles()\n",
    "global_stop = time.time()\n",
    "print(\"time for first quater articles: \", global_stop-global_start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertModel were not initialized from the model checkpoint at SpanBERT/spanbert-large-cased and are newly initialized: ['bert.pooler.dense.weight', 'bert.pooler.dense.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
      "Did not use initialization regex that was passed: _context_layer._module.weight_ih.*\n",
      "Did not use initialization regex that was passed: _context_layer._module.weight_hh.*\n"
     ]
    }
   ],
   "source": [
    "from allennlp.predictors.predictor import Predictor\n",
    "#constituency_predictor_1 = Predictor.from_path(article.path_constituency_parser)\n",
    "#constituency_predictor_2 = Predictor.from_path(article.path_constituency_parser)\n",
    "#constituency_predictor_3 = Predictor.from_path(article.path_constituency_parser)\n",
    "#constituency_predictor_4 = Predictor.from_path(article.path_constituency_parser)\n",
    "#constituency_predictor_5 = Predictor.from_path(article.path_constituency_parser)\n",
    "#constituency_predictor_6 = Predictor.from_path(article.path_constituency_parser)\n",
    "coref_predictor_1 = Predictor.from_path(\"/home/ubuntu/ma_project/coref-spanbert-large-2020.02.27.tar.gz\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def constituency_filter(article, constituency_predictor, start, stop):\n",
    "    #article.relation_dataset.df[\"filter\"] = 0\n",
    "    temp_time = 0\n",
    "    for k in range(start,stop):\n",
    "        db_ids = article.occurrence_matrix[:,-1][article.occurrence_matrix[:,k]==1]\n",
    "        entity_names = article.initial_entities_df[\"name\"].loc[article.initial_entities_df[\"id\"].isin(db_ids)]\n",
    "        search_terms = article.initial_entities_df[\"synonyms\"].loc[article.initial_entities_df[\"id\"].isin(db_ids)]\n",
    "        index_terms = search_terms.index\n",
    "        temp_time_start = time.time()\n",
    "        constituency = constituency_predictor.predict(article.sentences[k])\n",
    "        temp_time_stop = time.time()\n",
    "        temp_time += temp_time_stop-temp_time_start\n",
    "        constituency = constituency.get(\"hierplane_tree\").get(\"root\")\n",
    "        path_dic = dict()\n",
    "        n = 0\n",
    "        for i in range(len(index_terms)):\n",
    "            synonym_array = get_synonyms(search_terms[index_terms[i]])\n",
    "            lst = []\n",
    "            for j in range(len(synonym_array)):\n",
    "                path = ()\n",
    "                for item in get_constituency_path(constituency, synonym_array[j], (constituency.get(\"attributes\"),)):\n",
    "                    path = item\n",
    "                if len(path) > 0:\n",
    "                    lst.append(path)\n",
    "            path_dic[db_ids[n]] = lst\n",
    "            n += 1\n",
    "        for i in range(len(db_ids)):\n",
    "            for i_values in path_dic.get(db_ids[i]):\n",
    "                for j in range(i + 1, len(db_ids)):\n",
    "                    for j_values in path_dic.get(db_ids[j]):\n",
    "                        valid, split_point = is_valid(i_values, j_values)\n",
    "                        article.relation_dataset.df.loc[\n",
    "                            ((article.relation_dataset.df[\"db_id_1\"] == db_ids[i]) &\n",
    "                             (article.relation_dataset.df[\"db_id_2\"] == db_ids[j])),\n",
    "                            \"filter\"] += valid\n",
    "    print(\"constituency took: \", temp_time)\n",
    "    \n",
    "\n",
    "def is_valid(path_1, path_2):\n",
    "    valid = 0\n",
    "    split_point = ()\n",
    "    for idx, (i, j) in enumerate(zip(path_1, path_2)):\n",
    "        if i == j:\n",
    "            pass\n",
    "        else:\n",
    "            split_point = (i, j)\n",
    "            if [\"NP\"] in split_point:\n",
    "                if any(tag in split_point for tag in [[\"VP\"], [\"PP\"]]):\n",
    "                    valid = 1\n",
    "                    return valid, split_point\n",
    "                elif [\"NP\"] == split_point[0] and [\"NP\"] == split_point[1]:\n",
    "                    valid = 1\n",
    "                    return valid, split_point\n",
    "    return valid, split_point\n",
    "\n",
    "\n",
    "def get_constituency_path(dct, value, path=()):\n",
    "    if value.lower() in dct.get(\"word\").lower():\n",
    "        yield path\n",
    "    for key, lst in dct.items():\n",
    "        if isinstance(lst, list):\n",
    "            for item in lst:\n",
    "                if isinstance(item, dict):\n",
    "                    for pth in get_constituency_path(item, value, path + (item.get(\"attributes\"),)):\n",
    "                        yield pth   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "constituency took:  8.5869882106781\n",
      "constituency took:  9.235594511032104\n",
      "constituency took:  9.76126217842102\n",
      "constituency took:  9.937808513641357\n",
      "constituency took:  10.325183153152466\n",
      "constituency took:  11.310307264328003\n"
     ]
    }
   ],
   "source": [
    "import threading\n",
    "number = len(article.sentences)\n",
    "\n",
    "\n",
    "t_1 = threading.Thread(target = constituency_filter, args=(article, constituency_predictor_1, 0, int(number/6*1) ,))\n",
    "t_2 = threading.Thread(target = constituency_filter, args=(article, constituency_predictor_1, int(number/6*1), int(number/6*2) ,))\n",
    "t_3 = threading.Thread(target = constituency_filter, args=(article, constituency_predictor_1, int(number/6*2), int(number/6*3) ,))\n",
    "t_4 = threading.Thread(target = constituency_filter, args=(article, constituency_predictor_1, int(number/6*3), int(number/6*4) ,))\n",
    "t_5 = threading.Thread(target = constituency_filter, args=(article, constituency_predictor_1, int(number/6*4), int(number/6*5) ,))\n",
    "t_6 = threading.Thread(target = constituency_filter, args=(article, constituency_predictor_1, int(number/6*5), number ,))\n",
    "\n",
    "t_1.start()\n",
    "t_2.start()\n",
    "t_3.start()\n",
    "t_4.start()\n",
    "t_5.start()\n",
    "t_6.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-6:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.6/threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.6/threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "TypeError: constituency_filter() takes 1 positional argument but 4 were given\n",
      "\n",
      "Exception in thread Thread-5:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.6/threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.6/threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "TypeError: constituency_filter() takes 1 positional argument but 4 were given\n",
      "\n",
      "Exception in thread Thread-4:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.6/threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.6/threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "TypeError: constituency_filter() takes 1 positional argument but 4 were given\n",
      "\n",
      "Exception in thread Thread-7:\n",
      "Traceback (most recent call last):\n",
      "  File \"/usr/lib/python3.6/threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"/usr/lib/python3.6/threading.py\", line 864, in run\n",
      "    self._target(*self._args, **self._kwargs)\n",
      "TypeError: constituency_filter() takes 1 positional argument but 4 were given\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import threading\n",
    "number = len(article.sentences)\n",
    "\n",
    "\n",
    "t_1 = threading.Thread(target = constituency_filter, args=(article, constituency_predictor_1, 0, int(int(number/2)/2) ,))\n",
    "t_2 = threading.Thread(target = constituency_filter, args=(article, constituency_predictor_1, int(int(number/2)/2), int(number/2) ,))\n",
    "t_3 = threading.Thread(target = constituency_filter, args=(article, constituency_predictor_1, int(number/2), (int(number/2)+int(int(number/2)/2)) ,))\n",
    "t_4 = threading.Thread(target = constituency_filter, args=(article, constituency_predictor_1, (int(number/2)+int(int(number/2)/2)), number ,))\n",
    "\n",
    "t_1.start()\n",
    "t_2.start()\n",
    "t_3.start()\n",
    "t_4.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "constituency took:  10.047947883605957\n",
      "constituency took:  11.362336158752441\n",
      "constituency took:  11.523970127105713\n"
     ]
    }
   ],
   "source": [
    "number = len(article.sentences)\n",
    "\n",
    "\n",
    "t_1 = threading.Thread(target = constituency_filter, args=(article, constituency_predictor_1, 0, (int(number/3)+1) ,))\n",
    "t_2 = threading.Thread(target = constituency_filter, args=(article, constituency_predictor_1, (int(number/3)+1), (int(int(number/3)*2)+2) ,))\n",
    "t_3 = threading.Thread(target = constituency_filter, args=(article, constituency_predictor_1, (int(int(number/3)*2)+2), number ,))\n",
    "\n",
    "t_1.start()\n",
    "t_2.start()\n",
    "t_3.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time for corefs:  23.431174516677856\n",
      "time for corefs:  24.586010456085205\n"
     ]
    }
   ],
   "source": [
    "#number = len(article.sentences)\n",
    "import threading\n",
    "\n",
    "def use_coref(data, coref_predictor, start, stop):\n",
    "    time_start = time.time()\n",
    "    for i in range(start, stop):\n",
    "        coref_predictor.predict(data[\"text\"][i])\n",
    "    time_stop = time.time()\n",
    "    print(\"time for corefs: \", time_stop-time_start)\n",
    "\n",
    "\n",
    "t_1 = threading.Thread(target = use_coref, args=(data, coref_predictor_1, 0, 3 ,))\n",
    "t_2 = threading.Thread(target = use_coref, args=(data, coref_predictor_1, 3, 5 ,))\n",
    "\n",
    "t_1.start()\n",
    "t_2.start()\n",
    "\n",
    "t_1.join()\n",
    "t_2.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'constituency_predictor' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-1557f18e0b29>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mconstituency_predictor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Donald Trump is a bad US president, say the notorius RBG.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'constituency_predictor' is not defined"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "Can't pickle local object 'DirectConnectionPool.acquire.<locals>.<lambda>'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-32-ab2572908955>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mmanager\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mManager\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mns\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmanager\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNamespace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marticle\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marticle\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mnumber\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marticle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msentences\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/multiprocessing/managers.py\u001b[0m in \u001b[0;36m__setattr__\u001b[0;34m(self, key, value)\u001b[0m\n\u001b[1;32m   1063\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__setattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1064\u001b[0m         \u001b[0mcallmethod\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'_callmethod'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1065\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mcallmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'__setattr__'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1066\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__delattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1067\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'_'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/multiprocessing/managers.py\u001b[0m in \u001b[0;36m_callmethod\u001b[0;34m(self, methodname, args, kwds)\u001b[0m\n\u001b[1;32m    754\u001b[0m             \u001b[0mconn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_tls\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnection\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    755\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 756\u001b[0;31m         \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethodname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    757\u001b[0m         \u001b[0mkind\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    758\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/multiprocessing/connection.py\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m    204\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_closed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_check_writable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 206\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_send_bytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_ForkingPickler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    207\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrecv_bytes\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxlength\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/multiprocessing/reduction.py\u001b[0m in \u001b[0;36mdumps\u001b[0;34m(cls, obj, protocol)\u001b[0m\n\u001b[1;32m     49\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mdumps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0mbuf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mBytesIO\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m         \u001b[0mcls\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mbuf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetbuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: Can't pickle local object 'DirectConnectionPool.acquire.<locals>.<lambda>'"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37\n",
      "0 12\n",
      "12 24\n",
      "24 37\n"
     ]
    }
   ],
   "source": [
    "number = len(article.sentences)\n",
    "print(number)\n",
    "for i in range(3):\n",
    "    print(int((number/3)*i),int((number/3)*i) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>db_id_1</th>\n",
       "      <th>db_id_2</th>\n",
       "      <th>count</th>\n",
       "      <th>weight</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>filter</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7.0</td>\n",
       "      <td>88.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.202300</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.202300</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.0</td>\n",
       "      <td>252.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.202300</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7.0</td>\n",
       "      <td>254.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.202300</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>31.0</td>\n",
       "      <td>163.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.081081</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>88.0</td>\n",
       "      <td>156.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.945946</td>\n",
       "      <td>-0.296000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>88.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.202300</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>88.0</td>\n",
       "      <td>252.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.067433</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>88.0</td>\n",
       "      <td>253.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.540541</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>88.0</td>\n",
       "      <td>254.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.101150</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>88.0</td>\n",
       "      <td>316.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.702703</td>\n",
       "      <td>0.440400</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>88.0</td>\n",
       "      <td>317.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.702703</td>\n",
       "      <td>0.440400</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>88.0</td>\n",
       "      <td>318.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.702703</td>\n",
       "      <td>0.440400</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>156.0</td>\n",
       "      <td>253.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.675676</td>\n",
       "      <td>0.081400</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>156.0</td>\n",
       "      <td>254.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.243243</td>\n",
       "      <td>0.526700</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>156.0</td>\n",
       "      <td>319.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.675676</td>\n",
       "      <td>0.229400</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>156.0</td>\n",
       "      <td>322.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.243243</td>\n",
       "      <td>0.526700</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>156.0</td>\n",
       "      <td>323.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.243243</td>\n",
       "      <td>0.526700</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>184.0</td>\n",
       "      <td>252.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.202300</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>184.0</td>\n",
       "      <td>253.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.270270</td>\n",
       "      <td>0.851900</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>184.0</td>\n",
       "      <td>254.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.202300</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>184.0</td>\n",
       "      <td>313.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.273200</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>184.0</td>\n",
       "      <td>314.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.972973</td>\n",
       "      <td>0.493900</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>184.0</td>\n",
       "      <td>320.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.594595</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>252.0</td>\n",
       "      <td>253.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.540541</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>252.0</td>\n",
       "      <td>254.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.101150</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>253.0</td>\n",
       "      <td>319.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.675676</td>\n",
       "      <td>0.458800</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>254.0</td>\n",
       "      <td>313.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.459459</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>254.0</td>\n",
       "      <td>322.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.243243</td>\n",
       "      <td>0.526700</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>254.0</td>\n",
       "      <td>323.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.243243</td>\n",
       "      <td>0.526700</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>254.0</td>\n",
       "      <td>324.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.216216</td>\n",
       "      <td>0.296000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>316.0</td>\n",
       "      <td>317.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.702703</td>\n",
       "      <td>0.440400</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>316.0</td>\n",
       "      <td>318.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.702703</td>\n",
       "      <td>0.440400</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>317.0</td>\n",
       "      <td>318.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.702703</td>\n",
       "      <td>0.440400</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>322.0</td>\n",
       "      <td>323.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.243243</td>\n",
       "      <td>0.526700</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    db_id_1  db_id_2  count    weight  sentiment  filter\n",
       "0       7.0     88.0    1.0  0.918919   0.202300     0.0\n",
       "1       7.0    184.0    1.0  0.918919   0.202300     2.0\n",
       "2       7.0    252.0    1.0  0.918919   0.202300     0.0\n",
       "3       7.0    254.0    1.0  0.918919   0.202300     1.0\n",
       "4      31.0    163.0    1.0  0.081081   0.000000     1.0\n",
       "5      88.0    156.0    1.0  0.945946  -0.296000     0.0\n",
       "6      88.0    184.0    1.0  0.918919   0.202300     0.0\n",
       "7      88.0    252.0    3.0  0.918919   0.067433     0.0\n",
       "8      88.0    253.0    1.0  0.540541   0.000000     0.0\n",
       "9      88.0    254.0    2.0  0.918919   0.101150     3.0\n",
       "10     88.0    316.0    1.0  0.702703   0.440400     0.0\n",
       "11     88.0    317.0    1.0  0.702703   0.440400     0.0\n",
       "12     88.0    318.0    1.0  0.702703   0.440400     0.0\n",
       "13    156.0    253.0    2.0  0.675676   0.081400     1.0\n",
       "14    156.0    254.0    1.0  0.243243   0.526700     0.0\n",
       "15    156.0    319.0    2.0  0.675676   0.229400     0.0\n",
       "16    156.0    322.0    1.0  0.243243   0.526700     1.0\n",
       "17    156.0    323.0    1.0  0.243243   0.526700     1.0\n",
       "18    184.0    252.0    1.0  0.918919   0.202300     0.0\n",
       "19    184.0    253.0    1.0  0.270270   0.851900     0.0\n",
       "20    184.0    254.0    1.0  0.918919   0.202300     1.0\n",
       "21    184.0    313.0    1.0  1.000000   0.273200     0.0\n",
       "22    184.0    314.0    1.0  0.972973   0.493900     1.0\n",
       "23    184.0    320.0    1.0  0.594595   0.000000     0.0\n",
       "24    252.0    253.0    1.0  0.540541   0.000000     0.0\n",
       "25    252.0    254.0    2.0  0.918919   0.101150     1.0\n",
       "26    253.0    319.0    1.0  0.675676   0.458800     2.0\n",
       "27    254.0    313.0    1.0  0.459459   0.000000     2.0\n",
       "28    254.0    322.0    1.0  0.243243   0.526700     0.0\n",
       "29    254.0    323.0    1.0  0.243243   0.526700     1.0\n",
       "30    254.0    324.0    1.0  0.216216   0.296000     0.0\n",
       "31    316.0    317.0    1.0  0.702703   0.440400     0.0\n",
       "32    316.0    318.0    1.0  0.702703   0.440400     0.0\n",
       "33    317.0    318.0    1.0  0.702703   0.440400     0.0\n",
       "34    322.0    323.0    1.0  0.243243   0.526700     1.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "article.relation_dataset.df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>db_id_1</th>\n",
       "      <th>db_id_2</th>\n",
       "      <th>count</th>\n",
       "      <th>weight</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>filter</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7.0</td>\n",
       "      <td>88.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.202300</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.202300</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>7.0</td>\n",
       "      <td>252.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.202300</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>7.0</td>\n",
       "      <td>254.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.202300</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>31.0</td>\n",
       "      <td>163.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.081081</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>88.0</td>\n",
       "      <td>156.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.945946</td>\n",
       "      <td>-0.296000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>88.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.202300</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>88.0</td>\n",
       "      <td>252.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.067433</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>88.0</td>\n",
       "      <td>253.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.540541</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>88.0</td>\n",
       "      <td>254.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.101150</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>88.0</td>\n",
       "      <td>316.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.702703</td>\n",
       "      <td>0.440400</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>88.0</td>\n",
       "      <td>317.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.702703</td>\n",
       "      <td>0.440400</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>88.0</td>\n",
       "      <td>318.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.702703</td>\n",
       "      <td>0.440400</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>156.0</td>\n",
       "      <td>253.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.675676</td>\n",
       "      <td>0.081400</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>156.0</td>\n",
       "      <td>254.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.243243</td>\n",
       "      <td>0.526700</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>156.0</td>\n",
       "      <td>319.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.675676</td>\n",
       "      <td>0.229400</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>156.0</td>\n",
       "      <td>322.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.243243</td>\n",
       "      <td>0.526700</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>156.0</td>\n",
       "      <td>323.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.243243</td>\n",
       "      <td>0.526700</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>184.0</td>\n",
       "      <td>252.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.202300</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>184.0</td>\n",
       "      <td>253.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.270270</td>\n",
       "      <td>0.851900</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>184.0</td>\n",
       "      <td>254.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.202300</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>184.0</td>\n",
       "      <td>313.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.273200</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>184.0</td>\n",
       "      <td>314.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.972973</td>\n",
       "      <td>0.493900</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>184.0</td>\n",
       "      <td>320.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.594595</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>252.0</td>\n",
       "      <td>253.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.540541</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>252.0</td>\n",
       "      <td>254.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.918919</td>\n",
       "      <td>0.101150</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>253.0</td>\n",
       "      <td>319.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.675676</td>\n",
       "      <td>0.458800</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>254.0</td>\n",
       "      <td>313.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.459459</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>254.0</td>\n",
       "      <td>322.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.243243</td>\n",
       "      <td>0.526700</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>254.0</td>\n",
       "      <td>323.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.243243</td>\n",
       "      <td>0.526700</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>254.0</td>\n",
       "      <td>324.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.216216</td>\n",
       "      <td>0.296000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>316.0</td>\n",
       "      <td>317.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.702703</td>\n",
       "      <td>0.440400</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>316.0</td>\n",
       "      <td>318.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.702703</td>\n",
       "      <td>0.440400</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>317.0</td>\n",
       "      <td>318.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.702703</td>\n",
       "      <td>0.440400</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>322.0</td>\n",
       "      <td>323.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.243243</td>\n",
       "      <td>0.526700</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    db_id_1  db_id_2  count    weight  sentiment  filter\n",
       "0       7.0     88.0    1.0  0.918919   0.202300       0\n",
       "1       7.0    184.0    1.0  0.918919   0.202300       2\n",
       "2       7.0    252.0    1.0  0.918919   0.202300       0\n",
       "3       7.0    254.0    1.0  0.918919   0.202300       1\n",
       "4      31.0    163.0    1.0  0.081081   0.000000       1\n",
       "5      88.0    156.0    1.0  0.945946  -0.296000       0\n",
       "6      88.0    184.0    1.0  0.918919   0.202300       0\n",
       "7      88.0    252.0    3.0  0.918919   0.067433       0\n",
       "8      88.0    253.0    1.0  0.540541   0.000000       0\n",
       "9      88.0    254.0    2.0  0.918919   0.101150       3\n",
       "10     88.0    316.0    1.0  0.702703   0.440400       0\n",
       "11     88.0    317.0    1.0  0.702703   0.440400       0\n",
       "12     88.0    318.0    1.0  0.702703   0.440400       0\n",
       "13    156.0    253.0    2.0  0.675676   0.081400       1\n",
       "14    156.0    254.0    1.0  0.243243   0.526700       0\n",
       "15    156.0    319.0    2.0  0.675676   0.229400       0\n",
       "16    156.0    322.0    1.0  0.243243   0.526700       1\n",
       "17    156.0    323.0    1.0  0.243243   0.526700       1\n",
       "18    184.0    252.0    1.0  0.918919   0.202300       0\n",
       "19    184.0    253.0    1.0  0.270270   0.851900       0\n",
       "20    184.0    254.0    1.0  0.918919   0.202300       1\n",
       "21    184.0    313.0    1.0  1.000000   0.273200       0\n",
       "22    184.0    314.0    1.0  0.972973   0.493900       1\n",
       "23    184.0    320.0    1.0  0.594595   0.000000       0\n",
       "24    252.0    253.0    1.0  0.540541   0.000000       0\n",
       "25    252.0    254.0    2.0  0.918919   0.101150       1\n",
       "26    253.0    319.0    1.0  0.675676   0.458800       2\n",
       "27    254.0    313.0    1.0  0.459459   0.000000       2\n",
       "28    254.0    322.0    1.0  0.243243   0.526700       0\n",
       "29    254.0    323.0    1.0  0.243243   0.526700       1\n",
       "30    254.0    324.0    1.0  0.216216   0.296000       0\n",
       "31    316.0    317.0    1.0  0.702703   0.440400       0\n",
       "32    316.0    318.0    1.0  0.702703   0.440400       0\n",
       "33    317.0    318.0    1.0  0.702703   0.440400       0\n",
       "34    322.0    323.0    1.0  0.243243   0.526700       1"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "article.relation_dataset.df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# load the functionalities\n",
    "from entitynetwork.enititynetwork_pipeline.process_article import *\n",
    "from entitynetwork.database_creation.fill_database import *\n",
    "from entitynetwork.relation_extraction.relation_extraction import *\n",
    "#from entitynetwork.validation_tool.relation_validation import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Establishe DB Connection\n",
    "g = connect_database()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Tag selection\n",
    "\n",
    "all_tag = [\"B-PERSON\", \"I-PERSON\", \"B-NORP\", \"I-NORP\",\"B-FAC\", \"I-FAC\", \"B-ORG\", \"I-ORG\", \"B-GPE\", \"I-GPE\",\n",
    "          \"B-LOC\", \"I-LOC\", \"B-PRODUCT\", \"I-PRODUCT\", \"B-EVENT\", \"I-EVENT\", \"B-WORK_OF_ART\", \"I-WORK_OF_ART\",\n",
    "          \"B-LAW\", \"I-LAW\", \"B-LANGUAGE\", \"I-LANGUAGE\", \"B-DATE\", \"I-DATE\", \"B-TIME\", \"I-TIME\", \"B-PERCENT\",\n",
    "           \"I-PERCENT\", \"B-MONEY\", \"I-MONEY\", \"B-QUANTITY\", \"I-QUANTITY\", \"B-ORDINAL\", \"I-ORDINAL\", \"B-CARDINAL\",\n",
    "          \"I_CARDINAL\"]\n",
    "\n",
    "selected_tag = [\"B-PERSON\", \"I-PERSON\", \"B-FAC\", \"I-FAC\", \"B-ORG\", \"I-ORG\", \"B-GPE\", \"I-GPE\",\n",
    "                \"B-LOC\", \"I-LOC\", \"B-WORK_OF_ART\", \"I-WORK_OF_ART\", \"B-LAW\", \"I-LAW\"]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Article0\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "object of type 'NoneType' has no len()",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-5-6703c93ef9ad>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      7\u001b[0m                \u001b[0mdownload_model\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mselected_tags\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mselected_tag\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_shift\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mgraph\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mg\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m                \u001b[0mvalidate\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mentity_validation_data\u001b[0m\u001b[1;33m=\u001b[0m \u001b[0ment_val\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m                relation_validation_data = rel_val)\n\u001b[0m\u001b[0;32m     10\u001b[0m     \u001b[0marticle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprocess_articles\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Documents\\Masterarbeit\\CSS_Termpaper\\EntityNetwork\\entitynetwork\\enititynetwork_pipeline\\process_article.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, text, author, date, publication, article_id, model_version, download_model, selected_tags, n_shift, graph, split_tag, validate, entity_validation_data, relation_validation_data, initial_syn_sets)\u001b[0m\n\u001b[0;32m     55\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minitial_syn_sets\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0minitial_syn_sets\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     56\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minitial_entities_dict\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 57\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcreate_entities_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     58\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     59\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mprocess_articles\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Documents\\Masterarbeit\\CSS_Termpaper\\EntityNetwork\\entitynetwork\\enititynetwork_pipeline\\process_article.py\u001b[0m in \u001b[0;36mcreate_entities_dict\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     70\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     71\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mcreate_entities_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 72\u001b[1;33m         \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minitial_syn_sets\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     73\u001b[0m             self.initial_entities_dict.update(\n\u001b[0;32m     74\u001b[0m                 self.initial_entities_dict.fromkeys(get_synonyms(self.initial_syn_sets[\"synonyms\"][i]), i))\n",
      "\u001b[1;31mTypeError\u001b[0m: object of type 'NoneType' has no len()"
     ]
    }
   ],
   "source": [
    "# Start the validation and store the data\n",
    "for i in range(5):\n",
    "    print(\"Current Article\" + str(i))\n",
    "    ent_val = validation_data[validation_data[\"text_id\"]==data[\"id\"][i]]\n",
    "    rel_val = relation_validation_data[relation_validation_data[\"text_id\"]==data[\"id\"][i]]\n",
    "    article = Article(data[\"text\"][i], data[\"author_extracted\"][i], data[\"date\"][i], data[\"source\"][i], data[\"id\"][i],\n",
    "               download_model=False, selected_tags=selected_tag, n_shift=0,graph=g,\n",
    "               validate=True, entity_validation_data= ent_val,\n",
    "               relation_validation_data = rel_val)\n",
    "    article.process_articles()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def get_synonyms(synonyms):\n",
    "    synonym_array = []\n",
    "    for synonym in synonyms.split(\"_\"):\n",
    "        if synonym != \"\":\n",
    "            synonym_array.append(synonym)\n",
    "    return synonym_array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(_1079:Entity {group: 'labour', name: 'iain gray', side: 0, synonyms: '_iain gray__gray_', tag: 'NA'})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g.evaluate('MATCH (n:Entity) WHERE n.synonyms CONTAINS $name RETURN n',\n",
    "                            parameters={'name': \"gray\"})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(_945:Entity {group: 'political group', name: 'scottish government', side: 1, synonyms: '_scottish government__snp government__nationalist government_', tag: 'NA'})"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g.evaluate('MATCH (n) RETURN n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "matcher = NodeMatcher(g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(_13:Entity {name: 'iain gray', synonyms: '_iain gray__gray_', tag: 'PERSON'}),\n",
       " (_32:Entity {name: \"iain gray's\", synonyms: \"_iain gray's__gray_\", tag: 'PERSON'})]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(matcher.match(\"Entity\").where(\"_.synonyms CONTAINS 'gray'\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(_13:Entity {name: 'iain gray', synonyms: '_iain gray__gray_', tag: 'PERSON'})"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matcher.get(13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(_3:Entity {name: 'british medical association', synonyms: '_british medical association_', tag: 'ORG'}),\n",
       " (_4:Entity {name: 'nhs', synonyms: '_nhs__nhs information centre_', tag: 'ORG'}),\n",
       " (_5:Entity {name: 'bma', synonyms: '_bma_', tag: 'ORG'}),\n",
       " (_6:Entity {name: 'alan mcdevitt', synonyms: '_alan mcdevitt_', tag: 'PERSON'}),\n",
       " (_7:Entity {name: \"bma's scottish general practitioners' committee\", synonyms: \"_bma's scottish general practitioners' committee_\", tag: 'ORG'}),\n",
       " (_8:Entity {name: \"scottish national party's\", synonyms: \"_scottish national party's_\", tag: 'ORG'}),\n",
       " (_9:Entity {name: 'parliament', synonyms: '_parliament_', tag: 'ORG'}),\n",
       " (_12:Entity {name: 'labour', synonyms: '_labour_', tag: 'ORG'}),\n",
       " (_13:Entity {name: 'iain gray', synonyms: '_iain gray__gray_', tag: 'PERSON'}),\n",
       " (_14:Entity {name: 'snp', synonyms: '_snp_', tag: 'ORG'}),\n",
       " (_15:Entity {name: 'alex salmond', synonyms: '_alex salmond__salmond_', tag: 'PERSON'}),\n",
       " (_16:Entity {name: 'bill', synonyms: '_bill_', tag: 'LAW'}),\n",
       " (_17:Entity {name: 'lib dem', synonyms: '_lib dem_', tag: 'NORP'}),\n",
       " (_18:Entity {name: 'tavish scott', synonyms: '_tavish scott_', tag: 'PERSON'}),\n",
       " (_19:Entity {name: 'annabel goldie', synonyms: '_annabel goldie__goldie_', tag: 'PERSON'}),\n",
       " (_23:Entity {name: 'msps', synonyms: '_msps_', tag: 'ORG'}),\n",
       " (_24:Entity {name: 'holyrood', synonyms: '_holyrood_', tag: 'ORG'}),\n",
       " (_25:Entity {name: 'scottish', synonyms: '_scottish_', tag: 'NORP'}),\n",
       " (_26:Entity {name: 'uk government', synonyms: '_uk government_', tag: 'ORG'}),\n",
       " (_27:Entity {name: 'scots', synonyms: '_scots__scots tory_', tag: 'NORP'}),\n",
       " (_28:Entity {name: 'eu', synonyms: '_eu_', tag: 'ORG'}),\n",
       " (_29:Entity {name: 'treasury', synonyms: '_treasury_', tag: 'ORG'}),\n",
       " (_30:Entity {name: 'scotch', synonyms: '_scotch_', tag: 'ORG'}),\n",
       " (_31:Entity {name: \"salmond's\", synonyms: \"_salmond's__salmond_\", tag: 'PERSON'}),\n",
       " (_32:Entity {name: \"iain gray's\", synonyms: \"_iain gray's__gray_\", tag: 'PERSON'}),\n",
       " (_33:Entity {name: 'earth', synonyms: '_earth_', tag: 'LOC'}),\n",
       " (_34:Entity {name: 'scotland bill', synonyms: '_scotland bill_', tag: 'LAW'}),\n",
       " (_35:Entity {name: 'michael moore', synonyms: '_michael moore_', tag: 'PERSON'}),\n",
       " (_39:Entity {name: 'licensing act', synonyms: '_licensing act_', tag: 'LAW'}),\n",
       " (_40:Entity {name: 'alcohol concern', synonyms: '_alcohol concern_', tag: 'ORG'}),\n",
       " (_41:Entity {name: 'don shenker', synonyms: '_don shenker_', tag: 'PERSON'}),\n",
       " (_42:Entity {name: 'department of health', synonyms: '_department of health_', tag: 'ORG'}),\n",
       " (_43:Entity {name: 'ian gilmore', synonyms: '_ian gilmore_', tag: 'PERSON'}),\n",
       " (_44:Entity {name: 'uk alcohol health alliance', synonyms: '_uk alcohol health alliance_', tag: 'ORG'}),\n",
       " (_45:Entity {name: 'bbc', synonyms: '_bbc_', tag: 'ORG'}),\n",
       " (_46:Entity {name: 'anne milton', synonyms: '_anne milton_', tag: 'PERSON'}),\n",
       " (_47:Entity {name: 'tim straughan', synonyms: '_tim straughan_', tag: 'PERSON'}),\n",
       " (_51:Entity {name: 'a & e', synonyms: '_a & e_', tag: 'ORG'}),\n",
       " (_52:Entity {name: 'nhs information centre', synonyms: '_nhs information centre__nhs_', tag: 'ORG'}),\n",
       " (_53:Entity {name: 'government', synonyms: '_government_', tag: 'ORG'}),\n",
       " (_54:Entity {name: 'north - west', synonyms: '_north - west__north - east_', tag: 'LOC'}),\n",
       " (_55:Entity {name: 'chris sorek', synonyms: '_chris sorek_', tag: 'PERSON'}),\n",
       " (_56:Entity {name: 'drinka - ware', synonyms: '_drinka - ware_', tag: 'ORG'}),\n",
       " (_57:Entity {name: 'david poley', synonyms: '_david poley_', tag: 'PERSON'}),\n",
       " (_58:Entity {name: 'portman group', synonyms: '_portman group_', tag: 'ORG'}),\n",
       " (_59:Entity {name: 'diane abbott', synonyms: '_diane abbott_', tag: 'PERSON'}),\n",
       " (_60:Entity {name: 'simon litherland', synonyms: '_simon litherland_', tag: 'PERSON'}),\n",
       " (_61:Entity {name: 'diageo', synonyms: '_diageo_', tag: 'ORG'}),\n",
       " (_62:Entity {name: 'smirnoff', synonyms: '_smirnoff_', tag: 'ORG'}),\n",
       " (_63:Entity {name: 'guinness', synonyms: '_guinness_', tag: 'ORG'})]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(matcher.match(\"Entity\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "syns = g.run('MATCH (n:Entity) WHERE n.synonyms CONTAINS $name RETURN n.synonyms',\n",
    "                            parameters={'name': \"gray\"}).to_data_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "1",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-8-8d4ace2036da>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;31m#syns[\"n\"][1]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mprocess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfuzz\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpartial_ratio\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msyns\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"n.synonyms\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msyns\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"n.synonyms\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\series.py\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   1069\u001b[0m         \u001b[0mkey\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcom\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply_if_callable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1070\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1071\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_value\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1072\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1073\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mis_scalar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py\u001b[0m in \u001b[0;36mget_value\u001b[1;34m(self, series, key)\u001b[0m\n\u001b[0;32m   4728\u001b[0m         \u001b[0mk\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_convert_scalar_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkind\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"getitem\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4729\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 4730\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_value\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtz\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mseries\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"tz\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   4731\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4732\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mholds_integer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_boolean\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_value\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_value\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.Int64HashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.Int64HashTable.get_item\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 1"
     ]
    }
   ],
   "source": [
    "#syns[\"n\"][1]\n",
    "process.fuzz.partial_ratio(syns[\"n.synonyms\"][1],syns[\"n.synonyms\"][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "l = [\"david\", \"david cameron\"]\n",
    "g.run('MATCH (n:Entity) WHERE n.synonyms CONTAINS $name RETURN n',\n",
    "                            parameters={'name': l}).to_data_frame()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "syns[\"ID(n)\"][0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                              None\n",
       "1                              None\n",
       "2                              None\n",
       "3     _british medical association_\n",
       "4     _nhs__nhs information centre_\n",
       "                  ...              \n",
       "59                   _diane abbott_\n",
       "60               _simon litherland_\n",
       "61                         _diageo_\n",
       "62                       _smirnoff_\n",
       "63                       _guinness_\n",
       "Length: 64, dtype: object"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#g.run('MATCH (n) WHERE ID(n) = $id SET n.name = \"iain gray\" RETURN n', parameters={'id': int(syns[\"ID(n)\"][0])}).to_table()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "#synset_1 = get_synonym_array(syns[\"n.synonyms\"][0])\n",
    "#synset_2 = get_synonym_array(syns[\"n.synonyms\"][1])\n",
    "synset_1 = [\"uk government\", \"snp government\"]\n",
    "synset_2 = [\"scottish government\",\"snp government\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "78\n",
      "100\n",
      "69\n",
      "73\n",
      "67\n",
      "78\n",
      "75\n"
     ]
    }
   ],
   "source": [
    "print(process.fuzz.partial_ratio(synset_1,synset_2))\n",
    "print(process.fuzz.partial_token_set_ratio(synset_1,synset_2))\n",
    "print(process.fuzz.partial_token_sort_ratio(synset_1,synset_2))\n",
    "print(process.fuzz.token_set_ratio(synset_1,synset_2))\n",
    "print(process.fuzz.token_sort_ratio(synset_1,synset_2))\n",
    "print(process.fuzz.ratio(synset_1,synset_2))\n",
    "print(process.fuzz.QRatio(synset_1,synset_2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "#synset_1 = get_synonym_array(syns[\"n.synonyms\"][0])\n",
    "#synset_2 = get_synonym_array(syns[\"n.synonyms\"][1])\n",
    "synset_1 = [\"scotland bill\"]\n",
    "synset_2 = [\"scots__scots tory\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "41\n",
      "31\n",
      "31\n",
      "27\n",
      "27\n",
      "47\n",
      "33\n"
     ]
    }
   ],
   "source": [
    "print(process.fuzz.partial_ratio(synset_1,synset_2))\n",
    "print(process.fuzz.partial_token_set_ratio(synset_1,synset_2))\n",
    "print(process.fuzz.partial_token_sort_ratio(synset_1,synset_2))\n",
    "print(process.fuzz.token_set_ratio(synset_1,synset_2))\n",
    "print(process.fuzz.token_sort_ratio(synset_1,synset_2))\n",
    "print(process.fuzz.ratio(synset_1,synset_2))\n",
    "print(process.fuzz.QRatio(synset_1,synset_2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initiating fuzzy matching.......\n",
      "------------------------------------------------\n",
      "Input column name : bma\n",
      "Similarity Ratios when compared with the similar reference list items are as below :  [(\"bma's\", 90), (\"bma's scottish general practitioners' committee\", 60)]\n",
      "Associated Reference list item with highest similarity : \n",
      "(\"bma's\", 90)\n",
      "------------------------------------------------\n",
      "Input column name : british medical association\n",
      "Similarity Ratios when compared with the similar reference list items are as below :  [(\"bma's\", 54), (\"bma's scottish general practitioners' committee\", 53)]\n",
      "Associated Reference list item with highest similarity : \n",
      "(\"bma's\", 54)\n",
      "------------------------------------------------\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Datatypes Check : \n",
      "input_list_item           object\n",
      "similar_ref_list_item     object\n",
      "similarity_score         float64\n",
      "dtype: object\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "File Generation error : -----No output filename mentioned.------\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Output Data Frame looks like : \n",
      "               input_list_item similar_ref_list_item  similarity_score\n",
      "0                          bma                 bma's              0.90\n",
      "1  british medical association                 bma's              0.54\n",
      "***********************************************\n",
      "Fuzzy Matching Output generated in : \n",
      "--- 0.009972810745239258 seconds ---\n",
      "***********************************************\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input_list_item</th>\n",
       "      <th>similar_ref_list_item</th>\n",
       "      <th>similarity_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bma</td>\n",
       "      <td>bma's</td>\n",
       "      <td>0.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>british medical association</td>\n",
       "      <td>bma's</td>\n",
       "      <td>0.54</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               input_list_item similar_ref_list_item  similarity_score\n",
       "0                          bma                 bma's              0.90\n",
       "1  british medical association                 bma's              0.54"
      ]
     },
     "execution_count": 221,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from two_lists_similarity import Calculate_Similarity as cs\n",
    "csObj =cs(synset_1,synset_2)\n",
    "csObj.fuzzy_match_output()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input_list_item</th>\n",
       "      <th>similar_ref_list_item</th>\n",
       "      <th>similarity_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bma</td>\n",
       "      <td>bma's</td>\n",
       "      <td>0.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>british medical association</td>\n",
       "      <td>bma's</td>\n",
       "      <td>0.54</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               input_list_item similar_ref_list_item  similarity_score\n",
       "0                          bma                 bma's              0.90\n",
       "1  british medical association                 bma's              0.54"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "csObj.similarity_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "syns = g.run('MATCH (n) RETURN n.synonyms, ID(n)').to_data_frame()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "process.fuzz.token_set_ratio(get_synonym_array(syns[0][0]),[\"government\",\"scottish government\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "syns[\"n.synonyms\"][1] = get_synonym_array(syns[\"n.synonyms\"][1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "88"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "process.fuzz.token_set_ratio([\"salmond's\",\"salmond\"],[\"alex salmond\",\"salmond\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['department for business']"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "syns[\"n.synonyms\"][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table><tr><th>s</th></tr><tr><td style=\"text-align:left\">(_1219:Entity {group: &#039;political group&#039;, name: &#039;Labour&#039;, side: -1, synonyms: &#039;_labour__labour party__labours_&#039;, tag: &#039;NA&#039;})</td></tr></table>"
      ],
      "text/plain": [
       " s                                                                                                                           \n",
       "-----------------------------------------------------------------------------------------------------------------------------\n",
       " (_1219:Entity {group: 'political group', name: 'Labour', side: -1, synonyms: '_labour__labour party__labours_', tag: 'NA'}) "
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g.run(\"MATCH (s) WHERE ID(s) = 1219 SET s.name = $name RETURN s\", parameters={\"name\": \"Labour\"}).to_table()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'_snp government__scottish government__uk government_'"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"_\" + \"__\".join(set(synset_1) | set(synset_2)) + \"_\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "synset_combined = set(synset_1) | set(synset_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<py2neo.database.Cursor at 0x23b177e8cc8>"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#OCCURS_WITH = Relationship.type(\"OCCURS_WITH\")\n",
    "#node_1 = g.evaluate('MATCH (e_1:Entity) WHERE (e_1.synonyms CONTAINS $name) RETURN n',\n",
    "#                                        parameters={'name': \"labour\"})\n",
    "#node_2 = g.evaluate('MATCH (e_2:Entity) WHERE (e_2.synonyms CONTAINS $name) RETURN n',\n",
    "#                                        parameters={'name': \"tory\"})\n",
    "g.run(\"MATCH (e_1:Entity),(e_2:Entity) WHERE e_1.synonyms CONTAINS $name_1 AND e_2.synonyms CONTAINS $name_2 CREATE (e_1)-[r1:OCCURS_WITH {article: $id, count: $count, weight: $weight}]->(e_2)\",\n",
    "      parameters={\"name_1\": \"_\"+\"labour\"+\"_\", \"name_2\": \"_\"+\"tory\"+\"_\", \"id\": 2, \"count\": 5, \"weight\": 0.5})\n",
    "#rel = OCCURS_WITH(node_1, node_2)\n",
    "#rel[\"article_id\"] = int(1)\n",
    "#rel.__primarykey__ = \"article_id\"\n",
    "#rel[\"count\"] = int(5)\n",
    "#rel[\"weight\"] = 0.5\n",
    "#g.merge(rel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'_scottish government__snp government__nationalist government_'"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "syns[\"n.synonyms\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['', 'home office', '']"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "syns[\"n.synonyms\"][0].split(\"_\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def get_synonyms_from_db(graph):\n",
    "    syns = graph.run('MATCH (n) RETURN n.synonyms, ID(n)').to_data_frame()\n",
    "    for i in range(len(syns)):\n",
    "        syns[\"n.synonyms\"][i] = get_synonyms(syns[\"n.synonyms\"][i])\n",
    "    return syns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "syns = g.run('MATCH (n:Entity) RETURN n.synonyms, ID(n)').to_data_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "i = 1\n",
    "syns[\"n.synonyms\"][i] = get_synonyms(syns[\"n.synonyms\"][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "syns[\"ID(n)\"][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2020-07-03 16:03:21.228 INFO in 'deeppavlov.core.data.simple_vocab'['simple_vocab'] at line 115: [loading vocabulary from C:\\Users\\Daniel\\.deeppavlov\\models\\ner_ontonotes_bert_mult\\tag.dict]\n",
      "2020-07-03 16:04:05.428 INFO in 'deeppavlov.core.models.tf_model'['tf_model'] at line 51: [loading model from C:\\Users\\Daniel\\.deeppavlov\\models\\ner_ontonotes_bert_mult\\model]\n",
      "--- Logging error ---\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\logging\\__init__.py\", line 1028, in emit\n",
      "    stream.write(msg + self.terminator)\n",
      "ValueError: I/O operation on closed file.\n",
      "Call stack:\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\runpy.py\", line 193, in _run_module_as_main\n",
      "    \"__main__\", mod_spec)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\runpy.py\", line 85, in _run_code\n",
      "    exec(code, run_globals)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n",
      "    app.launch_new_instance()\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\traitlets\\config\\application.py\", line 664, in launch_instance\n",
      "    app.start()\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 563, in start\n",
      "    self.io_loop.start()\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 132, in start\n",
      "    self.asyncio_loop.run_forever()\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\asyncio\\base_events.py\", line 534, in run_forever\n",
      "    self._run_once()\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\asyncio\\base_events.py\", line 1771, in _run_once\n",
      "    handle._run()\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\asyncio\\events.py\", line 88, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\tornado\\ioloop.py\", line 758, in _run_callback\n",
      "    ret = callback()\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\tornado\\stack_context.py\", line 300, in null_wrapper\n",
      "    return fn(*args, **kwargs)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\tornado\\gen.py\", line 1233, in inner\n",
      "    self.run()\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\tornado\\gen.py\", line 1147, in run\n",
      "    yielded = self.gen.send(value)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 365, in process_one\n",
      "    yield gen.maybe_future(dispatch(*args))\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\tornado\\gen.py\", line 326, in wrapper\n",
      "    yielded = next(result)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 272, in dispatch_shell\n",
      "    yield gen.maybe_future(handler(stream, idents, msg))\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\tornado\\gen.py\", line 326, in wrapper\n",
      "    yielded = next(result)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 542, in execute_request\n",
      "    user_expressions, allow_stdin,\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\tornado\\gen.py\", line 326, in wrapper\n",
      "    yielded = next(result)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 294, in do_execute\n",
      "    res = shell.run_cell(code, store_history=store_history, silent=silent)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 536, in run_cell\n",
      "    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2855, in run_cell\n",
      "    raw_cell, store_history, silent, shell_futures)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2881, in _run_cell\n",
      "    return runner(coro)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\IPython\\core\\async_helpers.py\", line 68, in _pseudo_sync_runner\n",
      "    coro.send(None)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3058, in run_cell_async\n",
      "    interactivity=interactivity, compiler=compiler, result=result)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3249, in run_ast_nodes\n",
      "    if (await self.run_code(code, result,  async_=asy)):\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3326, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-7-b3e2610702b0>\", line 3, in <module>\n",
      "    ner_model = build_model(configs.ner.ner_ontonotes_bert_mult, download=False)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\deeppavlov\\core\\commands\\infer.py\", line 62, in build_model\n",
      "    component = from_params(component_config, mode=mode, serialized=component_serialized)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\deeppavlov\\core\\common\\params.py\", line 106, in from_params\n",
      "    component = obj(**dict(config_params, **kwargs))\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\deeppavlov\\core\\models\\tf_backend.py\", line 76, in __call__\n",
      "    obj.__init__(*args, **kwargs)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\deeppavlov\\core\\models\\tf_backend.py\", line 28, in _wrapped\n",
      "    return func(*args, **kwargs)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\deeppavlov\\models\\bert\\bert_sequence_tagger.py\", line 529, in __init__\n",
      "    **kwargs)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\deeppavlov\\models\\bert\\bert_sequence_tagger.py\", line 259, in __init__\n",
      "    self.load()\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\deeppavlov\\core\\models\\tf_backend.py\", line 28, in _wrapped\n",
      "    return func(*args, **kwargs)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\deeppavlov\\models\\bert\\bert_sequence_tagger.py\", line 457, in load\n",
      "    return super().load(exclude_scopes=exclude_scopes, **kwargs)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\deeppavlov\\core\\models\\tf_model.py\", line 251, in load\n",
      "    return super().load(exclude_scopes=exclude_scopes, **kwargs)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\deeppavlov\\core\\models\\tf_model.py\", line 55, in load\n",
      "    saver.restore(self.sess, path)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\training\\saver.py\", line 1284, in restore\n",
      "    logging.info(\"Restoring parameters from %s\", checkpoint_prefix)\n",
      "  File \"C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\tensorflow_core\\python\\platform\\tf_logging.py\", line 156, in info\n",
      "    get_logger().info(msg, *args, **kwargs)\n",
      "Message: 'Restoring parameters from %s'\n",
      "Arguments: ('C:\\\\Users\\\\Daniel\\\\.deeppavlov\\\\models\\\\ner_ontonotes_bert_mult\\\\model',)\n"
     ]
    }
   ],
   "source": [
    "from deeppavlov import configs, build_model\n",
    "\n",
    "ner_model = build_model(configs.ner.ner_ontonotes_bert_mult, download=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[['The',\n",
       "   'Scottish',\n",
       "   'population',\n",
       "   'is',\n",
       "   'unsatisfied',\n",
       "   'with',\n",
       "   'the',\n",
       "   'decision',\n",
       "   'of',\n",
       "   'the',\n",
       "   'Scottish',\n",
       "   'Labor',\n",
       "   'party',\n",
       "   'to',\n",
       "   'block',\n",
       "   'the',\n",
       "   'vote',\n",
       "   'on',\n",
       "   'the',\n",
       "   'Scottish',\n",
       "   \"Government's\",\n",
       "   'proposal',\n",
       "   'which',\n",
       "   'was',\n",
       "   'also',\n",
       "   'supported',\n",
       "   'by',\n",
       "   'the',\n",
       "   'UK',\n",
       "   'Government',\n",
       "   'and',\n",
       "   'UK',\n",
       "   'Labor',\n",
       "   '.']],\n",
       " [['O',\n",
       "   'B-NORP',\n",
       "   'O',\n",
       "   'O',\n",
       "   'O',\n",
       "   'O',\n",
       "   'O',\n",
       "   'O',\n",
       "   'O',\n",
       "   'B-ORG',\n",
       "   'I-ORG',\n",
       "   'I-ORG',\n",
       "   'I-ORG',\n",
       "   'O',\n",
       "   'O',\n",
       "   'O',\n",
       "   'O',\n",
       "   'O',\n",
       "   'B-ORG',\n",
       "   'I-ORG',\n",
       "   'I-ORG',\n",
       "   'O',\n",
       "   'O',\n",
       "   'O',\n",
       "   'O',\n",
       "   'O',\n",
       "   'O',\n",
       "   'B-ORG',\n",
       "   'I-ORG',\n",
       "   'I-ORG',\n",
       "   'O',\n",
       "   'B-ORG',\n",
       "   'I-ORG',\n",
       "   'O']]]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ner_model([\"The Scottish population is unsatisfied with the decision of the Scottish Labor party to block the vote on the Scottish Government's proposal which was also supported by the UK Government and UK Labor.\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "ename": "ClientError",
     "evalue": "SyntaxError: The old parameter syntax `{param}` is no longer supported. Please use `$param` instead (line 1, column 20 (offset: 19))\r\n\"MATCH (n:Node {id: {parameter1}})-[r:someType]-(m:Node {id: {parameter2}}) RETURN SIGN(COUNT(r))\"\r\n                    ^",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mClientError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-17-b15d7e96442e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"MATCH (n:Node {id: {parameter1}})-[r:someType]-(m:Node {id: {parameter2}}) RETURN SIGN(COUNT(r))\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\py2neo\\database.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, cypher, parameters, **kwparameters)\u001b[0m\n\u001b[0;32m    531\u001b[0m         \u001b[1;33m:\u001b[0m\u001b[1;32mreturn\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    532\u001b[0m         \"\"\"\n\u001b[1;32m--> 533\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbegin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mautocommit\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcypher\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwparameters\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    534\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    535\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mseparate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msubgraph\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\py2neo\\database.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, cypher, parameters, **kwparameters)\u001b[0m\n\u001b[0;32m    826\u001b[0m                                              \u001b[0mgraph\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    827\u001b[0m                                              \u001b[0mkeys\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 828\u001b[1;33m                                              entities=entities))\n\u001b[0m\u001b[0;32m    829\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mCypherError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merror\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    830\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mGraphError\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhydrate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m{\u001b[0m\u001b[1;34m\"code\"\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0merror\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcode\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"message\"\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0merror\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\py2neo\\internal\\connectors.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, statement, parameters, tx, graph, keys, entities)\u001b[0m\n\u001b[0;32m    290\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstatement\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtx\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgraph\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkeys\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mentities\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    291\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mtx\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 292\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_run_1\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstatement\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgraph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkeys\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mentities\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    293\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    294\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_run_in_tx\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstatement\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgraph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkeys\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mentities\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\py2neo\\internal\\connectors.py\u001b[0m in \u001b[0;36m_run_1\u001b[1;34m(self, statement, parameters, graph, keys, entities)\u001b[0m\n\u001b[0;32m    253\u001b[0m                     on_success=result.update_metadata, on_failure=self._fail, on_summary=result.done)\n\u001b[0;32m    254\u001b[0m         \u001b[0mcx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 255\u001b[1;33m         \u001b[0mcx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    256\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    257\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\neobolt\\direct.py\u001b[0m in \u001b[0;36mfetch\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    417\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    418\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 419\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fetch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    420\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mknown_errors\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0merror\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    421\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhandle\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0merror\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munresolved_address\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\neobolt\\direct.py\u001b[0m in \u001b[0;36m_fetch\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    459\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0msummary_signature\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34mb\"\\x7F\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    460\u001b[0m             \u001b[0mlog_debug\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"[#%04X]  S: FAILURE %r\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlocal_port\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msummary_metadata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 461\u001b[1;33m             \u001b[0mresponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_failure\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msummary_metadata\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    462\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    463\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mProtocolError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Unexpected response message with signature %02X\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0msummary_signature\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\neobolt\\direct.py\u001b[0m in \u001b[0;36mon_failure\u001b[1;34m(self, metadata)\u001b[0m\n\u001b[0;32m    749\u001b[0m         \u001b[0mhandler\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhandlers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"on_failure\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    750\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhandler\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 751\u001b[1;33m             \u001b[0mhandler\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmetadata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    752\u001b[0m         \u001b[0mhandler\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhandlers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"on_summary\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    753\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhandler\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\py2neo\\internal\\connectors.py\u001b[0m in \u001b[0;36m_fail\u001b[1;34m(cls, metadata)\u001b[0m\n\u001b[0;32m    286\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_fail\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmetadata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    287\u001b[0m         \u001b[1;32mfrom\u001b[0m \u001b[0mpy2neo\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdatabase\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mGraphError\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 288\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mGraphError\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhydrate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmetadata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    289\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    290\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstatement\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparameters\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtx\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgraph\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkeys\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mentities\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mClientError\u001b[0m: SyntaxError: The old parameter syntax `{param}` is no longer supported. Please use `$param` instead (line 1, column 20 (offset: 19))\r\n\"MATCH (n:Node {id: {parameter1}})-[r:someType]-(m:Node {id: {parameter2}}) RETURN SIGN(COUNT(r))\"\r\n                    ^"
     ]
    }
   ],
   "source": [
    "g.run(\"MATCH (e_1:Entity WHERE e_1.synonyms CONTAINS $name_1)-[r:OCCURS_WITH]-(e_2:Entity WHERE e_2.synonyms CONTAINS $name_2) RETURN SIGN(COUNT(r))\",\n",
    "      parameters={\"name_1\": name_1, \"name_2\": name_2,\n",
    "                                      \"id\": article_id, \"count\": count, \"weight\": weight})\n",
    "\n",
    "#cyphter_1 = \"MATCH (e_1:Entity),(e_2:Entity) WHERE e_1.synonyms CONTAINS $name_1 AND e_2.synonyms CONTAINS $name_2 \"\n",
    "#cyphter_2 = \"CREATE (e_1)-[r1:OCCURS_WITH {article: $id, count: $count, weight: $weight}]-(e_2)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>tag</th>\n",
       "      <th>count</th>\n",
       "      <th>weight</th>\n",
       "      <th>synonyms</th>\n",
       "      <th>locked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [name, tag, count, weight, synonyms, locked]\n",
       "Index: []"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "initial = EntityDataset()\n",
    "initial.df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>tag</th>\n",
       "      <th>count</th>\n",
       "      <th>weight</th>\n",
       "      <th>synonyms</th>\n",
       "      <th>locked</th>\n",
       "      <th>side</th>\n",
       "      <th>group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>scottish government</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>_scottish government__snp government__national...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>political group</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>uk government: dept for business</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>_department for business_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>political group</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>uk government: home office</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>_home office_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>political group</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>conservatives</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>_conservatives__tories__tory__tory party_</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>political group</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>labour</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>_labour__labour party__labours__labour's_</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>political group</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>paul wheelhouse</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>_paul wheelhouse__wheelhouse_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>snp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>steve mccabe</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>_steve mccabe__mccabe_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>labour</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>iain gray</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>_iain gray__gray_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>labour</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>house of commons</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>_commons__house of commons__british parliament_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>holyrood</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>_holyrood__holyrood's__holyrood's health and s...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>137 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 name  tag  count  weight  \\\n",
       "0                 scottish government  NaN      0       0   \n",
       "1    uk government: dept for business  NaN      0       0   \n",
       "2          uk government: home office  NaN      0       0   \n",
       "3                       conservatives  NaN      0       0   \n",
       "4                              labour  NaN      0       0   \n",
       "..                                ...  ...    ...     ...   \n",
       "132                   paul wheelhouse  NaN      0       0   \n",
       "133                      steve mccabe  NaN      0       0   \n",
       "134                         iain gray  NaN      0       0   \n",
       "135                  house of commons  NaN      0       0   \n",
       "136                          holyrood  NaN      0       0   \n",
       "\n",
       "                                              synonyms  locked  side  \\\n",
       "0    _scottish government__snp government__national...     1.0   1.0   \n",
       "1                            _department for business_     NaN  -1.0   \n",
       "2                                        _home office_     NaN   1.0   \n",
       "3            _conservatives__tories__tory__tory party_     1.0   1.0   \n",
       "4            _labour__labour party__labours__labour's_     1.0  -1.0   \n",
       "..                                                 ...     ...   ...   \n",
       "132                      _paul wheelhouse__wheelhouse_     NaN   0.0   \n",
       "133                             _steve mccabe__mccabe_     NaN   0.0   \n",
       "134                                  _iain gray__gray_     NaN   0.0   \n",
       "135    _commons__house of commons__british parliament_     NaN   NaN   \n",
       "136  _holyrood__holyrood's__holyrood's health and s...     NaN   NaN   \n",
       "\n",
       "               group  \n",
       "0    political group  \n",
       "1    political group  \n",
       "2    political group  \n",
       "3    political group  \n",
       "4    political group  \n",
       "..               ...  \n",
       "132              snp  \n",
       "133           labour  \n",
       "134           labour  \n",
       "135              NaN  \n",
       "136              NaN  \n",
       "\n",
       "[137 rows x 8 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv(\"C:/Users/Daniel/Documents/Masterarbeit/MUP_nodes.csv\", sep=';',encoding=\"utf-8\")\n",
    "data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'a' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-5-0a312ad38218>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0minitial\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0minitial\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"holyrood\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msort\u001b[0m\u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0minitial\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'a' is not defined"
     ]
    }
   ],
   "source": [
    "initial.df = pd.concat([initial.df,data.loc[[a.get(\"holyrood\")]]],sort= False)\n",
    "initial.df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['scottish government',\n",
       " 'snp government',\n",
       " 'nationalist government',\n",
       " 'scottish government',\n",
       " 'snp government',\n",
       " 'nationalist government',\n",
       " \"scottish government's\"]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_synonyms(data[\"synonyms\"][0])\n",
    "#get_synonyms(data[\"synonyms\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "a = dict()\n",
    "for i in range(len(data)):\n",
    "    a.update(a.fromkeys(get_synonyms(data[\"synonyms\"][i]), i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "136"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.get(\"holyrood\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tory\n",
      "3\n",
      "david cameron\n",
      "122\n",
      "prime minister\n",
      "122\n",
      "holyrood\n",
      "136\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "sentence = \"David Cameron, the Tory Prime Minister visited Holyrood today\"\n",
    "start = set()\n",
    "end = set()\n",
    "loc_set = set()\n",
    "for entities in a:\n",
    "    if entities in sentence.lower():\n",
    "        start_temp = sentence.lower().find(entities)\n",
    "        end_temp = sentence.lower().find(entities)+len(entities)\n",
    "        loc_temp = a.get(entities)\n",
    "        if not (((start_temp in start) or (end_temp in end)) and loc_temp in loc_set):\n",
    "            start.add(start_temp)\n",
    "            end.add(end_temp)\n",
    "            loc_set.add(loc_temp)\n",
    "            print(entities)\n",
    "            print(a.get(entities))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'a' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-193-3a9c70792e90>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mentitynetwork\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhelper_classes\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mentity\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mEntity\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mtemp_entity\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"holyrood\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mtemp_entity\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'a' is not defined"
     ]
    }
   ],
   "source": [
    "from entitynetwork.helper_classes.entity import Entity\n",
    "temp_entity = data.iloc[a.get(\"holyrood\"),0:5]\n",
    "temp_entity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "data = pd.read_csv(\"C:/Users/Daniel/Documents/Masterarbeit/mup_valid_articles.csv\", encoding=\"utf-8\")\n",
    "init_nodes = pd.read_csv(\"C:/Users/Daniel/Documents/Masterarbeit/MUP_nodes.csv\", sep=';',encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# load the functionalities\n",
    "from entitynetwork.enititynetwork_pipeline.process_article import *\n",
    "from entitynetwork.database_creation.fill_database import *\n",
    "from entitynetwork.relation_extraction.relation_extraction import *\n",
    "#from entitynetwork.validation_tool.relation_validation import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Establishe DB Connection\n",
    "g = connect_database()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Tag selection\n",
    "\n",
    "selected_tag = [\"B-PERSON\", \"I-PERSON\", \"B-FAC\", \"I-FAC\", \"B-ORG\", \"I-ORG\", \"B-GPE\", \"I-GPE\",\n",
    "                \"B-LOC\", \"I-LOC\", \"B-WORK_OF_ART\", \"I-WORK_OF_ART\", \"B-LAW\", \"I-LAW\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Article0\n",
      "Current Article1\n",
      "Current Article2\n",
      "Current Article3\n",
      "Current Article4\n",
      "Current Article5\n",
      "Current Article6\n",
      "Current Article7\n",
      "Current Article8\n",
      "Current Article9\n",
      "Current Article10\n",
      "Current Article11\n",
      "Current Article12\n",
      "Current Article13\n",
      "Current Article14\n",
      "Current Article15\n",
      "Current Article16\n",
      "Current Article17\n",
      "Current Article18\n",
      "Current Article19\n",
      "Current Article20\n",
      "Current Article21\n",
      "Current Article22\n",
      "Current Article23\n",
      "Current Article24\n",
      "Current Article25\n",
      "Current Article26\n",
      "Current Article27\n",
      "Current Article28\n",
      "Current Article29\n",
      "Current Article30\n",
      "Current Article31\n",
      "Current Article32\n",
      "Current Article33\n",
      "Current Article34\n",
      "Current Article35\n",
      "Current Article36\n",
      "Current Article37\n",
      "Current Article38\n",
      "Current Article39\n",
      "Current Article40\n",
      "Current Article41\n",
      "Current Article42\n",
      "Current Article43\n",
      "Current Article44\n",
      "Current Article45\n",
      "Current Article46\n",
      "Current Article47\n",
      "Current Article48\n",
      "Current Article49\n",
      "Current Article50\n",
      "Current Article51\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\Daniel\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Daniel\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package perluniprops to\n",
      "[nltk_data]     C:\\Users\\Daniel\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package perluniprops is already up-to-date!\n",
      "[nltk_data] Downloading package nonbreaking_prefixes to\n",
      "[nltk_data]     C:\\Users\\Daniel\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package nonbreaking_prefixes is already up-to-date!\n",
      "2020-08-20 22:29:11.15 INFO in 'deeppavlov.core.data.simple_vocab'['simple_vocab'] at line 115: [loading vocabulary from C:\\Users\\Daniel\\.deeppavlov\\models\\ner_ontonotes_bert_mult\\tag.dict]\n",
      "2020-08-20 22:29:56.559 INFO in 'deeppavlov.core.models.tf_model'['tf_model'] at line 51: [loading model from C:\\Users\\Daniel\\.deeppavlov\\models\\ner_ontonotes_bert_mult\\model]\n",
      "C:\\Users\\Daniel\\Documents\\Masterarbeit\\CSS_Termpaper\\EntityNetwork\\entitynetwork\\named_entity_extraction\\entity_extraction.py:65: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  article.entity_dataset.df[\"count\"][j] = article.entity_dataset.df[\"count\"][j] + duplicates[\"count\"][i]\n",
      "C:\\Users\\Daniel\\Documents\\Masterarbeit\\CSS_Termpaper\\EntityNetwork\\entitynetwork\\database_creation\\fill_database.py:185: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  syns[\"n.synonyms\"][i] = get_synonyms(syns[\"n.synonyms\"][i])\n",
      "C:\\Users\\Daniel\\Documents\\Masterarbeit\\CSS_Termpaper\\EntityNetwork\\entitynetwork\\database_creation\\fill_database.py:115: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  only_duplicates[only_duplicates[\"name\"] == only_first[\"name\"][i]][\"count\"]))\n",
      "C:\\Users\\Daniel\\Documents\\Masterarbeit\\CSS_Termpaper\\EntityNetwork\\entitynetwork\\database_creation\\fill_database.py:129: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  str(max(sub_graph[\"id_1\"][i], sub_graph[\"id_2\"][i]))\n",
      "C:\\Users\\Daniel\\Documents\\Masterarbeit\\CSS_Termpaper\\EntityNetwork\\entitynetwork\\database_creation\\fill_database.py:136: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  only_duplicates[only_duplicates[\"id_comb\"] == only_first[\"id_comb\"][i]][\"count\"]))\n",
      "C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\pandas\\core\\ops\\__init__.py:1115: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  result = method(y)\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-44-135e3998571e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      4\u001b[0m     article = Article(data[\"text\"][i], data[\"author_extracted\"][i], data[\"date\"][i], data[\"source\"][i], data[\"id\"][i],\n\u001b[0;32m      5\u001b[0m                download_model=False, selected_tags=selected_tag, n_shift=1,graph=g,initial_syn_sets=init_nodes)\n\u001b[1;32m----> 6\u001b[1;33m     \u001b[0marticle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mprocess_articles\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Documents\\Masterarbeit\\CSS_Termpaper\\EntityNetwork\\entitynetwork\\enititynetwork_pipeline\\process_article.py\u001b[0m in \u001b[0;36mprocess_articles\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     60\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     61\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mprocess_articles\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 62\u001b[1;33m         \u001b[0mextract_entities_from_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     63\u001b[0m         \u001b[0mrelation_extraction\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     64\u001b[0m         \u001b[0mnetwork_matrix_sentences\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Documents\\Masterarbeit\\CSS_Termpaper\\EntityNetwork\\entitynetwork\\named_entity_extraction\\entity_extraction.py\u001b[0m in \u001b[0;36mextract_entities_from_text\u001b[1;34m(article)\u001b[0m\n\u001b[0;32m     54\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# If there is no selected tag at the next place, the previous entity is finished,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     55\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mnew_entity\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mname\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;34m\"\"\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mnew_entity\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mname\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0malready_found\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 56\u001b[1;33m                     \u001b[0marticle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mentity_dataset\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd_entity\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_entity\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     57\u001b[0m                     \u001b[0mnew_entity\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mEntity\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"\"\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# Now reset the entity\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     58\u001b[0m                     \u001b[0mcurrent_tag\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Documents\\Masterarbeit\\CSS_Termpaper\\EntityNetwork\\entitynetwork\\named_entity_extraction\\entity_data_frame.py\u001b[0m in \u001b[0;36madd_entity\u001b[1;34m(self, new_entity)\u001b[0m\n\u001b[0;32m     44\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate_entity\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mentity_index\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnew_entity\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     45\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 46\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend_entity\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_entity\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     47\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     48\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget_entity\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mentity_index\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnew_entity\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Documents\\Masterarbeit\\CSS_Termpaper\\EntityNetwork\\entitynetwork\\named_entity_extraction\\entity_data_frame.py\u001b[0m in \u001b[0;36mappend_entity\u001b[1;34m(self, new_entity)\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mappend_entity\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnew_entity\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 14\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnew_entity\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__dict__\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mignore_index\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     15\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mentity_exists\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnew_entity\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py\u001b[0m in \u001b[0;36mappend\u001b[1;34m(self, other, ignore_index, verify_integrity, sort)\u001b[0m\n\u001b[0;32m   7136\u001b[0m             \u001b[0mignore_index\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mignore_index\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   7137\u001b[0m             \u001b[0mverify_integrity\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mverify_integrity\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 7138\u001b[1;33m             \u001b[0msort\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msort\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   7139\u001b[0m         )\n\u001b[0;32m   7140\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\reshape\\concat.py\u001b[0m in \u001b[0;36mconcat\u001b[1;34m(objs, axis, join, join_axes, ignore_index, keys, levels, names, verify_integrity, sort, copy)\u001b[0m\n\u001b[0;32m    256\u001b[0m     )\n\u001b[0;32m    257\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 258\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    259\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    260\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\reshape\\concat.py\u001b[0m in \u001b[0;36mget_result\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    471\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    472\u001b[0m             new_data = concatenate_block_managers(\n\u001b[1;32m--> 473\u001b[1;33m                 \u001b[0mmgrs_indexers\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnew_axes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconcat_axis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    474\u001b[0m             )\n\u001b[0;32m    475\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\internals\\managers.py\u001b[0m in \u001b[0;36mconcatenate_block_managers\u001b[1;34m(mgrs_indexers, axes, concat_axis, copy)\u001b[0m\n\u001b[0;32m   2052\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2053\u001b[0m             b = make_block(\n\u001b[1;32m-> 2054\u001b[1;33m                 \u001b[0mconcatenate_join_units\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjoin_units\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconcat_axis\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2055\u001b[0m                 \u001b[0mplacement\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mplacement\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2056\u001b[0m             )\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\internals\\concat.py\u001b[0m in \u001b[0;36mconcatenate_join_units\u001b[1;34m(join_units, concat_axis, copy)\u001b[0m\n\u001b[0;32m    266\u001b[0m                 \u001b[0mconcat_values\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mconcat_values\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    267\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 268\u001b[1;33m         \u001b[0mconcat_values\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_concat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_concat_compat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mto_concat\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mconcat_axis\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    269\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    270\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mconcat_values\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\dtypes\\concat.py\u001b[0m in \u001b[0;36m_concat_compat\u001b[1;34m(to_concat, axis)\u001b[0m\n\u001b[0;32m    172\u001b[0m                 \u001b[0mto_concat\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"object\"\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mto_concat\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    173\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 174\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mto_concat\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    175\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    176\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<__array_function__ internals>\u001b[0m in \u001b[0;36mconcatenate\u001b[1;34m(*args, **kwargs)\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Start the validation and store the data\n",
    "for i in range(len(data)):\n",
    "    print(\"Current Article\" + str(i))\n",
    "    article = Article(data[\"text\"][i], data[\"author_extracted\"][i], data[\"date\"][i], data[\"source\"][i], data[\"id\"][i],\n",
    "               download_model=False, selected_tags=selected_tag, n_shift=1,graph=g,initial_syn_sets=init_nodes)\n",
    "    article.process_articles()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>tag</th>\n",
       "      <th>count</th>\n",
       "      <th>weight</th>\n",
       "      <th>synonyms</th>\n",
       "      <th>locked</th>\n",
       "      <th>side</th>\n",
       "      <th>group</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>scottish government</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>_scottish government__snp government__national...</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>political group</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>uk government: dept for business</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>_department for business_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>political group</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>uk government: home office</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>_home office_</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>political group</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>conservatives</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>_conservatives__tories__tory__tory party_</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>political group</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>labour</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>_labour__labour party__labours__labour's_</td>\n",
       "      <td>1</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>political group</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>snp scotland</td>\n",
       "      <td>GPE</td>\n",
       "      <td>1</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>_snp scotland_</td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>anne milton eu</td>\n",
       "      <td>ORG</td>\n",
       "      <td>1</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>_anne milton eu_</td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>house of commons scotland</td>\n",
       "      <td>GPE</td>\n",
       "      <td>1</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>_house of commons scotland_</td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>milton</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>1</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>_milton_</td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>jeremy beadles</td>\n",
       "      <td>PERSON</td>\n",
       "      <td>1</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>_jeremy beadles_</td>\n",
       "      <td></td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>144 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 name     tag count    weight  \\\n",
       "0                 scottish government     NaN     0         0   \n",
       "1    uk government: dept for business     NaN     0         0   \n",
       "2          uk government: home office     NaN     0         0   \n",
       "3                       conservatives     NaN     1         0   \n",
       "4                              labour     NaN     0         0   \n",
       "..                                ...     ...   ...       ...   \n",
       "139                      snp scotland     GPE     1  0.888889   \n",
       "140                    anne milton eu     ORG     1  0.777778   \n",
       "141         house of commons scotland     GPE     1  0.666667   \n",
       "142                            milton  PERSON     1  0.222222   \n",
       "143                    jeremy beadles  PERSON     1  0.111111   \n",
       "\n",
       "                                              synonyms locked  side  \\\n",
       "0    _scottish government__snp government__national...      1   1.0   \n",
       "1                            _department for business_    NaN  -1.0   \n",
       "2                                        _home office_    NaN   1.0   \n",
       "3            _conservatives__tories__tory__tory party_      1   1.0   \n",
       "4            _labour__labour party__labours__labour's_      1  -1.0   \n",
       "..                                                 ...    ...   ...   \n",
       "139                                     _snp scotland_          NaN   \n",
       "140                                   _anne milton eu_          NaN   \n",
       "141                        _house of commons scotland_          NaN   \n",
       "142                                           _milton_          NaN   \n",
       "143                                   _jeremy beadles_          NaN   \n",
       "\n",
       "               group  \n",
       "0    political group  \n",
       "1    political group  \n",
       "2    political group  \n",
       "3    political group  \n",
       "4    political group  \n",
       "..               ...  \n",
       "139              NaN  \n",
       "140              NaN  \n",
       "141              NaN  \n",
       "142              NaN  \n",
       "143              NaN  \n",
       "\n",
       "[144 rows x 8 columns]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "article.entity_dataset.df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "g.run(\"MATCH (a:Entity)-[rel:OCCURS_WITH]->(a) DELETE rel\")\n",
    "sub_graph = g.run(\"MATCH (e:Entity)-[r:OCCURS_IN]->(a:Article) WHERE a.id = $id RETURN e.name AS name,r.count AS count, r.weight AS weight ,ID(r) as ID order by weight desc\",\n",
    "                  parameters={\"id\": }).to_data_frame()\n",
    "all_duplicates = sub_graph[sub_graph[\"name\"].duplicated(keep= False)]\n",
    "only_duplicates = sub_graph[sub_graph[\"name\"].duplicated(keep= \"first\")]\n",
    "only_first = all_duplicates[~all_duplicates.isin(only_duplicates)].dropna().reset_index(drop=True)\n",
    "for i in range(len(only_first)):\n",
    "    only_first[\"count\"][i] = int(only_first[\"count\"][i] + sum(only_duplicates[only_duplicates[\"name\"] == only_first[\"name\"][i]][\"count\"]))\n",
    "    g.run(\"MATCH ()-[r:OCCURS_IN]->() WHERE ID(r) = $id SET r.count = $count\",\n",
    "          parameters={\"id\": only_first[\"ID\"][i], \"count\": only_first[\"count\"][i]})\n",
    "for i in range(len(only_duplicates)):\n",
    "    g.run(\"MATCH ()-[r:OCCURS_IN]->() WHERE ID(r) = $id DELETE r\", parameters={\"id\": only_duplicates[\"ID\"][i]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Daniel\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  import sys\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_1</th>\n",
       "      <th>id_2</th>\n",
       "      <th>count</th>\n",
       "      <th>weight</th>\n",
       "      <th>ID</th>\n",
       "      <th>id_comb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>151</td>\n",
       "      <td>5</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>91</td>\n",
       "      <td>10_151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>112</td>\n",
       "      <td>4</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>97</td>\n",
       "      <td>10_112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>94</td>\n",
       "      <td>151</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>107</td>\n",
       "      <td>94_151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>94</td>\n",
       "      <td>112</td>\n",
       "      <td>3</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>112</td>\n",
       "      <td>94_112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>112</td>\n",
       "      <td>151</td>\n",
       "      <td>8</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>71</td>\n",
       "      <td>112_151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>112</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.138889</td>\n",
       "      <td>177</td>\n",
       "      <td>10_112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>123</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.138889</td>\n",
       "      <td>186</td>\n",
       "      <td>10_123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>136</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>0.138889</td>\n",
       "      <td>154</td>\n",
       "      <td>10_136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>151</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>0.138889</td>\n",
       "      <td>164</td>\n",
       "      <td>10_151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>136</td>\n",
       "      <td>16</td>\n",
       "      <td>1</td>\n",
       "      <td>0.083333</td>\n",
       "      <td>152</td>\n",
       "      <td>16_136</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>122 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     id_1  id_2  count    weight   ID  id_comb\n",
       "0      10   151      5  1.000000   91   10_151\n",
       "1      10   112      4  1.000000   97   10_112\n",
       "2      94   151      1  1.000000  107   94_151\n",
       "3      94   112      3  1.000000  112   94_112\n",
       "4     112   151      8  1.000000   71  112_151\n",
       "..    ...   ...    ...       ...  ...      ...\n",
       "117   112    10      1  0.138889  177   10_112\n",
       "118   123    10      1  0.138889  186   10_123\n",
       "119   136    10      1  0.138889  154   10_136\n",
       "120   151    10      2  0.138889  164   10_151\n",
       "121   136    16      1  0.083333  152   16_136\n",
       "\n",
       "[122 rows x 6 columns]"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cy_1 = \"MATCH (e_1:Entity)-[r:OCCURS_WITH]->(e_2:Entity) WHERE r.article = $id RETURN ID(e_1) AS id_1,\"\n",
    "cy_2 = \" ID(e_2) AS id_2, r.count AS count, r.weight AS weight ,ID(r) as ID order by weight desc\"\n",
    "sub_graph = g.run(cy_1 + cy_2, parameters={\"id\": 4}).to_data_frame()\n",
    "sub_graph[\"id_comb\"] = \"\"\n",
    "for i in range(len(sub_graph)):\n",
    "    sub_graph[\"id_comb\"][i] = str(min(sub_graph[\"id_1\"][i],sub_graph[\"id_2\"][i])) + \"_\" + \\\n",
    "                              str(max(sub_graph[\"id_1\"][i],sub_graph[\"id_2\"][i]))\n",
    "all_duplicates = sub_graph[sub_graph[\"id_comb\"].duplicated(keep= False)]\n",
    "only_duplicates = sub_graph[sub_graph[\"id_comb\"].duplicated(keep= \"first\")]\n",
    "only_first = all_duplicates[~all_duplicates.isin(only_duplicates)].dropna().reset_index(drop=True)\n",
    "for i in range(len(only_first)):\n",
    "    only_first[\"count\"][i] = int(only_first[\"count\"][i] + sum(only_duplicates[only_duplicates[\"id_comb\"] ==\n",
    "                                                                              only_first[\"id_comb\"][i]][\"count\"]))\n",
    "    g.run(\"MATCH ()-[r:OCCURS_IN]->() WHERE ID(r) = $id SET r.count = $count\",\n",
    "          parameters={\"id\": only_first[\"ID\"][i], \"count\": only_first[\"count\"][i]})\n",
    "for i in range(len(only_duplicates)):\n",
    "    g.run(\"MATCH ()-[r:OCCURS_IN]->() WHERE ID(r) = $id DELETE r\", parameters={\"id\": only_duplicates[\"ID\"][i]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "all_duplicates = sub_graph[sub_graph[\"id_comb\"].duplicated(keep= False)]\n",
    "only_duplicates = sub_graph[sub_graph[\"id_comb\"].duplicated(keep= \"first\")]\n",
    "only_first = all_duplicates[~all_duplicates.isin(only_duplicates)].dropna().reset_index(drop=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_1</th>\n",
       "      <th>id_2</th>\n",
       "      <th>count</th>\n",
       "      <th>weight</th>\n",
       "      <th>ID</th>\n",
       "      <th>id_comb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10.0</td>\n",
       "      <td>151.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>91.0</td>\n",
       "      <td>10_151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10.0</td>\n",
       "      <td>112.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>97.0</td>\n",
       "      <td>10_112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>94.0</td>\n",
       "      <td>112.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>112.0</td>\n",
       "      <td>94_112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>112.0</td>\n",
       "      <td>151.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>71.0</td>\n",
       "      <td>112_151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.944444</td>\n",
       "      <td>87.0</td>\n",
       "      <td>10_81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>94.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.944444</td>\n",
       "      <td>104.0</td>\n",
       "      <td>81_94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>112.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.944444</td>\n",
       "      <td>67.0</td>\n",
       "      <td>81_112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>81.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>113.0</td>\n",
       "      <td>81_150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>112.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>68.0</td>\n",
       "      <td>112_150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9.0</td>\n",
       "      <td>112.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>149.0</td>\n",
       "      <td>9_112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>19.0</td>\n",
       "      <td>112.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>165.0</td>\n",
       "      <td>19_112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>81.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>114.0</td>\n",
       "      <td>9_81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>112.0</td>\n",
       "      <td>136.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>70.0</td>\n",
       "      <td>112_136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>150.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>129.0</td>\n",
       "      <td>9_150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>16.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>167.0</td>\n",
       "      <td>16_81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>81.0</td>\n",
       "      <td>152.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>119.0</td>\n",
       "      <td>81_152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>112.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>73.0</td>\n",
       "      <td>16_112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>112.0</td>\n",
       "      <td>152.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>74.0</td>\n",
       "      <td>112_152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>89.0</td>\n",
       "      <td>9_10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>94.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>106.0</td>\n",
       "      <td>9_94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>10.0</td>\n",
       "      <td>136.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.722222</td>\n",
       "      <td>90.0</td>\n",
       "      <td>10_136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>112.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>77.0</td>\n",
       "      <td>73_112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>112.0</td>\n",
       "      <td>146.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>82.0</td>\n",
       "      <td>112_146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>112.0</td>\n",
       "      <td>153.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>83.0</td>\n",
       "      <td>112_153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>10.0</td>\n",
       "      <td>123.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>102.0</td>\n",
       "      <td>10_123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>112.0</td>\n",
       "      <td>123.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>84.0</td>\n",
       "      <td>112_123</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id_1   id_2  count    weight     ID  id_comb\n",
       "0    10.0  151.0    5.0  1.000000   91.0   10_151\n",
       "1    10.0  112.0    4.0  1.000000   97.0   10_112\n",
       "2    94.0  112.0    3.0  1.000000  112.0   94_112\n",
       "3   112.0  151.0    8.0  1.000000   71.0  112_151\n",
       "4    10.0   81.0   10.0  0.944444   87.0    10_81\n",
       "5    94.0   81.0    5.0  0.944444  104.0    81_94\n",
       "6   112.0   81.0   14.0  0.944444   67.0   81_112\n",
       "7    81.0  150.0    8.0  0.916667  113.0   81_150\n",
       "8   112.0  150.0    7.0  0.916667   68.0  112_150\n",
       "9     9.0  112.0    4.0  0.888889  149.0    9_112\n",
       "10   19.0  112.0    2.0  0.888889  165.0   19_112\n",
       "11   81.0    9.0    7.0  0.888889  114.0     9_81\n",
       "12  112.0  136.0    2.0  0.888889   70.0  112_136\n",
       "13  150.0    9.0    2.0  0.888889  129.0    9_150\n",
       "14   16.0   81.0    1.0  0.833333  167.0    16_81\n",
       "15   81.0  152.0    2.0  0.833333  119.0   81_152\n",
       "16  112.0   16.0    1.0  0.833333   73.0   16_112\n",
       "17  112.0  152.0    1.0  0.833333   74.0  112_152\n",
       "18   10.0    9.0    2.0  0.777778   89.0     9_10\n",
       "19   94.0    9.0    1.0  0.777778  106.0     9_94\n",
       "20   10.0  136.0    2.0  0.722222   90.0   10_136\n",
       "21  112.0   73.0    1.0  0.666667   77.0   73_112\n",
       "22  112.0  146.0    3.0  0.555556   82.0  112_146\n",
       "23  112.0  153.0    1.0  0.222222   83.0  112_153\n",
       "24   10.0  123.0    1.0  0.166667  102.0   10_123\n",
       "25  112.0  123.0    1.0  0.166667   84.0  112_123"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "only_first"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cy_1 = \"MATCH (e:Entity)-[r:OCCURS_IN]->(a:Article) WHERE a.id = $id RETURN e.name AS name, \"\n",
    "cy_2 = \"r.count AS count, r.weight AS weight ,ID(r) as ID order by weight desc\"\n",
    "sub_graph = article.graph.run(cy_1 + cy_2, parameters={\"id\": int(237)}).to_data_frame()\n",
    "sub_graph.shape[1] != 0\n",
    "#all_duplicates = sub_graph[sub_graph[\"name\"].duplicated(keep=False)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\" (p) ONE of Britain\\'s biggest pub groups claims the BULGARIANS could scupper the Prime Minister\\'s plans for minimum pricing on booze.',\n",
       " \"(p) Pedigree Ale brewer MARSTON'S said the country had already objected to Scotland's pricing plan amid concerns it will clobber Bulgarian wine exports.\",\n",
       " '(p) Marston\\'s chief exec Ralph Findlay told The Sun: \"\"I would think if they made a challenge in Scotland they will make one in England and Wales.\"\"',\n",
       " '(p) He claimed pubs will embrace plans to charge at least 45p a unit as it will narrow the gap on supermarkets.',\n",
       " \"Marston's annual profits rose 9 per cent to £87.8million.\",\n",
       " 'It will open 25 boozers next year.\"']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "article.sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer as SIA\n",
    "result = []\n",
    "for sentence in article.sentences:\n",
    "    pol_score = SIA().polarity_scores(sentence)\n",
    "    result.append(pol_score)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'neg': 0.0, 'neu': 1.0, 'pos': 0.0, 'compound': 0.0},\n",
       " {'neg': 0.0, 'neu': 1.0, 'pos': 0.0, 'compound': 0.0},\n",
       " {'neg': 0.0, 'neu': 0.951, 'pos': 0.049, 'compound': 0.0772},\n",
       " {'neg': 0.0, 'neu': 0.897, 'pos': 0.103, 'compound': 0.3182},\n",
       " {'neg': 0.0, 'neu': 0.707, 'pos': 0.293, 'compound': 0.4404},\n",
       " {'neg': 0.0, 'neu': 1.0, 'pos': 0.0, 'compound': 0.0}]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'neg': 0.0, 'neu': 0.941, 'pos': 0.059, 'compound': 0.6705}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SIA().polarity_scores(data[\"text\"][298])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#type(article.sentences[0])\n",
    "type(data[\"text\"][298])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SIA().polarity_scores(article.sentences[0]).get(\"neu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from nltk import tokenize\n",
    "import nltk\n",
    "import os\n",
    "path_to_gs=\"C:\\\\Program Files\\\\gs\\\\gs9.52\\\\bin\"\n",
    "os.environ['PATH']+=os.pathsep+path_to_gs\n",
    "sentences = tokenize.sent_tokenize(data[\"text\"][64])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['\" (p) DRINKERS will pay a minimum price for alcohol under plans instigated by David Cameron to tackle a growing health crisis, The Daily Telegraph can disclose.',\n",
       " 'The Prime Minister has ordered officials to develop a scheme in England to stop the sale of alcohol at below 40p to 50p a unit in shops and supermarkets.',\n",
       " '(p) Ministers could copy Scottish proposals, which would ban the sale of alcohol below 45p a unit, or bring in a more sophisticated system of taxes based on the number of alcohol units contained in the drink.',\n",
       " '(p) Both options would cost drinkers an estimated extra £700million a year, with any extra tax revenue potentially going to the NHS.',\n",
       " 'The Daily Telegraph understands that the Prime Minister personally ordered the radical \"\"big bang\"\" approach, which will be included in the Government\\'s forthcoming alcohol strategy.',\n",
       " 'It was due for release next month, but has now been delayed until February.',\n",
       " '(p) A recent official study found that setting a minimum price of 30p per unit would prevent 300 deaths a year, 40p about 1,000 deaths, and 50p more than 2,000 premature deaths.',\n",
       " '(p) The Downing Street diktat has led to intense Whitehall discussions and disagreements over how the minimum price, which has widespread support among the medical profession, can be introduced.',\n",
       " '(p) Theresa May, the Home Secretary, is said to favour taxing drink on the basis of alcoholic units.',\n",
       " 'The Business Department has warned that forcing firms to charge a minimum price could be illegal under European law.',\n",
       " '(p) Andrew Lansley, the Health Secretary, favours a voluntary approach, but he has been overruled by Mr Cameron, although the compulsory scheme might fall foul of government lawyers.',\n",
       " '(p) The minimum price would be accompanied by an \"\"aggressive\"\" public health campaign and a more draconian approach to curtailing the sale of alcohol in shops, pubs and clubs.',\n",
       " '(p) A Whitehall source said: \"\"The Prime Minister has decided that when it comes to alcohol, something pretty radical now has to be done and he is keen on the minimum price.',\n",
       " 'It is complicated how this can be delivered, particularly under European law, but it is clear that the voluntary approach has not worked.\"\"',\n",
       " 'At present, the system of alcohol taxation is relatively crude.',\n",
       " 'Beer and lager is taxed at about 18p a unit, compared with 19p a unit for wine and about 25p a unit for standard spirits.',\n",
       " 'VAT at 20 per cent is also charged on alcoholic drinks.',\n",
       " '(p) The most radical scheme would involve a sharp rise in alcohol taxation.',\n",
       " 'However, this is likely to prove politically unpalatable and would penalise responsible drinkers who are already struggling with high taxes elsewhere.',\n",
       " '(p) A more sophisticated scheme would target cheap drink sold in supermarkets and shops, while not hitting those sold in pubs or more expensive alcoholic drinks.',\n",
       " '(p) A well-placed source said: \"\"The minimum price is really designed to push up the cheapest alcohol prices, which cause the most damage, rather than an acrossthe-board rise.',\n",
       " 'The Prime Minister is very concerned about protecting traditional pubs.\"\"',\n",
       " '(p) Mr Cameron is thought to have opted for a \"\"big bang\"\" approach to the alcohol problem after noting the success of the ban on smoking in public places.',\n",
       " '(p) Scotland is currently proposing a minimum alcohol price of about 45p a unit and several councils in England, including Greater Manchester and Merseyside, are considering bylaws to set minimum alcohol prices.',\n",
       " '(p) Scottish estimates suggest that a minimum price per unit of 45p would result in the steepest price increases for cider, gin and vodka, while wine, beer and whisky would see more modest rises.',\n",
       " '(p) A bottle of own-brand gin with around 37.5 per cent alcohol content would go up from £6.95 to £11.85.',\n",
       " 'A two-litre bottle of own-brand cider would more than triple in price from £1.20 to £3.75.',\n",
       " '(p) The cost of a £12 bottle of whisky would rise to £12.60, while a bottle of cheap wine would go up from around £3.75 to £4.20.',\n",
       " 'A four-pack of beer with more than five per cent alcohol content would cost a minimum of about £3.95.',\n",
       " '(p) European law is complicated and minimum prices are only likely to be allowed if the authorities can demonstrate that they are tackling a major health problem without undermining competition.',\n",
       " 'Last month, a group of 19 leading medical organisations warned that \"\"pocketmoney prices\"\" for alcohol were endangering thousands of lives every year.',\n",
       " '(p) In a letter to The Daily Telegraph, doctors from the British Medical Association and Royal College of Physicians claimed that minimum pricing for alcohol would be the most \"\"simple and effective mechanism\"\" for tackling the problem.',\n",
       " '(p) Following the warning, the Prime Minister said he would \"\"look very carefully\"\" at how to tackle the problem of cheap alcohol.',\n",
       " '(p) Historically, governments have been reluctant to look at minimum pricing because of concerns about the legality of the move and accusations of promoting a \"\"nanny state\"\" culture.',\n",
       " 'Mr Cameron may also face criticism from Tories unhappy at any move to carry out such an intervention by raising taxes.',\n",
       " '(p) The Coalition has already banned supermarkets from selling alcohol at a loss and introduced higher duties for super-strength beers and ciders.',\n",
       " 'But lager can still be sold for about 38p a can and wine for £2 per bottle.',\n",
       " \"Dr Sarah Wollaston, an MP on the Common's health committee and a former GP, said that alcohol misuse was costing the nation £20billion, or £800 for every family.\",\n",
       " 'She has campaigned in Parliament for minimum pricing, arguing that alcohol abuse was the \"\"single largest cause of deaths among young people.\"\"',\n",
       " '(p) \"\"Most health experts feel that changing pricing is the most effective way of achieving results,\"\" she said.',\n",
       " '(p) Figures published earlier this month showed that twice as many people were being treated in hospital because of alcohol misuse compared with 10 years ago.',\n",
       " '(p) Alcohol is linked to more than one million admissions to hospital each year, about 13,000 new cases of cancer and one in four deaths of people aged 15 to 24.',\n",
       " '(p) Gavin Partington, of the Wine and Spirit Trade Association, said there was \"\"no evidence [a minimum price per unit] will tackle alcohol misuse\"\".',\n",
       " '(p) The association would rather see \"\"enforcement, education and a series of policies to address the root causes of alcohol misuse\"\", he added.',\n",
       " 'Editorial Comment: Page 25 (p) Who picks up the tab?',\n",
       " 'How the system could work (p) There are two basic options for fixing a minimum price for alcohol - a tax per unit or a minimum price per unit.',\n",
       " '(p) Both methods would add about £700million a year to the national drinks bill.',\n",
       " '(p) If higher taxation were introduced, the extra revenue would go to the Treasury, while a minimum price per unit would benefit retailers and the drinks industry.',\n",
       " '(p) Scotland has opted for a minimum price per unit - the simplest mechanism - of 45p.',\n",
       " 'This would add nearly £5 to a £6.95 bottle of gin and more than triple the price of the cheapest two-litre bottle of cider to £3.75.',\n",
       " '(p) An alternative to this plan would be a straightforward tax per unit.',\n",
       " 'A report from the Institute of Fiscal Studies stated: \"\"If the intention is to raise the overall price of alcohol, it would seem preferable that these revenues flow to the Government.',\n",
       " '(p) \"\"At the moment, different types and strengths of alcohol are taxed at very different effective rates per unit, without a clear rationale.\"\"',\n",
       " '(p) Alcohol is currently taxed in a complex banding system, with different rates for each type of drink.',\n",
       " 'After the last Budget, the amount of duty on a pint of beer was raised to about 41p per pint.',\n",
       " 'Duty on wine is about £1.69 per bottle and spirits are currently taxed at about £6.45 per bottle.',\n",
       " 'This means that beer and lager is taxed at about 18p a unit, compared with 19p a unit for wine and about 25p a unit for standard spirits.',\n",
       " '(p) A hybrid scheme combining tax and a minimum price could be developed to stop the sale of cheap drinks while protecting traditional pubs and responsible drinkers, it is claimed.',\n",
       " '(p) Rowena Mason\"']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAACb8AAABiCAIAAAD1ZaslAAAJMmlDQ1BkZWZhdWx0X3JnYi5pY2MAAEiJlZVnUJNZF8fv8zzphUASQodQQ5EqJYCUEFoo0quoQOidUEVsiLgCK4qINEWQRQEXXJUia0UUC4uCAhZ0gywCyrpxFVFBWXDfGZ33HT+8/5l7z2/+c+bec8/5cAEgiINlwct7YlK6wNvJjhkYFMwE3yiMn5bC8fR0A9/VuxEArcR7ut/P+a4IEZFp/OW4uLxy+SmCdACg7GXWzEpPWeGjy0wPj//CZ1dYsFzgMt9Y4eh/eexLzr8s+pLj681dfhUKABwp+hsO/4b/c++KVDiC9NioyGymT3JUelaYIJKZttIJHpfL9BQkR8UmRH5T8P+V/B2lR2anr0RucsomQWx0TDrzfw41MjA0BF9n8cbrS48hRv9/z2dFX73kegDYcwAg+7564ZUAdO4CQPrRV09tua+UfAA67vAzBJn/eqiVDQ0IgALoQAYoAlWgCXSBETADlsAWOAAX4AF8QRDYAPggBiQCAcgCuWAHKABFYB84CKpALWgATaAVnAad4Dy4Aq6D2+AuGAaPgRBMgpdABN6BBQiCsBAZokEykBKkDulARhAbsoYcIDfIGwqCQqFoKAnKgHKhnVARVApVQXVQE/QLdA66At2EBqGH0Dg0A/0NfYQRmATTYQVYA9aH2TAHdoV94fVwNJwK58D58F64Aq6HT8Id8BX4NjwMC+GX8BwCECLCQJQRXYSNcBEPJBiJQgTIVqQQKUfqkVakG+lD7iFCZBb5gMKgaCgmShdliXJG+aH4qFTUVlQxqgp1AtWB6kXdQ42jRKjPaDJaHq2DtkDz0IHoaHQWugBdjm5Et6OvoYfRk+h3GAyGgWFhzDDOmCBMHGYzphhzGNOGuYwZxExg5rBYrAxWB2uF9cCGYdOxBdhK7EnsJewQdhL7HkfEKeGMcI64YFwSLg9XjmvGXcQN4aZwC3hxvDreAu+Bj8BvwpfgG/Dd+Dv4SfwCQYLAIlgRfAlxhB2ECkIr4RphjPCGSCSqEM2JXsRY4nZiBfEU8QZxnPiBRCVpk7ikEFIGaS/pOOky6SHpDZlM1iDbkoPJ6eS95CbyVfJT8nsxmpieGE8sQmybWLVYh9iQ2CsKnqJO4VA2UHIo5ZQzlDuUWXG8uIY4VzxMfKt4tfg58VHxOQmahKGEh0SiRLFEs8RNiWkqlqpBdaBGUPOpx6hXqRM0hKZK49L4tJ20Bto12iQdQ2fRefQ4ehH9Z/oAXSRJlTSW9JfMlqyWvCApZCAMDQaPkcAoYZxmjDA+SilIcaQipfZItUoNSc1Ly0nbSkdKF0q3SQ9Lf5RhyjjIxMvsl+mUeSKLktWW9ZLNkj0ie012Vo4uZynHlyuUOy33SB6W15b3lt8sf0y+X35OQVHBSSFFoVLhqsKsIkPRVjFOsUzxouKMEk3JWilWqUzpktILpiSTw0xgVjB7mSJleWVn5QzlOuUB5QUVloqfSp5Km8oTVYIqWzVKtUy1R1WkpqTmrpar1qL2SB2vzlaPUT+k3qc+r8HSCNDYrdGpMc2SZvFYOawW1pgmWdNGM1WzXvO+FkaLrRWvdVjrrjasbaIdo12tfUcH1jHVidU5rDO4Cr3KfFXSqvpVo7okXY5upm6L7rgeQ89NL0+vU++Vvpp+sP5+/T79zwYmBgkGDQaPDamGLoZ5ht2GfxtpG/GNqo3uryavdly9bXXX6tfGOsaRxkeMH5jQTNxNdpv0mHwyNTMVmLaazpipmYWa1ZiNsulsT3Yx+4Y52tzOfJv5efMPFqYW6RanLf6y1LWMt2y2nF7DWhO5pmHNhJWKVZhVnZXQmmkdan3UWmijbBNmU2/zzFbVNsK20XaKo8WJ45zkvLIzsBPYtdvNcy24W7iX7RF7J/tC+wEHqoOfQ5XDU0cVx2jHFkeRk4nTZqfLzmhnV+f9zqM8BR6f18QTuZi5bHHpdSW5+rhWuT5z03YTuHW7w+4u7gfcx9aqr01a2+kBPHgeBzyeeLI8Uz1/9cJ4eXpVez33NvTO9e7zofls9Gn2eedr51vi+9hP0y/Dr8ef4h/i3+Q/H2AfUBogDNQP3BJ4O0g2KDaoKxgb7B/cGDy3zmHdwXWTISYhBSEj61nrs9ff3CC7IWHDhY2UjWEbz4SiQwNCm0MXwzzC6sPmwnnhNeEiPpd/iP8ywjaiLGIm0iqyNHIqyiqqNGo62ir6QPRMjE1MecxsLDe2KvZ1nHNcbdx8vEf88filhICEtkRcYmjiuSRqUnxSb7JicnbyYIpOSkGKMNUi9WCqSOAqaEyD0tandaXTlz/F/gzNjF0Z45nWmdWZ77P8s85kS2QnZfdv0t60Z9NUjmPOT5tRm/mbe3KVc3fkjm/hbKnbCm0N39qzTXVb/rbJ7U7bT+wg7Ijf8VueQV5p3tudATu78xXyt+dP7HLa1VIgViAoGN1tubv2B9QPsT8M7Fm9p3LP58KIwltFBkXlRYvF/OJbPxr+WPHj0t6ovQMlpiVH9mH2Je0b2W+z/0SpRGlO6cQB9wMdZcyywrK3BzcevFluXF57iHAo45Cwwq2iq1Ktcl/lYlVM1XC1XXVbjXzNnpr5wxGHh47YHmmtVagtqv14NPbogzqnuo56jfryY5hjmceeN/g39P3E/qmpUbaxqPHT8aTjwhPeJ3qbzJqamuWbS1rgloyWmZMhJ+/+bP9zV6tua10bo63oFDiVcerFL6G/jJx2Pd1zhn2m9az62Zp2WnthB9SxqUPUGdMp7ArqGjzncq6n27K7/Ve9X4+fVz5ffUHyQslFwsX8i0uXci7NXU65PHsl+spEz8aex1cDr97v9eoduOZ67cZ1x+tX+zh9l25Y3Th/0+LmuVvsW523TW939Jv0t/9m8lv7gOlAxx2zO113ze92D64ZvDhkM3Tlnv296/d5928Prx0eHPEbeTAaMip8EPFg+mHCw9ePMh8tPN4+hh4rfCL+pPyp/NP637V+bxOaCi+M24/3P/N59niCP/Hyj7Q/Fifzn5Ofl08pTTVNG02fn3Gcufti3YvJlykvF2YL/pT4s+aV5quzf9n+1S8KFE2+Frxe+rv4jcyb42+N3/bMec49fZf4bmG+8L3M+xMf2B/6PgZ8nFrIWsQuVnzS+tT92fXz2FLi0tI/QiyQvpTNDAsAAAAJcEhZcwAADdcAAA3XAUIom3gAAAAddEVYdFNvZnR3YXJlAEdQTCBHaG9zdHNjcmlwdCA5LjUyELw8aQAAIABJREFUeJzt3U9sW0me4PkQqf/yH9GZkjOrtlIy3VloyFgMylT2LhYLOBumDum8poy+TffBMtCYSwPdkm7Tc5Oq+jgzgFSHmbqKfdutygXIwlbeZsdibQ9m5d7uWjGlBqoyLWWJttOSLOsP9/BLRYVfxHsMko//pO8HhQKTph7f43sRL1784hfRU6lUFAAAAAAAAAAAAABceol27wAAAAAAAAAAAAAAdASipwAAAAAAAAAAAACgFNFTAAAAAAAAAAAAABBETwEAAAAAAAAAAABAKaKnAAAAAAAAAAAAACCIngIAAAAAAAAAAACAUkr1tnsHAAAAAACIUy6XW19ff/jwYSqVSqfT7d4dAAAAAEA3IfcUAAAAAHBxLCwslMvlxcXFQqGwsrLS7t0BAAAAAHSZnkql0u59AAAAAAAgHrOzs7lcTl4XCoVsNtve/QEAAAAAdBeipwAAAACAi6NYLK6srKRSqenp6dnZ2XbvDgAAAACgyxA9BQAAAABcQLL66fLycrt3BAAAAADQTVj3FAAAAABwcSwsLMiL2dnZcrnc3p0BAAAAAHSd3nbvAAAAAAAAsSkUChJALZfLMzMz7d4dAAAAAECXYeZeAAAAAMCFUi6Xi8ViNptt944AAAAAALoP0VMAAAAAAAAAAAAAUIp1TwEAAAAAAAAAAABAED0FAAAAAAAAAAAAAKWIngIAAAAAAAAAAACA6G33DgAAAAAAUKf/8x//8f/47//9/yqVtn//+/2jo6Pj49fHx8enpxWllFIJpZKJxFB//zsjI3908+b/8uGH/+uHH2bv3GnzTgMAAAAAOlhPpVJp9z4AAAAAAFqquLVV3t/X//nr7e3AB37z7Jl+fXR8/M2rV7/d21NKHZ2cHB4fH7x5I/90VqkcHR+fnT9Xvjk5kRfypHnWtP3vND36RU9PQv6XSPT09PQnk0qp3mQy0dOjlEr09Ny8dk0+eXVwsDeZVEr9j9///g/eecfe5vvXr78/Omq/nx4bS4+PN+UwAAAAAABETwEAAADAXyDoaPrnr79+dXTk/BN58er1633jA1+/fKlfvz4+Pj37Q6jx5eHhyelppVI5M57XTuTh7fwd/SjHEx2i9SiVTPxh1Z5kIjHQ25tIvLWOT6KnR0K5Sqn+ZHJkYEAp9d716zrWG+Ha0NCHN2/enZjw3yXSfwEAAAB0MqKnAAAAAJouIuho+ur5869evNB/UvXzz16+/Pr880qpiK94c3JyfHamMyPF6dnZGQ9E3SPR0zPQ2/vm5OTUOGt/cuvWx3/8x4P9/f/v73639uSJUurRvXuLDx5Idubqr3619ItffLm7+6MPPvjw5s3f7+//l81NHcO+OjjYn0x++/r1m9PTwHf1JZMn59P/ou1Sw8N9vd8tPNSXSOhAr+na0FDveUj4w5s3I7Y2Ojw8OjxsvnN7fDw9Nua/P6T/AgAAABcb0VMAAACgc3kGHU3/2z/8gxlQrOr5wcE/ff21+c7RycnZWeiUqwfHx0fHx8dWtAltlEwkTsNPWafp7+0dSCaVUt+6UnX/1Q9+MDIwkEwkjo6Pj05ODt68MecQvjsxkR4bS4+Nzdy5oyNYC2trq1988fzg4P7U1PLsbGZy0txgeX9/6ec/lw9IYFUpVdzeXv/yy+L29i+fPtWf/NEHH4xdvdrT0zPxzjubu7vl/X17QuPUyMi1wcHrw8M9Sv3+1auBvr7eRCJQgoCAgd7e965f1//Zl0xeHRyM+PzVwUH9+feuXx/s66v6FTM1pvNmJidTIyM1/QkAAABweRA9BQAAwAVUR9Axv7GhX78+PvYJQOqgzuHx8eH5MpBOp2dnb05O9o+O9iM/hoskNTzcl0wqpd6cnARCm9eGhn77/HnL9iSh1EBf3+HxcYPbSSYSgcmEA1IjI+9fv75/dCTBntfHx88PDl4cHtqfvD81pZTKTEwopW5cudKj1FfPn/f39ha3t4tbW88PDuRjo8PDmcnJzMSEpAba073q1NK7ExPLs7MR88GW9/cXcrmffvGFUmr+k08WP/1Uh46KW1vFra3NnZ3C06c6XKq/evrWrd5E4urgoFQRpd3d8sGBuZPmQb08PLz17rtKqe+nUr8tlyXuVTzfZml398vd3bA97EsmrwwOSsJrbyJhbx9oow9v3rwyMBD9mb5k8urQkLxOj42l3k7wjVZr+m9qZCQwTgIAAACIEdFTAAAA/EHBiCB6yof/ydY33zjfNzPJtOOzs29dURalVFgABhfeYF/f64ZjfoENfj+VktfXBgdfvn59cp5EO9jXd31oSCn1zFiOVFQqlX/Z22vkS4f6+5VSJ6en375+XeufT7zzzuGbN8fn8ddKpRJXXG2kvz/R06OUSiQSh8fHgWmNTXcnJiTWKFPdPj84ODo5uXntmpTl0u6uXUJvjY1JLERCpNO3bqWGhyVbtLSzU9rdzW9slHZ3S7u7Zn7n3YmJzORkanh45s6d6Ny4wsbGQi736+3tW2Njiw8ezH38sc8hl3Z25n72s18+fTo6PLz46acLDx44t1zc3pZgqg523hoby0xMSM6r3jE5kNLu7ubOjlKquL3tTFeVn0KCSTeuXMlMTJi5wsXt7b1Xr5RS5YOD0vnXmRmxtpvXrkl82iwgN0ZG/u9/+RefH0EuSEkff1XjBdmXTJJ3jm50ZWAgMJ/ztaEh5xTQWqaWpXwV6b8AAAAXC9FTAACAGMQbdDR9/eKF7hx3Bh0D9oyEy69fvDgKD4cAPpKJRH8yqVcTFEcnJ/Y6kY3QwTnt5eGhxDXN/u5APH7y3XeVq1y8OjryKSxhJDR1fHpqJmteGxzc/v3va93Ue9evjwwMnIT8Vv4l1DOKPH716kBfnwREtejd1r+8ThQb7O8f6e9XSv3m2bMbIyM6pOeM58mf67+V4IEdEpB45N6rV4GZciWsKKmlmclJz0yy4tbWQi4nEdC5e/eWHz70+avA/ujI6/Ls7OxHH4V9sry/X9zakkBvcXtbB1Mlynt7fDwzMWEnvMpfqfN6XnJPw9JV1XmQxvnr6TR6HaPVGwzbpjYyMPDDmzeVUsdnZ8PnU7+mRkYO37z59vVrSROMTofV9MSzL1+/VkpdO591ttZCcXVwUHdA1Bq4xSXUm0j09PSwArHQQ1I81Zr+K2M7/D9P+i8AALjYiJ4CAIB2qjXoWD44WP/yy1r/ZOO3v42eVTXgH7/6Kt50N6BB41ev6teBaWB7e3vrjmRUJaEdmyQRmu/oMP/3RkffHx2VNyWApI0OD48ODxetzDzP+E2YH33wgZlC6lRH+vLo8LBMwSqOz84SPT2Dvb3yny8OD18fH984j3J55vwppXRgzGdm6T9+7z1JWj08PtZn+dnLl9eGhnoTiegv1edOd6CbPePpsTGZgVZqVHntPBEyga06j/DJ7JrRnebFra3S7u76l18GIo6yV+mxMQk61pF3VdrZWfrFL5yz79ZBz/p7f2pq8cGDiFl/zR3QC6aakcv7U1OeMWCJhkooVH5zZ7qqGZb2+c11vFYZQ3N05Nv5FfbXHZ+eXhkY0LOzXhsaenl4qJR67/r1w+PjknEeo1NjtVvvvqvneb4+PDzY26sTu6Xs/ObZs1eutW/D3Lx2LZlI6GENQ319fclkeX//ZV2B2KG+vmQy2d5VnPsSieP4VixO9PRETKyNuPT09Az09iZ6epJvD1U5OTs7PTuLd2hRHa4ODkpO7eGbN53Wmg1rVISpNf1XJjnw/7xePBsAAMBG9BQAgK7UmqCj2VV6fHoaNquq9s/Pnu3X0g2KS6U3kUj09LS9V7FuqZGRGyMjuhQkE4n+80CaikwNjH3a4Yjsk4h+xkCXYmCuVBVeSzQj2Gn+5+HxcaVSuX6+VJ46D0wO9fcP9fWpehNJzS7a49PTSqXy/vXr8p/PXr48fPMmNTIiX6o3fuPKFf9Duz40NDo8rGM2/cmkPd9vxF7tvXolObWyM++Pjg709sqeXBsa+uKf/slnI/p06/UCAwE2M1CnIjMgb53PKyvbMWfZrXpQEreTqW5Lu7uB1FKZ7Xb61q302FgjWUrl/f2ln//8x59/rpR6dO/e4oMHsXR5y2ZXv/ji+cFBHZvVC6aaObXmgqmZiQnPDXqmqwaC2WHJvtH7XHcyq/52ZVx+vcmkFKUrg4O9iYTepnq79ojesvbe9eu6qJops0qp90dH/+V8gIgkznpuU9MzHn+3/dPTPmPi1r5k8usXL2qK5mr69z89OzurVA6OjhqJYvYmEicNh1T1hOGq3jnDw/QlkwPnP2ODR4p4yWLbEeMFe3t7+xIJfWG0fYaS3kTiqnH3t9W6gH2z1Zr+aw81i1Zr+q9SymfcDwAAqA/RUwDA5dL6oKMpLABJ0FEpNdTfn1Tq4M2b2FIwLrFkT8/wedeYOHjz5rT5rT7plW5qosPt8fGR/v7k27PIHh4fS1/hzWvXAp+XOJy87ksmr55n76nac56q8k/T1CK6yQKBK7vuctZOzvqn1gCDyQyWKKVkdcYrg4MS3nh9fPz1ixfyTxKMfHV09NXz5/rzVQOBTnbvpP6VJMP16tCQzEeqk+1Gh4crStWRG6feDqgcvnkj3cqvj49riobqnZS5hTOTk7959kzy9q4PD3vm//mHRbVAyC0il1Fv31x9U9W18J4ZMgzEmyUFUzYeY5fuwtqaBDjvT00tz87GPldkeX9/IZdrPKVVL5ha3NrSp8C5YGpN7HRVZ5i/1nTVaLEks6qQ1Ofb4+NmkNWcx17SoPU+RH+LJtf24Zs3h2/e6Lm+Xx8fvzg8NO8Lv387ElNr3SiRXX3H0Sslq/PFknUyet2Th8uwDHOdYx3ranBqAZ2PGEs4ynkZx7gks3o7+ivqW6nXR0IpGp/N05dMOvNBv0uQPW+kXTNaaE6xz65Rq0DuewRJi1dNGD/XOHuhhGik/wIALjOipwAAL+0NOgZI970i6GgZ6u/vfXsOMaXUydmZz3N+U13gqeTsOKVSKpFIJM7De60ZNa97RVvwpYHIli6PQsfVlLVEpTBz7GzHp6f/z29/G9/OxpOmafIJCJlRB81M8zLfLFt9zf5hP1sg2CkkevH84MDs1/7he+99/eLFy7c79QKhhTpyTO0dsNddk2w5HRSx89XqC5/IEo86sHF9eLhHKX3IRycnOtAbwY6Giv/59m190VbOL3vPSJIzXU9fYz7xrdLOjjnLruTzxTXLblV64c/ywYEZFFRK3Z2YSI+NSWqpf5JlTfTkuncnJpZnZ5uaZFPa2Zn72c9kOdXFTz9dePCgka2FLZh6a2wsOzUVtmBqHV8RuDDsCkSuCj2MQwpgjL+kbiLWkcxqVtFVS0fh7SBrWJKr76zCxlfLj6MrSfnSb1+//uevv5b5xu1vUbXXkGZarVLqyuDg0dvNs8l33/36xQudJHptaKg3maw713/inXfM/3z5+rUdnWowHBWIwcQyT6xeYdcMISsjtOY5tKWmrwusch34xmYE7ZI9PUPnU2STv9sut8fHT8/ODs6fJYes9rxyDZJwkun09X9+9eKFT5OjeXQ5stnznEfHrdueoBzgbOhGqDX9V53fJf2R/gsAFxjRUwBoro4KOmqN5CFdeP3J5IAxG2dPT0/SCEpp3x4eNj6pWiMCQ/I7bWKrJtFLPyYTidchT/Iy3DvezrUIYcO3m73WVPTCUbfHxsLK+PODg6OTEztHUymlswZvXLkS+KfYK42I/Q8LavqnadaqjcFOZ3DXPszXx8fXhobMxNmvnj//3fPnk8aqnIF98w86mgLnxe5y0qmQmiyfqf+zuL1tBtTrmLdTKfU/3Lgxdn4Rmut9Hp2cHJ2cyHS+Pl2TZhebeSzmtfTt69f6h80bQSD5MaPjFtFh0VovS2kwBGbZdV5dgVl2pZsvxuyNwsaG7EZgaU89Ia1cBs3uLixsbCzkcr/e3r41Nrb44MHcxx839euc37s8Ozv70UexbLa8v194+tReMPXuxMR3wdRqC6bWRNJVpTxWTVc1Y+3NSwOSQQDy2i5r/pnZzmTWsKRe80sDbWyzzvSMUAZ67c26+vb4+NjVqzLG4srgoCTH599+CgjU0rWGRfX9QlJszbFKMg2AeUdQSj17+bL37RZsfXerkYGBW2Nj5tQpOp5kNrQk77bupFsx0Nv7zpUrgUCmZN/GEsIMTN2slDo+PT05PXWGzWIMmtqhLB2vdQaumhGvdc6R++r16+Yt+pvo6RkeGGjvusLdSNaJsOdZUUqZS7w7W/La0cnJ84ODwJwrYfwXbm8Su2CGCQyMixZLpdQMzU7/tdvq0RocYwcAFxvRUwCdyNmRXeVPLnHQUaJozQ7V1KQvmXz3yhVZETAwctypxeGuCGFPMvGuF+VP93S0/vyGPceaJ1RPS2VqzRRVMrWdcg2g1q4NDjZ1jq/opY+cI51l0lF5/cP33jN7NAKVXkSyi1ZfeCxCxBHZ6YNaRJpmfXNUVmX2hmuBoJ1+M/BOgz+aZ7BTvT1uXb5R96fbNyz7flRHH3fEbLfCDj9HTA4cEWaoafck4PHy8PDk9PT90dErAwOyzKdS6v3R0fL+vlStfcnk7qtXVQMJPtFQ3QfknHfUMywanQ9Xd3gpMMuu7Ez0LLvy7foAmxGwlAIlKZKl3d1AaqnkLM7cudPKufWKW1sLuZzkgM7du7f88GFrvtekc17vT00tPngQ+y9f2tkpPH0aWDBVnU993Lxc3g5JV43WsmRWp8BjiHlzCdTVnjWh2VEeuJmad4rAbMaqacFXISfXbJbIL6OnHNftEPsOVXc67B+Nj+uWj8wTcH14eNAYs2gOkdF2Xr78rTEhfK10i1EZjUazBRtLw1UmP9izxjJKiEtCWeb7epLnGANXIwMD71qj3/Ru2PugjBUNmhE/608mR8JDd56LhdfNbH8en5zEu1KD01BfXzKZvMDR4vtTU9Kci/iMtPTU2ys72AJDPZzRTVncOvBm3ZVPXKSkB960C74Ii3Cbq4qIof7+ofD4cexLjTSo1vRfVXsAuNb03yY9cgKAInoKXDAdG3RsJC+neezVdNoVIQsTmHHLZEeM9KpIAW2fbMf+nZ3alcrZCZfBH42PD/b3O9dD1Qb7+qpGT1tzru2oaiBAHrgUmx1MjU7BjH5Uk4DEt69fhw1JHh0etrMPRURN2OFpmq0fXywpUME3WxLsdA7urhrsFPJD2ZFae88bnMtR1TLbbWD3nFuL6PoP7K3/rga63b8/OqprJHM2yK9fvDg5O5MawKcg2DNnymtnNPQPu31+Usw2TK1hUfPr9G8bS+eLXPCB6JTz1wjMsisBnhaUUFmbc+/Vq0DczkwtjTcP0l9pZ2fpF79ofP3RWJT395d+/nNZb/XRvXuLDx40L35c3Nr6LphqzI2sz0jdC6bWug86XVUKVNV0Vblo27VqXXQya9X6zV5m2Kx5ag0Vm3e6iFmFPVsIgSrXvG2ZI5bM6sJ+DIw3+Grfp8y9sidIeP/6dXsefvsuX3eT6aNbtwJRnA9v3tSjc77bh9HRgd5eezLkRr5XOHNY7QGFKqYsUjPqrM5DNeaMstqzly+dYZh4oy8/+uCDwM4IOxauf+QbVvXVjHitmeMbPT9tU4dXBlbT0FrwtBvIMD49O3vZqlVX5atbP943Yuin2fr90Ap5mqTqSI2M2GXK+WHn+bW1vRMsUHWEOTw+rlQqPscuJt991zOdd6i/f6i/P/bxvo2LHgNtixhG7ET6L3CRED0FlCLoWKOqwbCOmkHUfx4Yedj2ifN1QoqkGViNyL1TTV6zx0fYA6RqR5wysOKUensGJGF3RjhPt7m8TWvmO4qIpiulXr5+7cwEFZ2WgmkKy1z856+/fnV09P716++Pjgb+KR8+ITZpmo1zBjudv7n9azc4JNwz2OmMDdvd9/bN3Z6Mt12z3Xr2ywfORf7t3NA6UqDU2z+y3lVZofZ7o6Pvj47qxKPJd981f59ao6FmQTB/gYhAizMsWmvCWfPCoibJjTOnRVUhZ0F+cP1ryC61sryXdnaK29vrX34ZWH1TKXV/aio9NiYLcLa9CpJQ5Y8//1wp1exQZU3K+/sLuVyLA7oS4ZZpfs0FU3VaaitXWXMuu2tf7eaE0k3Nlq6DTmY1R5PouiW6/nTWLY1noivrJpV/e1XXOtZ+Dlu2OXDTdJ6UqsHXwE2n1ttldPBVGT+p3fxzTqQfaIHU3cwzb1uSV2dOiayU2vrmm6H+/kAO2W+ePbPXOGjwyf1HH3yg1+3W5HvNJxGdLRrLfKTXhoZuu1qz74+O7h8dBXZGnacMBr5X5wjGO3zQjNGaTisVM0Zrevfq1a9cacoSwmlG14qZ2azpZ3PntMxNHe0a8dytlOpLJI5OTlowRVC7ljUxmT9F6/scPLMzdeNQr7rtZG5KFuS2PzPY3/+elerqXHDEqe1JvYFfzB5ia7Jj4fIgYwuLiJujajS7Vtfa/vvYokeQ22pN/43otXBq10A6oAWInqIhdQQdnQ8/Vf/E85avdWnQ0dT2JRVNgTFr9nNdgPl4WTXK2NT5gmrifPhRxrI0mjPLs10hVef6QE4tjmrbccoAGaocsWSLHrAcNpVNa1YxMefniZ6Wx56ERxvq739+cFB1ib5GNJ6CGfEB3dEWUe1HjClpZZpmxEIyEccYNjtQV4wDdS6u3Jpgp33VOYPlzuG3VR+xCla/bWfOduup8Ha3uNkWqq+XPNDLEIjZv2M8/2/u7Dhngfb5uRqMhmo6DFxHWFQX6poWO2ycDuWas+yGFRlzll3947QlbiS1dHF7e3Nnp7S7a55lOZsSfkuPjXVU5bawtiYpnvenppZnZztq30RpZ2fuZz+TyYQXP/104cGDln21nNP8xoa9YGpmcnJ6crJdicLq7WEEEWXELCDtTVeN1kheu4o7mTVMxN2k1hpehU8AoN5uGvmcr8BdOzDtQcuCr+YO2w0ku3VkdzXU3TSyG0WZiYlAn/73RkevDA6awQ89Q3Ls028Ephh1ZqyGrY4ZS66qvn0fn56aU+yYkQ+dLmwHOeQpT6cLa/E+O8hOylrC5vtmED1wEiWnWV6/OjrS8Vp7ptmWxWu1iMmKhvr7T8/OmvfsHJ34q1qY+yuqdr4le3q+iQzRxci56rBqR6efz5KrVYc+m2Muozf1m2fP7LzzMGFjoG1t7++tmrqqq5TAcJwwzsXLI4wOD292eWZOs9N/q3Zw2TpkwB86ENHT9iDoWBPnUiXRpGXW9glLTdGRpOOzs8CknREhJWeExtko+erFi6ZGa6oy29AR5LQ6x2aa2nVCA48oYRehGVJtcZan2aFgP/tp8qQavQaJPLg6x+KZnh8etmDwnbPUmMH7sBmrzP80M0RVk4P0TUrBFNH9Vs5kwe/+yTU/qv6nsA3GezuIGH4b0Q6OmPGm7TlSjbD78pyBZ2fIucEOo0aCnTX95t04262nwKFFTIrrf7IiZmjU50L6Tz+8efPq4KDZ+VtrX7nZaRI2FWRNUQ1d+TiXJ4zepeiwaGueYGX/9c5HzLKrT1OnRYCKW1ul3V1JUgyUI1lBU37Vju0R0MuL3p2YWJ6d7dj9FIWNjYVc7tfb27fGxpZnZ2c/+qj1+6CTiVu8YGpNLkC6arTOTGYNY965Ita09rwRB26+znuWqv2G21HB18DO260aZ8vNblfX15y299a8VPSjnPmZr54//93z5+9dv25OueRsSTY4bM75dDYyMPDGekhPjYzsWAOLrw0Nffv6deOTwYQ9WQz09sqimwHfGx39nZWTKrl6YU9D8aaa2SEreWYPi6xsffONGaM1HZ+eRvSJNaNDT590e8YmFTluPjAAuqmP4dEryzjnZ2r24jJC5o6yh+Nr5uIXLevsin6m69hUXZ9gWCBm9tXz51+5+r7s2a1qmlzQv7O9E2Yt9s9SNX+6sHRecXVoKLAAsJ67yOaMCtc6NSPpv1V1wmPpRdVl0VOCjjUJG5UW0diyw1etaVJ4kntq9BwOprAZLJ2evXwZiLKEaXuKpM4ErZoDqmNLPjmRN0ZG2nVoYZPrBq5VZ6u3xSHViIadzPUkr53ragTmY7HngFLWPEjKGm8be6Ke053vfa8/ZEIkETFjkjoP8IcNM2x2UL81KZg+Yk/TjL3JeAnTNGvlPImtCXY6axtnsNnZsK47wHwBZrv1VHh7Ftyw3mRVSzPPPKjAyTJPUzKROD3v0Al8dTOiofVdDI2ERfXv0MawqKYvaXOW3bCL1pxlV+9zRw3X0GmI5YMDc4FMdT6ta3psrEPiZ1WZkcjFBw/mPv643XvkS0d8709NLT540N5oX3Frq7i1tb615VwwdfrWrezUVOdcwOq87jVTup33SjtdtaNKoqfo2cjrSGY174xNvfAi1s+ub954824VuD+aDcu6z3JTg6/KarQE2smBFktEk9ge72gPL7OblHU/AthNfbsleXt8fKivzwwwXBkc/OHNm87xl86+sgafT2UnzUdpZTxNmz34OqBoZnyaD8uxPCmHDYfVJ93cJTP9TmLY8joQyW5XjFZIG/vrFy/sIfg/fO+9q4ODzslgdcAjooO0SZEhO0hvdnw50wykvytswqp41/0NiM4GPnzzJiIFtjWzl9m9bYF+NrsfuJXTql0fGhoZGIieSk16Alu/EJVPeKzqEHnhEwn79fb2Xb8AW02RDv+k3tZ0NkbwT0it+rObSSNS0UVvUKpBPRuED/8fVnRa5Mhz1ILmeambwnoOw3TsAMqao6elnZ2VX/3K/U8XJehotjn+v5B5LarebEydsESiqabxET7F4/nBwT+cR93MIXI+s4LIzPJtH0XiX2vsvXplB/DCpusMtN5aM8uozV5CwJ591Cf7s/Wpnz7Xqs8EDvYAtP/9v/038wPyAGbPC2RqTX1V9ZDDYlpb33wjD3I6KBtxsd24cqWph9PGFMwYFTY2whK8AkjT7BZmG8bZaGnkVDbueZgPAAAgAElEQVQY7Gx2S1Ffz901223d9PGaJde/sRGRDBo4tLDj0heb3gHPDqawVesaj4aGWVhbM68Kz7Co3ZXfOeMnFtbWImYQ1UXVnGW3c3Y+zOqvfrX25InZqaHDY3IIHfu0GSb7k5/ILLhz9+4tP3zY7t2pmazSKrMNP7p3b/XP/7zde/QdWTB1c2en8PRpYMHUxU8/7eTrXAKNZiK4s9qU+jk9NjY9OdlFEfdoZqhSh6z8a2bdfSFt7Jk7d9pVIZihwYhZhT37Z5334scff9yM5kQg+Gp3TAceBGptO4UFX+uu/ZzD++x5ie3nl7o7x53zEssL50lxzovjjMg6H7IafLwK7K18qY7IZiYn705MhIUfnM8IcYUJnQ/75vOCLr/OhTlUZNQkrEs23hhnxJO+2XB9/PHH5YODsLmRovP8OiFeazI7AFMjI//qBz8w/zW6J7ypXZ0yO3fY0kLKWjj5+6mUM5TbgphZxKR0kphrr411Y2REL0TayqiepAsLZ+pRYFdbn3HkGWK0ew5jbB7YUzJEcK4o5N7sJUjq1W6Pj0e3YyOml3OqNXuw1vTfTvjNA3ymIjeFdadXLRo1R08LGxszf/d3zn+qdadV7WnLdcxb3WBvXeZv/zZiZLrnRmqNz0d0gju1vWe8vL9/49/8G/t9/1+ppp/If/BCM/pqc0+ePPyP/zHwpufdy/8wPTP06ztA5yHEFadUfieovVMK2FesT/i8auVTteQ2tXN2YW3tx59/br7TOSmYXSpQUuJN01SX4zfsNGbZd9bbzlNpn8TOD7TY9PXcltluW08fr1lyA3fhQKUde2KrNJjDoqHmj9ze2qDnL/5CdUlY1FPPX/yFOR2o6qRZduuW/clPyvv7EjSauXOnq49FzP6H/5AeG1v89NOuHt9T3t9fyOVSw8OdGQDWmcql3d3C06e5v/zLLm17ONNVuzTu3qDoNaSlm37tL/+yLXNK1ycQBcy/vaqrvSj43r//9x1YaQS6OD2Dr3cnJop/+7ct2cEodv+sswc2cAhmL2rLToozoOjsl49eRHb+k0/iqj3CQghh0QJnqNiMDLW4/Ib1zkdENz1jtPm//uvW33Eigg1xxWvvT00V/uZvGtxPU1iYXEQswaOqBT/sUEfdJ6VqFKdq2KZqnMYOM9daFnxCTT5hvKoJhfYP61OZ++ye5+zBnkEvewxKjFVfd4kuZaZWJvVe+NNRx/Sx/oF2UWv6b9g4jKrnostm7gUAAAAAAAAAAACAJqm+xCMAAAAAAAAAAAAAXAZETwEAAAAAAAAAAABAKaKnAAAAAAAAAAAAACB69atCoaCUSqVSmUwm4g9KpVKpVJLXmUwmlUoppcrlcrFYND+WzWYb37lisVgul9PpdDqd1q9LpdLKykoulwt8uFAoLC0tyVHoXU2lUrKHEQqFgnODzk/m83ml1PLycvSbzgOR30T2MJ1Ol8tl+03ZbfkrOfCqe9V6zsNRHbznnleCqnY2V1dXNzc3nf+kfwd9rco36t9ElyxnCaqPfVz+F7Nzay0r2v6b9SlfzeD5S16won3J2Wezt7f35ORE/pWz1naN1G9hG1R+9bb8q7zZYL0dJuL+EsFu+dSxwVhubc7KsJEiE7jB1XT2PW/65mk1jzHGo2iZ2EuHswWurBuZ2XKo+gRRh/ra9uZe6TPo/2Y38i+qqmkVmn9j2xbWLMzlcuvr6zMzM7E0PgPqbmHWVNx8vqWm03dpdd3jZwvYha6RYmjXh82u4UWgERLRsIlWU4l2HlprjtdH1RtWjLt3MVrXnXwDqq+FL9p1d2hkn6N5HlFnNsir/iz6mjHPi/PNRsTV49fiFnJ9dXtr6pNSqbS0tPT48eNmV/uddusJPAs7uzHbXu4uDNqx3e4PuaepVCqfzy8tLemTF2ZhYUG/kKZGsViUF/JPa2trseyc3qyQzWazWV3CTZlMJnAnW1lZCbSEnMI26Pzk8vKy3bqy37TNzMzoO9za2prU5s439c8bb0M2Xt21555Xgqp2Nufm5sL+sFQqmZf9ysqKvNC/SaFQ0L+JXYLqYx+X/8Xs1PqiXXWznuUrdv6/5AUr2pdc4MRdv36ds9Y5GqzfbP71tn5nbW2tahupPhH3lwh2y6eODcZ1a3PWe3UL3OBqOvv+N31lHIuI9yhaJvbS4WyBK9eNzHkxxKXutr2z6vZ/s+vU1wqNt0KrqdwFOJuFCwsL5XJ5cXGxWCyurq7GtZ9a3S3Mmoqbz7fUdPouM1rXAXaha6QYqpbX8CLQCIlo2ESrqUTLxSM1j77RO99sl+gbVoyn42K0rjv5BjQ3N1f3vrXr7lDfU4kP/yPqwAZ59M+ir5lSqaQj7s43GxRXj1+LW8j11e2tqU/S6XQqlYq3JnTqzFuP+SzcgeXuIqEd29X+kHuayWRWVlaWl5dzuVygL8kkNYtU/dlsdmFhIXNOhoFks9m4AuYy5lG/SKVSesuyh+VyeXl5OZVK6cGGeqRGoVCQ2H5gEGKhUNCX6eLioq4Owjao4kh6y2Qy9+/fz+fz2Ww2m83m83kZVGK/qX9DpZS8Pzs72+C3x6679jzsSigWi/pObF4Jzi3INXPjxo2wz8zNzeXzebk+ZcyIXIrmbzI7Ozs7O+ssQTEel7IuZhV+2ZtaWbSdm/XZSX3s9idlWF+5XJ6Zmcnn848fPy4Wi/l8fnFxcW1tLaJor66urq+v6697+PChPiP2L2kfSIuLdrFYXFpaWl5ernou/D8JYZ+4H/3oR51ZrV1mzvotUKjN6n16ejrsrPnX25lMJpvNyiNNvANCnfcXZxUnE2/ozywvL9stn5o2GO+tzVnvNfKzOG9wPmc/4uYYEDgWFVKl130UTbK8vLy5ubm4uJhOp2X4+ePHj+UQqt79y+XywsLC9PT03t5exK1NhbfA7VNvvxPXkfq37e2dt6vuwJvOJlkslbxdVJXV5lxaWlJKyQh6+X999dpF1adC8y+q8nvGXqE12Ni2m4XFYvH27dvSU7mwsCD9j3WXR//HOp/TZ3Z2mOXI/8kioKbTd2l11+Onj7AHkNXVVefzS6BysAudsxiatwn5Rrl92PvjrCTruAjD9t++WTubK3bDxlkqG+yrMW/9+tdwvumjVCo57612nRBxBzf53LDirRO6unXd4TcgoftXdUn36fpo8d3BWSrtn9F5wXv25HgeUS6Xa3aD3L7CwwqyzxNTsVi8ceOGXDM6yOp8My6x9Pi1rIVs1+2dWZ/kcrm1tTWplqMfn1VkcQjbfry3HuVdPMP+3H4W7vwH4e518dqxl07l3Obm5srKSqVSefToUSXS/fv3na/t/2zQ3t7e/Pz80tLS/Pz8+vr6/Py8/pb19fVKpbK+vr60tBT27fPz8/l83nxnc3NTb8R8HbbBSqWytrYW2IjzGKse+P379/U3mt/rfDOfz+fz+fn5+bW1tejNtkt37bl9Jezt7X322Wf6deCaN8/m5uam/td8Ph9xoldWVuRblpaWNjc39abm5+fn5+flx7G330iRsY+r4rqYwy57W8uKtr3Z6J0MnBH7k+ZpkhpD/6G83tvbC3y1WbRHR0f1KdMXRkS1ENi3VhbttbU12XiMn4Rmn7iOrdYup+hSqQv1o0ePpMivr69HnzX/etu+UzTOeX8Jqwz1QVWMWqXiccOqWmfGeGuz67262Tc4z7Pv/NsI9s0uxqNoBmkbVyoVabRHNGWdp97z1hbRAs/n8ysrK5999plsx3kxxKhq2z7sr+yq2975sE82wi6qdptTJvupnDc5dKmMLvvRFZp/Ua00p0KrtbHtpE/3/Py82YCxH8fqU/Wxzuf0VVzlyP/Jwqmm03dpddfjpw/nA0jFen4JqxzsQuesIefn5+3tOMVVw1d9/opuhFQtlfYGw/42mlw5Pm9W5VknhN3BnRt03rCaUSdcgNZ1h9+ARkdH5RfWe+XfP9Oyu4OzVIb9jIFrxv9w/I+oZQ1y8/w629I+T0wxViY+Guzxa1cL2fz1Oq0+kWN89OiRfEXY47PzR/Z8ttLiulo8i2fV7QQ6gTv8QbirXbx27KXyh9zTlZUVGUlRKpUKhYI9CM6pqYMRUufp848fPzaHHKrzURKZTKamqURLpZKMu5f/NHPzAxtcWFgolUoy4v7x48cNH4pS5wNJAhN3ON+UgTlmDlwH6t49V0oVi8WHDx/KaxnuEfbJUqmkP5nNZiOut9nZ2aWlpWw2u7e3Zw4a0mPZ5ubm7GLVjBIUuJgjLvsILR5n5L+Tzk+ap2l2dtYeoawPx1m0M5mMPmUyFljOlGc908qi7T+8lDSF+tgnrluqtUvCLpV2oV5cXFxYWNDDQiO25l9vLy0tyab8G0hVOe8vYZWheasKS7Pw32CTbm3Oei9GPme/cc0+igalzqdbWFlZCVTynnd/n1tbRAtc6NHfqlo7p10iqm5z56M/WQe7qDrbnPJdZpkKO1+eFVpNRbUZFZrNv7Ftu3HjhtkU1Cvv1se/ovA8fcoqR40crKr3IeIS6urHT1vYA4h6+/mlvkc57fHjxzIbzcrKSnSjSGu8ho9+/vJvhKiQ9k8jd//l5WV7ojXnmzXxqRMi7uBhAjesJtUJF6l1LTrnBqTO01tlT/TyjZ6FumV3B2epjPgZzWumpjrK/4ia2iAPq0PstrR/ZdVKjfT4tauFbOrA+mRlZWVmZkYvRel8fPaMJjjFfuvxLJ61brbDH4S73QVrx14qveZ/yCksl8tSPVX942Kx2Ow5IaUKSKfTjU+Xoc5nKqg6u4tM1iEfi3fi6eXl5bm5ucCB2G82Y4X2ZujePU+n0+ZDS8TM+KlUam1tTYqDuaC385NKqdXV1enpaec3ZjIZaaLpN1tQgpT3ZW9qzY6Z/HfS+cl0Oq1rrYgyG1a0i+dL0Cul6luUomVFu1wu53I5n7lf/D+JgMCJ65Zq7XJyFupcLidrFJXL5bm5uYg6wbPelm+RWtGzgeTDeX8JqwzNJ7TAraSODTbv1uasDJukxa21MPLJqhWF58c8ra+vSw0/MzMT9pk67v6msBa4XDaFQmFhYcHcuLOd00bOA3fufLyVvF1UPducYefLs0LzL6q5XK4ZFZrzSz0b2zYZCac7EdbX1+tuz9RUUTRy+uo+WFXXQ0SEtlRKLdO9j582zweQBitz3VV3+/btqve1ZtTwdhn0b4QoV6ls8O6/t7ent6YrGeebjYioE3zu4Cr8XKjm33C7unWtv6UTbkD6282mlBy+Z6GO9+4Q/UXOh4hGGjBhX6Q8jkj+s0mPFf51iGdllclkFhYW9DWcy+VkLln7zTgPw1DTKWhXC9nUgfWJLE+7uro6NzcX1uXYyL049ltPgy3PCK18nL+ELlI7tus00p3+XfR0bm6uXC7LGI1cLlcoFKTWsDdRKBRkKm35Tz1ORGbcln+qaa2XaMViUYYC6Q3KDsjuLSwsFItFvRiGfLtepUAGWkoYX95Mp9PyQrZ248aNhYUFe4N/9md/JlPKqPP2uqyFIEcd+Bbnm84fTb7i4cOHstxRxJsRm+oEXbfn0VdCqVTSl7F9NmXAjl5XIJVKRbR7Hj58ODc3pweSmIVF/laGFTtLUCzH5Swdzsve3lrri7a52bCdtM+I85PpdHpmZkbelFn+1fmIQtnC48ePpQtMZvxQbxdteV9+Kz1gzflLOn+0Vhbt1dXVxcVFPYI1lk9C2Ceuk6u1SyisVNqFOp/PSyVQLpf1kMwwPvW2LEHRYFKCLez+4qwMp6enzR3QI1V9blj2Bpt0awvUe40I3OBkbLjP2U+lUvbN0fkVq6urm5ubciwzMzOywk2tR1Eul3/605/mz1dOavBjnm7fvr2+vi6/ie7s8Ln7S/+CfGx1ddXsuLeFtcClV0XWa/mrv/or+2KI5RhVSAPb5+Q6q25751dXV+U3jLeSt4uq3eaUx5ZsNlsul6VkyUlxln3/Cs2nqCqlFhYWmlGh+Te2nZzNQr3zjWSWh1UUzgvM5/RVLW7RTxZh++l5+qpqS6XUAl33+OnDfgBRIc8vzsrBLnRhNeTjx49nZ2ejU0niquF9nr+ko9BuhPiXSrtQyyXheT3oDZofc77pw7w49b01rE6w7+BhGwy7YalYb7gXo3XdsTcgpZSEiHST8vbt20opz/4ZEdfdIVrVhwj9M9oXfCqV8j8cnyOSOGWMjxUBzitcLv5AQfZ8YpKnD1kuVN6RdaPtNxvf+Vh6/FrcQrbr9o6qTyT4Kt2JmUxGVqd2djlGRxOqPlvFe+tx1nI17U/gWVh+iuaVu0vuQrZju05D3entnjq4ufb29uzZw51v2qRCb85+odUauRIqlcrm5qZeAKCj+B+C/yfbKJbD2dzcjJ7f3y7aTVrVtUn8L8XOvGiB2NmFuitqPOG8v4Ttv0/LxH+DHXtrE4201lp2AaysrPjcQTw/1gxdVBY8dcUR1V0pNdheba/Ydz7GA6/psS7e09dinV8pQavpFHiWr7DbvSy02UZ2saqpEWL/eSN9NZubm+bClhFvNqgT6oS6dVfrupNvQKIrfs+ankrq+1jn8K9D/Csr5zZb1rHcXaeg8/c29lqlGbeezv8ZgY5Sd3d6T6VSaXp4FwCaTw9qW11d9c+RXV1dXVpamp2dZcIEAEBNCoWCJNbE8jEAaBCVUrdozQNIoVDI5/PFYjGTyfCkAwAAANSK6CkAAAAAAAAAAAAAKKVUot07AAAAAAAAAAAAAAAdgegpAAAAAAAAAAAAAChF9BQAAAAAAAAAAAAARG+7dwAAgA5S3t8vPH1a2Ngo7e7enZiYuXMne+dOu3cKAAAAAAAAQFcqbm0Vnj79L5ub5f39/ymdpr+xK/RUKpV27wO6W2FjY+bv/i7/139NgQfQvQobG/mNjeL29i+fPrX/9f7UVIZIKgAAQAfI/uQnSqnC3/xNu3cEAAAACFXY2Chub69/+WXh6dPnBwdKqWRPTzKReHN6Kh+4OzGRnZqavnUrMzGRHh9v687CgdxTAMAlJcO+zEbMrbGx+U8+mb5169F/+k8zd+6s/vmf6w/8+PPPf/z554pIKgAAAAAAAACLnZ5xa2xMQqTZqam5//yfUyMjq//6X+uoqnQ26o/dHh/PTk1lJifbdgAwED0FAFwi5f393JMn61tbhadPv9zdVUqNDg/PfvTR9ORkdmpKxnnlnjx5cXg4c+dOamRk9qOPZj/6SJ3P6EskFQAAAAAAAIB6u8Pw19vb8uatsbFH9+6ZnY3i19vb8598kh4fT4+PS3+jMgKuuSdPJLtjdHg4MzkpXY6ZycnUyEjrjwuK6CkA4DLIPXkSaMd8Nj39+OOPneO58hsbSindiBFEUgEAAAAAAIBLrrSzU3j6dH1rq7i1pXsa705MyIR22akpZ7yztLPj3FrW6Essbm0Vt7Yk6+OX55mpMsGvpKUywW8rET0FAFxMxa2ttf/6X825MqQdUzXAmXvy5LPp6YiBXURSAQAAAAAAgEtC1v/a3NnRs9kppe5PTUlPo0+GaGl3Vyk1E9lVmJmczExOzimljC7H4va2OcFvZmJCYrRM8NtsRE8BABdHaWcnt74eWMr00b17M3fuhI38CpBZMqKbMiYiqQAAAAAAAMAFU9jY0AuU6jl1s1NTs9PTdXT0SfTUfxpes8sxsDN/v74ub0qvY0TCKxpB9BQA0N0kZpnf2DCXMs1OTUnEtNYZLZzT9noikgoAAAAAAAB0Kb0KqZ7K7tbYWHZqqvF0z82dHaVU3VswJ/g1pw7Waal3JyYyk5P2YquoG9FTAEBXktaMuZTp/ampsKVM/VWdttcTkVQAAAAAAACgk5m9drqPUaayizcSWdrdHR0ejmVT6fHxufFxe4Lfn37xxU+/+EKdJ5akx8boeGwE0VMAQNeQNQbyGxu1LmXqqdZpez0RSQUAAAAAAAA6gZm7qSOm0sfYvFlwywcHzVipNDDBr/SdSjD179fXzY5HJvitFdFTAEBHkwaNpJmaS5lOT07OfvRRvLf8Rqbt9UQkFQAAAAAAAGglCStu7uzolb+UUvenpiQrIzM52eywYnFrq6ldjiIzOaljtNKnKoesJ/j9biLiyUnzk3AiegoA6Dg6rJhbXzeXMp2+dWt2erp5c/fHNW2vJyKpAAAAAAAAQDMUNjaK29vS2yYpGdLBODs93fretucHB6mYZu71JBP8yuvy/n5xa0uWdM09eaIn+M1MTtL3GIboKQCgUziXMp2dnn74J3/SgsFQTZq21xORVAAAAAAAAKAR0rtY3N7Wy359l21561Z2aqpd2ZaFjQ2l1O2mJYRUlRoZyRqdimYm7i/PM1PvTkzID5WZmGhe7koXIXoKAGgnPR3/36+vyzt6mYEWTGdhasG0vZ6IpAIAAAAAAABVmV1nOh9DL/uVnZrqnEBgemys3bvwncAEv5KhW9zeDkzwe3t8vI0h57YjegoAaDW9Nruke6rOaNO0eNpeT0RSAQAAAAAAAE13LRa3tnTEVOdjZKemOq1/T3I2OjMMmR4fT4+P63wSSd4t7e7qbltldD+2YIHYzkH0FADQCvZAML2UaScMYmrvtL2eiKQCAAAAAADgEjInm/1yd1fevD81Nf/JJ90S1ev8PVRKBSb4LW5trW9t2RP8Slpq5+T1NgPRUwBAE9mLDehmTUdF+Dpn2l5PRFIBAAAAAABwgRU2NmRS2cLTp5IHKckYs9PT3dXlVdzevjsx0e69qJlM8DunlDK6H80JfnVuTGZiootOhyeipwCAmOmlTHXL5tbYWMdOnSE6c9peT0RSAQAAAAAAcAHYmRiyBmeHTF9Xn/L+fpf2Ompm96N6O7D99+vr8qb0QHZyD3BNiJ4CAGJQ3t/PPXkiMznI7Bmjw8OzH33UacuzO3XFtL2eiKQCAAAAAACgW9irfSmlbo2NPbp3ryv6FX38ent7/pNP2r0XcTIn+DWXodVpqXcnJjKTk9OTk5LA2rYdbQDRUzQqNTJy/0IMJQDQiMy/+3cSNP1sevrxxx9311iw8v6+6qppez1FR1LnP/lk+eHDdu8jAABAbTJdOOkZAAAAwmR/8hMJmt6dmOjwuevqdmtsrN270ETp8fG58XE9wW9xa0uyh3/6xRc//eILpdT6v/23XdRRrPVUKpV27wMAoOvlnjxJDQ+TztgVJJKaHhvrxoYLAAAAAAAALozVX/1KOqkuWMQU6nx9t4UHD9q9I/UgegoAAAAAAAAAAAAASimVaPcOAAAAAAAAAAAAAEBHIHoKAAAAAAAAAAAAAEoRPQUAAAAAAAAAAAAA0dvuHcAlVSwWy+VyNptVShUKBaVUOp0ul8v2m+l0ur27Clwq5XK5WCwG3lxZWcnlcvVtsFQqlUoleZ3JZFKplP0VUuqbqlAoLC0tScUSC/O4UqlUJpMJezMW/vvv88lSqZRKpVKpVCz7Rn0OAABiFGioFAqFsLZojA082jMAAAANsrubIhpyPlsL9CgqV79lUzsV4+2Oa5zd8di8rkh/TW2T7+/vj4yMtKtBTu4p2mZmZkZXnWtra1IDOt8E0DLFYlFaIQsLC0qptbW1bDZbLpcb2aZsSl4Uz5lf0ehOe8hkMsvLyzFuUGonORZdUznfjIX//vt8cmVlxY6RN4L6HAAAxCXQUIloi8bbwKM9AwAA0Ai7u6nBTsVAj6Jy9VvWv7se4u2Oa5zd8di8rkh/TW2TX7t2rY0NcnJPL5disajrlOnp6dnZ2cCbi4uLqVRKjwoplUorKyu3b9+em5tbXV3N5/OLi4tra2vlcnl5eTmVSskH9PalnBQKBX0dywbtPclkMvfv38/n89lsNpvN5vN5GRxhv9ns3wSAKZPJyHiuVCqVzWb1WB5plOiyr/xKulIqnU7LppRS2Wx2YWFhcXHR+RW1Wl1dXV9f19/78OHDTCbjrKkKhUI+n5ejCxyO5MKmUin/6kvog5JjjHizJs4ds/e/VCotLCxMT0/v7e0FTkrgk8qq+VOplIzkkk/q9o3z2J2/Z2Cfqc8BALjM7GdM+wHTv+lVKBScDRVnW9SzgZTL5dbX18vl8szMTD6ff/z4sT0qn/YMAABAI8JaccpqyNXdo5g513inYkDs3XH2BmdmZuyAS1gjOYzd8dh4V6TN2eOqQjoJnb9SfQEj5WqT/+mf/mk7G+QVXCaPHj3a29urVCrr6+tra2uVSmVvb++zzz6Tf93b23v06JG8vn//vv6r+fl5/aa8lo2YG9Qf29zc1J83X9vu37+vP2B+hf0mgNYzK4H79++vr69XKpX19fWlpaVKLSXd3pTzdX1GR0c3Nzflta7K7JrK/rp8Pi8HIn9Va/Vlbiefz/u86cm5Y/b+V1wnJeyTds0/Pz8f2MOIYw/7PQPfSH0OAMDlFGhpOB8wa2p62Q0V/2aPs9WqH3KXlpZ4PgUAAGgSuxVXsZpnsfQo2v/ZoNi745wbtAMuEd8bvbeB37mRrkgnZ49rxa/TtZGAUcXV/G5jg5zc08tlcXFxYWFBB/mVUsVi8eHDh/KvPsvgyWABc5pK/Vr+qVQqlctlnVYfnZsvAyICGf3ONwG0lwwgymQykklQU0k3xT6frR5XlU6nC4WCjLcK1FRhf6veHpblf1DLy8v6Y9Fv1sHesehP6pMSxq75bdHH7vN7Up8DAHA5BVoaYQ+YjTS9lHezx/5kqVTS+zM7O2sOhLfRngEAAIid2TzrkB5FW+zdcZ4b9P9eu+Mxrq5I5145e1yVRydhgwEj5Wp+t6tBTvT0csnlcqurq0qpcrk8NzeXy+XS6fTKyopM4auU0osMa9FXs/mvpVJJ1uytmmNuWl5enpubC5Q355sAOketJV0Ui8V4l/WWKUGkriiVSg0uFO9/UHt7e/KiVCrpiSmcb3YCu+a3P1PfCQ2gPgcA4BIKtDSWl5ejHzC1WJofVaXT6aWlJWklOltBAbRnAAAAmqdDehTbq761YO2Ox+Z1RTbS4xSdpQoAAAIuSURBVNp4wEi5mt9taZATPb1c8vm8FKpyuSwjcNPptMyyLYuY6rSkx48f6+EAUlqWlpZkam/5V6mwpqenzQEOy8vL5gaVUjdu3HCOgCgUCqVSaXV1dW5u7uHDh0tLS2FvAmgxWapKyrskEOiCKYu0y53Pp6Sr83Kt/1VnvZtfUfedT7rDZH9kywsLC3ZNJf8p78tuy+gwWbteBi7JAXoelDIWrp+bm4t+sybOHdvc3Azsv1lbrq6u6jaNfaTKVfM/fvx4aWlJliWQj4WdUOfvGUB9DgDAZRZoaTgfMGtqegUaKul02tkWtZs9zgZSOp2emZmRb5FFqpxHQXsGAACgQXZ3k9makoac/qf6ehRVfJ2Kpti745wbtAMuYY3ksP20Ox4b74p0sntclXena90BI+Vqfre3Qd5TqVRa+X1oOymKgfECzjdlmLDPsI5CoSDLNUdvEMAF0/aSns1mC4VCvNv0PChnDelfbbaYfVD+9wIAAIBoni0Nz7+t6c9rUiqVcrlck+Y3AwAAgGcr7tJ2QDXYc2j/eZO6Ihvvcb0YASOipwCArrS6urq0tDQ7O9vsCd8AAADQ1fSg+NXVVSbgBQAAAMLQ46oRPQUAAAAAAAAAAAAApZRKtHsHAAAAAAAAAAAAAKAjED0FAAAAAAAAAAAAAKWIngIAAAAAAAAAAACAIHoKAAAAAAAAAAAAAEoRPQUAAAAAAAAAAAAAQfQUAAAAAAAAAAAAAJRS6v8HRoQyBBTBEzkAAAAASUVORK5CYII=",
      "text/plain": [
       "Tree('S', [('A', 'DT'), ('Whitehall', 'NNP'), Tree('NP', [('source', 'NN')]), ('told', 'VBD'), ('the', 'DT'), ('Telegraph', 'NNP'), (':', ':'), Tree('NP', [('The', 'DT'), ('prime', 'JJ'), ('minister', 'NN')]), ('has', 'VBZ'), ('decided', 'VBN'), ('that', 'IN'), (',', ','), ('when', 'WRB'), ('it', 'PRP'), ('comes', 'VBZ'), ('to', 'TO'), ('alcohol', 'VB'), (',', ','), Tree('NP', [('something', 'NN')]), ('pretty', 'RB'), ('radical', 'JJ'), ('now', 'RB'), ('has', 'VBZ'), ('to', 'TO'), ('be', 'VB'), ('done', 'VBN'), ('and', 'CC'), ('he', 'PRP'), ('is', 'VBZ'), ('keen', 'JJ'), ('on', 'IN'), Tree('NP', [('the', 'DT'), ('minimum', 'JJ'), ('price', 'NN')]), ('.', '.')])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent_tokens = nltk.pos_tag(tokenize.word_tokenize(\" A Whitehall source told the Telegraph: \"\"The prime minister has decided that, when it comes to alcohol, something pretty radical now has to be done and he is keen on the minimum price.\"))\n",
    "grammer_np=r\"NP:{<DT>?<JJ>*<NN>}\"\n",
    "chunk_parser = nltk.RegexpParser(grammer_np)\n",
    "chunk_result = chunk_parser.parse(sent_tokens)\n",
    "chunk_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from nltk import ne_chunk, pos_tag, word_tokenize\n",
    "from nltk.sem.relextract import semi_rel2reldict, tree2semi_rel, rtuple, extract_rels\n",
    "text = data[\"text\"][64]\n",
    "chunked = ne_chunk(pos_tag(word_tokenize(text)))\n",
    "rels = semi_rel2reldict(tree2semi_rel(chunked))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from nltk import tokenize\n",
    "data = pd.read_csv(\"/mnt/efs/fs1/Batches/batch_1.csv\", encoding=\"utf-8\")\n",
    "sentences =tokenize.sent_tokenize(data[\"text\"][535])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'— Elizabeth Warren (@ewarren) 2019<U+5E74>5<U+6708>1<U+65E5> <U+6C83><U+502B><U+9084><U+8AAA>,<U+570B><U+6703><U+61C9><U+8A72><U+958B><U+59CB><U+5F48><U+52BE><U+7E3D><U+7D71><U+7684><U+7A0B><U+5E8F><U+3002> <U+6B64><U+6B21><U+5DF4><U+723E><U+5728><U+53C3><U+8B70><U+9662><U+53F8><U+6CD5><U+59D4><U+54E1><U+6703><U+53C3><U+52A0><U+807D><U+8B49><U+6703>,<U+662F><U+7279><U+6717><U+666E><U+653F><U+5E9C><U+6210><U+54E1><U+7B2C><U+4E00><U+6B21><U+5C31><U+7A46><U+52D2><U+5831><U+544A><U+7684><U+5167><U+5BB9><U+4F5C><U+8B49><U+3002> <U+5916><U+754C><U+9810><U+671F>,<U+7A46><U+52D2><U+672C><U+4EBA><U+5C07><U+65BC><U+672C><U+6708><U+5728><U+570B><U+6703><U+4F5C><U+8B49><U+3002><U+5DF4><U+723E><U+5DF2><U+7D93><U+8868><U+793A>,<U+4E0D><U+6703><U+963B><U+6B62><U+7A46><U+52D2><U+51FA><U+5E2D><U+807D><U+8B49><U+6703><U+3002>'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences[6]\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}