TUD_Library_Scripting_Testing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is in case you update modules while working\n",
    "%load_ext autoreload \n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Bad Crawler\n",
    "\n",
    "Experiment to test traditional HTML crawling. Inefficient and it probably overloads the library website."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from urllib.request import urlretrieve\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import time\n",
    "import shutil, os.path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "pages = []\n",
    "for i in range(1, 6):\n",
    "    url = ('https://repository.tudelft.nl/islandora/search?page=' + str(i) + \n",
    "    '&collection=education&f%5B0%5D=mods_genre_s%3A%22master%5C%20thesis%22')\n",
    "    pages.append(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://repository.tudelft.nl/islandora/search?page=1&collection=education&f%5B0%5D=mods_genre_s%3A%22master%5C%20thesis%22\n",
      "File exists: data/tudelft_repo/TUD_1575733a-8813-4c5d-b916-0eb23ac477f9.pdf\n",
      "File exists: data/tudelft_repo/TUD_05957f39-d946-4956-be68-52f5675e100d.pdf\n",
      "File exists: data/tudelft_repo/TUD_e675fadc-fd8d-4ced-b59e-45b6faba6fdd.pdf\n",
      "File exists: data/tudelft_repo/TUD_c9492ef8-9254-4ba4-83b3-8b0ad46149c5.pdf\n",
      "File exists: data/tudelft_repo/TUD_6e274095-9920-4d20-9e11-d5b76363e709.pdf\n",
      "File exists: data/tudelft_repo/TUD_b2308402-21ed-41a2-92b7-7edd7175dca7.pdf\n",
      "File exists: data/tudelft_repo/TUD_c46eca4b-eaf5-461a-9be2-dd6b3b1b44e0.pdf\n",
      "File exists: data/tudelft_repo/TUD_ca8da1e3-94c0-438b-9281-ee4a76cb7129.pdf\n",
      "File exists: data/tudelft_repo/TUD_15227443-568e-4bd8-af88-a165c4776f11.pdf\n",
      "File exists: data/tudelft_repo/TUD_cff45740-9100-4a10-9d49-41da874146d5.pdf\n",
      "https://repository.tudelft.nl/islandora/search?page=2&collection=education&f%5B0%5D=mods_genre_s%3A%22master%5C%20thesis%22\n",
      "File exists: data/tudelft_repo/TUD_cb4c6d30-588f-4d9c-a619-5bb205c87a35.pdf\n",
      "File exists: data/tudelft_repo/TUD_83c4f538-b6ed-449f-a0ed-4359b5a0fb23.pdf\n",
      "File exists: data/tudelft_repo/TUD_87709465-b9a1-48da-9ba5-eba98bc263d7.pdf\n",
      "File exists: data/tudelft_repo/TUD_6ac288c7-40fc-438b-8623-2533a686d8e7.pdf\n",
      "File exists: data/tudelft_repo/TUD_74d36a36-b9f7-448b-9947-0ed671c12ad0.pdf\n",
      "File exists: data/tudelft_repo/TUD_d57d7f66-ef72-474e-b8e1-add9e28b6f2f.pdf\n",
      "File exists: data/tudelft_repo/TUD_a9171959-8a02-4c97-8a56-d5188d223471.pdf\n",
      "File exists: data/tudelft_repo/TUD_27b75b3f-8b4d-4977-82ee-e531fd986839.pdf\n",
      "File exists: data/tudelft_repo/TUD_5a298e79-819c-4f47-affa-c15b266c3112.pdf\n",
      "File exists: data/tudelft_repo/TUD_2a625496-e85d-4207-8d6b-0bd06565fdf9.pdf\n",
      "https://repository.tudelft.nl/islandora/search?page=3&collection=education&f%5B0%5D=mods_genre_s%3A%22master%5C%20thesis%22\n",
      "File exists: data/tudelft_repo/TUD_7312afe5-28cc-4d04-b6d9-15899bf49a51.pdf\n",
      "File exists: data/tudelft_repo/TUD_ea9282a7-d6a8-47da-b8cf-dbeb198225b7.pdf\n",
      "File exists: data/tudelft_repo/TUD_2d25af25-f4dc-4537-8153-71408cf22fa0.pdf\n",
      "File exists: data/tudelft_repo/TUD_d826e500-4551-4a8e-bd80-fefea35a8c10.pdf\n",
      "File exists: data/tudelft_repo/TUD_c372f6e8-92f2-4793-9b14-e4b445b1a6c8.pdf\n",
      "File exists: data/tudelft_repo/TUD_5ad5ceeb-5f90-44af-928e-747e8accb5fb.pdf\n",
      "File exists: data/tudelft_repo/TUD_ec15c141-289b-4a8d-ae76-652bae893ab4.pdf\n",
      "File exists: data/tudelft_repo/TUD_f9226b86-abd1-4fe9-bab5-8bc4d4d4fb78.pdf\n",
      "File exists: data/tudelft_repo/TUD_5e88d71c-3878-4c4e-967b-0b5727bd0d8f.pdf\n",
      "File exists: data/tudelft_repo/TUD_566adfe1-1f1b-4060-8847-74ad797d4cf2.pdf\n",
      "File exists: data/tudelft_repo/TUD_b0e50a76-351e-4678-82ec-5c5868e8fc88.pdf\n",
      "File exists: data/tudelft_repo/TUD_a47f9e44-6f68-4b39-b9a9-89b16b99cf9c.pdf\n",
      "https://repository.tudelft.nl/islandora/search?page=4&collection=education&f%5B0%5D=mods_genre_s%3A%22master%5C%20thesis%22\n",
      "File exists: data/tudelft_repo/TUD_479b6e60-0521-49ed-9431-a3640c35d281.pdf\n",
      "File exists: data/tudelft_repo/TUD_4c0839c9-f00b-4c4a-a327-1bc214078871.pdf\n",
      "File exists: data/tudelft_repo/TUD_76fc815e-b11a-4fdc-932b-35e398c4d89d.pdf\n",
      "File exists: data/tudelft_repo/TUD_dddb8c36-c82a-4281-965e-12b855844322.pdf\n",
      "File exists: data/tudelft_repo/TUD_a275789e-9a45-402c-9617-8f23b9866c7b.pdf\n",
      "File exists: data/tudelft_repo/TUD_2d9bc31b-9bc3-4cac-a69f-3c1c06cd5f7f.pdf\n",
      "File exists: data/tudelft_repo/TUD_bfda3b1e-9c27-4faa-8583-7c7acdfc90a6.pdf\n",
      "File exists: data/tudelft_repo/TUD_f9a658fe-9d0b-4a1d-808a-908698c8e8c2.pdf\n",
      "File exists: data/tudelft_repo/TUD_e7b9324f-f297-4c7e-9257-331e6bc6eef6.pdf\n",
      "File exists: data/tudelft_repo/TUD_d0e3de61-f9f1-4a2f-9af5-7a9cfd8c174c.pdf\n",
      "File exists: data/tudelft_repo/TUD_b3f0f792-610b-46e0-89e1-133e27d93e8b.pdf\n",
      "https://repository.tudelft.nl/islandora/search?page=5&collection=education&f%5B0%5D=mods_genre_s%3A%22master%5C%20thesis%22\n",
      "File exists: data/tudelft_repo/TUD_d01c0eca-50f3-4339-b7bf-dce035dd4ebb.pdf\n",
      "File exists: data/tudelft_repo/TUD_509c2759-d8d6-44a9-af3f-b0dc9911dc40.pdf\n",
      "File exists: data/tudelft_repo/TUD_4633114e-32c9-45c8-97f2-4663b7d4d9b3.pdf\n",
      "File exists: data/tudelft_repo/TUD_d3804b8f-0c4d-42aa-886e-b47f68a60f98.pdf\n",
      "File exists: data/tudelft_repo/TUD_1843361f-52ce-4f80-b651-eec1e4e5633b.pdf\n",
      "File exists: data/tudelft_repo/TUD_2a1ed71d-589e-474f-912e-2a82f9c6bf35.pdf\n",
      "File exists: data/tudelft_repo/TUD_292c84e9-033b-448f-9c78-48eef9c79101.pdf\n",
      "File exists: data/tudelft_repo/TUD_92e0f356-37ab-4b73-a01f-1c49f3cfaaea.pdf\n",
      "File exists: data/tudelft_repo/TUD_8d739e31-1c9b-405d-a6b7-af171e04acd3.pdf\n",
      "File exists: data/tudelft_repo/TUD_9d440bc9-ec54-43bf-b4ea-2787ea47dca5.pdf\n",
      "File exists: data/tudelft_repo/TUD_20170a8c-7e1e-434f-b5d6-9ba933e2ab6e.pdf\n",
      "File exists: data/tudelft_repo/TUD_b7fd7229-debb-4270-8b47-7e5f084ed91a.pdf\n",
      "File exists: data/tudelft_repo/TUD_dae2563e-b12c-467f-8842-3e0a2498a259.pdf\n",
      "File exists: data/tudelft_repo/TUD_3be70155-068a-4f41-8397-2a0bb71db5d3.pdf\n",
      "File exists: data/tudelft_repo/TUD_ccd3a80f-224e-46b9-9183-69960b74c6f7.pdf\n",
      "58 files already existed, 0 files downloaded\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "j = 0\n",
    "for item in pages:\n",
    "    page = requests.get(item)\n",
    "    soup = BeautifulSoup(page.text, 'html.parser')\n",
    "    thesis_list = soup.find(class_='islandora-basic-collection-list')\n",
    "    thesis_list_items = thesis_list.find_all('a')\n",
    "    print(item)\n",
    "    for thesis_name in thesis_list_items:\n",
    "        url = thesis_name.get('href')\n",
    "        uuid = url[25:-21]\n",
    "        download = 'https://repository.tudelft.nl/islandora/object/uuid:' + uuid + '/datastream/OBJ/download'     \n",
    "        name = 'data/tudelft_repo/TUD_' + uuid + '.pdf' \n",
    "        if os.path.exists(name):\n",
    "            print('File exists:', name)\n",
    "            i = i + 1\n",
    "            continue\n",
    "        r = requests.get(download, stream=True)\n",
    "        if r.status_code == 200:\n",
    "            with open(name, 'wb') as f:\n",
    "                r.raw.decode_content = True\n",
    "                shutil.copyfileobj(r.raw, f)\n",
    "            print('Downloaded:', name)\n",
    "            j = j + 1\n",
    "        if r.status_code != 200:\n",
    "            ('Download failed, status', r.status_code, 'link', download)\n",
    "\n",
    "        time.sleep(1)\n",
    "        \n",
    "print(i, 'files already existed,', j, 'files downloaded')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# OAI Extraction\n",
    "\n",
    "Obrain metadata from the OAI-PMH site of the Library (Open Archives Initiative Protocol for Metadata Harvesting)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from urllib.request import urlretrieve\n",
    "from bs4 import BeautifulSoup \n",
    "from sickle import Sickle\n",
    "import shutil, os.path, pickle, time, requests\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "sickle = Sickle('http://oai.tudelft.nl/ir')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24539\n"
     ]
    }
   ],
   "source": [
    "filter_types = ['master thesis', 'doctoral thesis']\n",
    "items = {}\n",
    "records = sickle.ListRecords( **{'metadataPrefix': 'oai_dc', 'ignore_deleted': 'True'})\n",
    "for item in records:\n",
    "#     item = records.next()\n",
    "    uuid = item.metadata['identifier'][0][32:]\n",
    "    try:\n",
    "        if item.metadata['type'][0] in filter_types and item.metadata['description']:\n",
    "            items[uuid] = item.metadata\n",
    "    except KeyError:\n",
    "        continue\n",
    "    \n",
    "print(len(items))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "fac = []\n",
    "for uuid in items:\n",
    "    try:\n",
    "        fac.append(items[uuid]['description'][1])\n",
    "    except IndexError:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "fac = []\n",
    "for uuid in items:\n",
    "    if len(items[uuid]['description']) > 1:\n",
    "        tmp = items[uuid]['description'][1]\n",
    "        if 'computer' in tmp.lower():\n",
    "            fac.append(items[uuid]['description'][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "hidden": true,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'Computer Engineering': 7,\n",
       "         'Computer Science': 3,\n",
       "         'Computer Science, Embedded Software': 1,\n",
       "         'Computer and Information Science - Software Engineering': 1,\n",
       "         'Electrical Engineering, Mathematics and Computer Science': 18,\n",
       "         'Master of Science Computer Science': 1,\n",
       "         'Microelectronics & Computer Engineering': 4})"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Counter(fac)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 345,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "items 57771 types 62105 x 62105\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Counter({'Text': 1,\n",
       "         'abstract': 155,\n",
       "         'bachelor thesis': 1128,\n",
       "         'book': 1926,\n",
       "         'book chapter': 525,\n",
       "         'conference paper': 10767,\n",
       "         'contribution to periodical': 29,\n",
       "         'doctoral thesis': 8331,\n",
       "         'journal article': 14483,\n",
       "         'lecture notes': 489,\n",
       "         'master thesis': 16201,\n",
       "         'other': 77,\n",
       "         'patent': 544,\n",
       "         'public lecture': 1228,\n",
       "         'report': 5797,\n",
       "         'review': 46,\n",
       "         'student report': 300,\n",
       "         'working paper': 78})"
      ]
     },
     "execution_count": 345,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = 0\n",
    "types = []\n",
    "items = {}\n",
    "\n",
    "records = sickle.ListRecords( **{'metadataPrefix': 'oai_dc', 'ignore_deleted': 'True'})\n",
    "for r in records:\n",
    "    types.append(r.metadata['type'][0])\n",
    "    uuid = ''\n",
    "    uuid = r.metadata['identifier'][0][32:]\n",
    "    items[uuid] = r.metadata\n",
    "    x = x + 1\n",
    "\n",
    "print('items', len(items), 'types', len(types), 'x', x)\n",
    "Counter(types) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "with open('tud_metadata.pickle', 'wb') as handle:\n",
    "    pickle.dump(items, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 403,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "with open('tud_metadata.pickle', 'rb') as handle:\n",
    "    items = pickle.load(handle)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "failed = []\n",
    "for uuid in items.keys():\n",
    "    download = 'https://repository.tudelft.nl/islandora/object/uuid:' + uuid + '/datastream/OBJ/download'     \n",
    "    name = 'data/tudelft_repo/TUD_' + uuid + '.pdf' \n",
    "    if os.path.exists(name):\n",
    "        print('File exists:', name)\n",
    "        print('.', end = '')\n",
    "        continue\n",
    "    r = requests.get(download, stream=True)\n",
    "    if r.status_code == 200:\n",
    "        with open(name, 'wb') as f:\n",
    "            r.raw.decode_content = True\n",
    "            shutil.copyfileobj(r.raw, f)\n",
    "        print('Downloaded:', name)\n",
    "    if r.status_code != 200:\n",
    "        print('Download failed, status', r.status_code, 'link', download)\n",
    "        failed.append(uuid)\n",
    "    time.sleep(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "for uuid in items.keys():\n",
    "    download = 'https://repository.tudelft.nl/islandora/object/uuid:' + uuid + '/datastream/OBJ/download'\n",
    "    name = working_dir + 'TUD_' + uuid + '.pdf'\n",
    "    if os.path.exists(name):\n",
    "        print('File exists:', name)\n",
    "        print('.', end='')\n",
    "        continue\n",
    "    r = requests.get(download, stream=True)\n",
    "    if r.status_code == 200 and items[uuid]['type'][0] in filter_types:\n",
    "        with open(name, 'wb') as f:\n",
    "            r.raw.decode_content = True\n",
    "            shutil.copyfileobj(r.raw, f)\n",
    "        print('Downloaded:', name)\n",
    "        x += 1\n",
    "    if r.status_code != 200:\n",
    "        print('Download failed, status', r.status_code, 'link', download)\n",
    "        failed.append(uuid)\n",
    "    time.sleep(1)\n",
    "    if x == 20:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'200f6974-3d2c-4383-87bd-1760fad834d8'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-59-68f574c2380c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0mpath\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdirpath\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0muuid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'_'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m         \u001b[0mdownloaded\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0muuid\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mitems\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0muuid\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m: '200f6974-3d2c-4383-87bd-1760fad834d8'"
     ]
    }
   ],
   "source": [
    "downloaded = {}\n",
    "for dirpath, dirs, files in os.walk('data/tudelft_repo/'):\n",
    "    for file in files: \n",
    "        path = os.path.join(dirpath, file)\n",
    "        uuid = path.split('_')[-1].split('.')[0]\n",
    "        downloaded[uuid] = items[uuid]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "uuids = items.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'contributor': ['Hermans, L.M. (mentor)'],\n",
       " 'creator': ['Reijtenbagh, R. (author)'],\n",
       " 'date': ['2010-10-05'],\n",
       " 'description': ['Rainwater harvesting is the primary source of fresh water on the majority of isolated and volcanic islands in the Eastern Caribbean island chain that lack reliable sources fresh water. Recent research studies have determined that the microbiological quality of rainwater in cistern systems is generally unstable and can lead to serious health implications when used for human consumption. Another related issue is that rainwater shortages occur more frequently due to fluctuating rainfall patterns and increasing rainwater consumption rates. In this paper minimum design requirements are determined, using data from the islands of Saba and St Eustatius, for the construction of new domestic rainwater systems on small volcanic islands in the Eastern Caribbean. These minimum design requirements can be used to prevent potential water quality and quantity issues. One of the conclusions of this research study is that mainly the collection and storage elements of rainwater systems need to be constructed using non-toxic and non-corrosive materials to mitigate potential health risks. Also, first-flush devices need to be installed in the collection system to divert the mostly contaminated first load of rainwater after a dry period. The storage tanks need to be watertight and protected from sunlight, preferably located at least 30 meters away from active bacterial sources (such as open street sewers or cesspits). The minimum rainwater catchment area for a standard rainwater system, based on a 4-person family with a modern water consumption pattern, is 113m2; the minimum cistern storage capacity is 24m3.',\n",
       "  'Policy Analysis',\n",
       "  'Multi Actor Systems',\n",
       "  'Technology, Policy and Management'],\n",
       " 'identifier': ['http://resolver.tudelft.nl/uuid:06a2ebaf-78a3-44f2-a3bb-e0d491aa8c5f'],\n",
       " 'language': ['en'],\n",
       " 'rights': ['(c) 2010 Reijtenbagh, R.'],\n",
       " 'subject': ['rainwater harvesting',\n",
       "  'water demand management',\n",
       "  'drinking water',\n",
       "  'water quality',\n",
       "  'catchment area',\n",
       "  'rainwater cistern',\n",
       "  'design requirement'],\n",
       " 'title': ['Minimum design requirements for domestic rainwater-harvesting systems on small volcanic islands in the Eastern Caribbean to prevent related water quality and quantity issues'],\n",
       " 'type': ['master thesis', 'Text']}"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "items[list(uuids)[0]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Delete Large Files\n",
    "\n",
    "For easier processing. There are some very heavy pdfs that may not be necessary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "target_size = 100 * 1048514\n",
    "large = []\n",
    "\n",
    "for dirpath, dirs, files in os.walk('data/tudelft_repo/'):\n",
    "    for file in files: \n",
    "        path = os.path.join(dirpath, file)\n",
    "        if os.stat(path).st_size > target_size:\n",
    "            os.remove(path)\n",
    "            large.append(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "with open('large_deleted.pickle', 'wb') as handle:\n",
    "    pickle.dump(large, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Collection in Mongo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import pymongo\n",
    "import elasticsearch\n",
    "from elasticsearch import helpers\n",
    "import nltk\n",
    "import config as cfg\n",
    "\n",
    "sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "client = pymongo.MongoClient('localhost:' + str(cfg.mongoDB_Port))\n",
    "publications_collection = client.TU_Delft_Library.publications\n",
    "es = elasticsearch.Elasticsearch([{'host': 'localhost', 'port': 9200}],\n",
    "                                 timeout=30, max_retries=10, retry_on_timeout=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def extract_metadata(documents):\n",
    "    list_of_docs = []\n",
    "    for i, r in enumerate(documents):\n",
    "        extracted = {\n",
    "                \"_id\": \"\",\n",
    "                \"title\": \"\",\n",
    "                \"publication\": \"\",\n",
    "                \"year\": \"\",\n",
    "                \"content\": \"\",\n",
    "                \"abstract\": \"\",\n",
    "                \"authors\": [],\n",
    "                \"references\": []}\n",
    "        extracted['_id'] = r['_id']\n",
    "        extracted['title'] = r['title']\n",
    "        extracted['publication'] = r['faculty']\n",
    "        extracted['year'] = r['year']\n",
    "        try:\n",
    "            extracted['content'] = r['content']['fulltext']\n",
    "        except KeyError:\n",
    "            pass\n",
    "        try:\n",
    "            extracted['abstract'] = r['content']['abstract']\n",
    "        except KeyError:\n",
    "            extracted['abstract'] = ''\n",
    "        extracted['authors'] = r['authors']\n",
    "        try:\n",
    "            extracted['references'] = r['content']['references']\n",
    "        except KeyError:\n",
    "            extracted['references'] = ''\n",
    "        list_of_docs.append(extracted)\n",
    "    return list_of_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "extracted_publications = []\n",
    "query = {}\n",
    "results = publications_collection.find(query)\n",
    "extracted_publications.append(extract_metadata(results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "955"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(extracted_publications[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['content', 'authors', 'title', 'abstract', '_id', 'year', 'references', 'publication'])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extracted_publications[0][0].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TUD_862e5b58-b9d1-462a-90b0-6f94a054632e\n",
      "TUD_960eb3cd-080d-4423-b9cb-f82417e9e13c\n",
      "TUD_c35fa6e0-e632-4f17-bf84-0f1cc8f98c0c\n",
      "TUD_f116361b-8ff7-4995-a8e5-e5354f2902b9\n",
      "TUD_46081d76-17fc-41f6-bdc4-fbdc6d7bb939\n",
      "TUD_ed609f8d-6cc5-489f-aa45-669b797f677a\n",
      "TUD_f514548b-63c6-4015-a824-c86c9a63477c\n",
      "TUD_beca2029-d274-4d14-9ea0-a59490019bc1\n",
      "TUD_fc4aa0e6-1d20-41e9-a660-237a6873bb3c\n",
      "TUD_84ff0f8e-ec0a-4b77-8fa9-9df7eaef6b60\n",
      "TUD_9ced4cb2-9809-4b58-af25-34e458e8ea23\n",
      "TUD_b0c2a859-4351-4b6a-b66b-a7bbfed76ecb\n",
      "TUD_280fe4d8-d1de-4a46-ad99-be8129d85dab\n",
      "TUD_78bf003b-e784-40ca-888c-48e0246b3883\n",
      "TUD_ba867fef-5421-4692-b955-6c2c468653cc\n",
      "TUD_b6ea3435-825a-4306-a4d4-4edee26f233c\n",
      "TUD_8150156e-e633-4319-8685-ccac7b083434\n",
      "TUD_c061a03a-84a5-4ac3-ba22-c5b315f08edd\n",
      "TUD_32789615-a48a-4b0f-ba27-d11992a201d1\n",
      "TUD_187e5e62-db64-4f62-8b08-c5dddfc174a3\n",
      "TUD_22662036-5abb-4a5a-9f1a-a4a9a68e0997\n",
      "TUD_d8f95cd0-f243-4989-89f3-a8a262b1bfd6\n",
      "TUD_8bd86467-7565-481c-beec-4d76423f1bff\n",
      "TUD_167d7879-7f61-4c5e-9390-7d1435e9eda1\n",
      "TUD_a69af610-68f9-447d-9c82-cdae18994f50\n",
      "TUD_975a4a47-7935-4f76-9503-6d4e36b674a3\n",
      "TUD_c281056f-a646-4736-9f4b-bb53ecaa5d72\n",
      "TUD_88ece5de-f233-4c13-a940-f5c862a9b154\n",
      "TUD_011a9efc-4813-4e4f-8e7f-a5f7c2e27f1b\n",
      "TUD_84f0e48c-a6f8-4708-92f6-047e849f565a\n",
      "TUD_e57d0dc4-14aa-4223-9042-4d46c6f0ee8c\n",
      "TUD_5e522da0-6712-42c7-b90a-55a7f80d64b5\n",
      "TUD_17de3262-b7a3-400b-8fbc-1b0a25d4d957\n",
      "TUD_36f8bedc-2fb1-461d-a979-9bfdb9520930\n",
      "TUD_026c9807-3632-4272-b69f-a3b6f5452607\n",
      "TUD_d5b89811-d405-469e-a508-7f062d13558f\n",
      "TUD_7cd6c046-956b-47bf-bd21-d631a1677e98\n",
      "TUD_f2723dd7-d3f4-4ee0-a59c-3394562c4150\n",
      "TUD_7ddd648a-f5b2-4d18-8cee-75a7c7210b57\n",
      "TUD_ed582f9d-2d93-48d1-8b81-3711fd22742c\n",
      "TUD_e8fd5e0c-4f2d-47b0-9952-06cea4fe0c98\n",
      "TUD_6efb78a4-5221-46f4-851b-fa776cb174de\n",
      "TUD_6547d4c2-0e5f-430a-aed4-bc67c8bf7744\n",
      "TUD_38d9e021-b5ec-48f8-8aea-93a4393a0478\n",
      "TUD_427a64d8-6ba1-4418-a04f-e5f092c9ff60\n",
      "TUD_2a54b619-6f90-474b-83a3-95656874206b\n",
      "TUD_9f7a367d-100d-4ff1-a02e-3532d96689ee\n",
      "TUD_b38bc8f5-3e86-4ef7-83df-c292193bbea2\n",
      "TUD_38efa276-e4e9-40b2-bd04-b7f9c5e2c492\n",
      "TUD_4600d978-3988-4daf-b6a4-4534e54634f6\n",
      "TUD_1306f207-66c9-460c-8c73-e41c4af74ef4\n",
      "TUD_4eb03123-3847-43ce-8f8f-214c382b5698\n",
      "TUD_1ad87d96-5c2d-4463-957e-e104495b05cb\n",
      "TUD_a5639b34-9df5-4be8-bd49-7c3e972046be\n",
      "TUD_7348938d-c490-4b13-a5e8-e771a28b71f2\n",
      "TUD_33c6e093-a129-47ec-a663-3ce808343a93\n",
      "TUD_73783036-0241-4dff-bae8-e56ad155ce00\n",
      "TUD_ab5d5b57-daed-4f7f-96fa-f0f9bb2fd83e\n",
      "TUD_97097039-766a-4784-810e-79447d5b036c\n",
      "TUD_27d7627f-8163-4cb9-aa3e-591ecdd2370e\n",
      "TUD_215714d3-63dc-48e4-ac11-39484cb120ac\n",
      "TUD_c290585a-8e2d-448a-9b31-52540877195e\n",
      "TUD_48a8afda-774f-4658-a283-3170bbe9ef73\n",
      "TUD_504b4d73-c4ab-4e5e-bcaf-ca6d2ff7347b\n",
      "TUD_4c9abeb7-6b98-4f07-9cbb-ed27bbdade06\n",
      "TUD_1e24b455-728d-42c0-82c5-e55f64956161\n",
      "TUD_8cd415c3-1ca3-4974-982f-40b5508bd8e1\n",
      "TUD_db220b8a-c131-4cb2-bde0-604a269703ef\n",
      "TUD_73f1f696-0349-4d79-a7a2-75c38c137473\n",
      "TUD_59d5a00d-6bb7-490d-9887-32e9530eb33e\n",
      "TUD_49a3bfaa-012f-4e3b-ac34-b8d9a760c4fc\n",
      "TUD_4f520286-ec98-49fb-a005-f18023c1ef9b\n",
      "TUD_e36d3aba-e35b-457e-8dae-5a6fcdd4d86b\n",
      "TUD_70d91e29-e24b-4e31-b701-b2357ffc3a8e\n",
      "TUD_a9711610-96bf-4741-af78-b3d393f0b875\n",
      "TUD_1d235433-6f66-42a6-87b1-ef2843f0db38\n",
      "TUD_a9f7c378-9896-4ab7-ad3d-b8c093c36ffc\n",
      "TUD_0edd0472-39df-4296-b692-e9916e79fb1e\n",
      "TUD_b0671ce6-355a-4c8a-a525-81931b0f92a0\n",
      "TUD_5acbd4cd-ab5f-486a-8268-61cd7416550d\n",
      "TUD_46a0d7b7-5f30-481b-a48c-de32513f69d3\n",
      "TUD_3dbdac61-5e13-432a-b05a-cb66f5f55b7d\n",
      "TUD_5d7ae484-2b60-40f4-b7b8-1ca0a32425b3\n",
      "TUD_1390209c-a35a-49af-a082-1c855663d124\n",
      "TUD_42501e33-6bdf-467a-8788-2d49e5098638\n",
      "TUD_11f0e772-3564-4e22-b4d7-fe94c6b69987\n",
      "TUD_62bee645-b8a0-40c6-be48-28ebbb541c42\n",
      "TUD_ce583533-45ea-4237-b18d-fe31272ea1ee\n",
      "TUD_008b12b6-5115-4c48-9dd0-f88999ccb34b\n",
      "TUD_34c62679-c826-4169-a84f-f101d012478e\n",
      "TUD_7ab16201-272c-4f18-899c-98a7fb527093\n",
      "TUD_69f7fbd3-4922-4526-b92d-0f3a6bbe6ba8\n",
      "TUD_bab69ac3-07b7-4dae-8b80-670443af2faa\n",
      "TUD_4bada907-7f8c-4dfb-bb80-b1c687bb5b7f\n",
      "TUD_355f297a-bc22-445e-a489-934582d1d1d2\n",
      "TUD_aafcf8fe-4292-406b-9c58-bc368fe55994\n",
      "TUD_9203ba22-72e0-4da5-bb50-f9e88339cc9a\n",
      "TUD_1eab77a9-5693-4fd1-abf7-0306233f4388\n",
      "TUD_9df0aca7-a587-413e-b18f-07bcd7e608b9\n",
      "TUD_9a3f74a3-6049-4c24-8b52-5ae044e46201\n",
      "TUD_437e234e-bed8-45e3-a9b4-1378abd1dcda\n",
      "TUD_9acfce4e-0cab-4d85-b1a3-ce9ed0cdb748\n",
      "TUD_4d437aa6-1593-423c-bc3a-d12d46806675\n",
      "TUD_f96e4cae-a5ea-408e-b520-9a1cc3c42f8d\n",
      "TUD_6bf48a02-371d-47fe-b642-a88f5c917a1d\n",
      "TUD_b3daf825-2d23-4161-ae1a-a1577614c33c\n",
      "TUD_077f4577-c9b8-4be7-91a7-49bf751e73ef\n",
      "TUD_4cd6734c-1976-4cd9-8d90-bd7145e5d2a6\n",
      "TUD_2bcc9188-aaaf-4232-8c85-4a0997a3935a\n",
      "TUD_a8128b6f-c8a1-4739-a845-8bdb3de536ce\n",
      "TUD_e7cf3e77-7b81-4028-8c7a-c2a62e94261b\n",
      "TUD_f2ecf719-d8d4-4197-be07-1822e84f6968\n",
      "TUD_5e0bf5b4-f361-4d36-8558-7cb76337fb0c\n",
      "TUD_64c8efc7-7a36-4acf-a493-147d5a1051d5\n",
      "TUD_333007e2-0fe5-43da-892c-d86b0a80d9ef\n",
      "TUD_6d74b3b5-cb4e-462c-be1d-91af20a8e057\n",
      "TUD_dea2ad58-69f6-46db-bea7-831a0cca7e46\n",
      "TUD_e91fb785-f2b8-459c-a5c7-ab926da5d9cf\n",
      "TUD_f4bb7503-1c62-4a0a-a77c-e119381e4307\n",
      "TUD_282aa7de-cba5-422c-98df-7ad24e89924b\n",
      "TUD_85873447-3211-43cb-a139-935f804e36ac\n",
      "TUD_2ad8a0a4-4536-4a3d-88ec-2734509868da\n",
      "TUD_6e483ba3-052d-4711-aeb1-a03e84ba2c08\n",
      "TUD_4c8919ec-534c-4da7-8e48-273485c2b9f2\n",
      "TUD_ef7d8499-ff37-4ebc-97d6-ee9804938f5c\n",
      "TUD_7fef8c52-6adc-45d5-8913-6d3fe044139b\n",
      "TUD_76ab40b5-ccdb-4db1-95ef-db71c01e0c7f\n",
      "TUD_827cfa3f-0027-45cd-9cf2-6fd43551db07\n",
      "TUD_633f3846-b8ea-4bf5-84d4-21752c584f06\n",
      "TUD_52874750-4eab-4974-8a42-8da5b700adff\n",
      "TUD_31002d86-b88d-46e2-9462-6c214a552543\n",
      "TUD_576da4a7-781c-4950-a36e-2366211022a7\n",
      "TUD_9bc1729f-6471-4dd6-b61b-8e2a16f3c951\n",
      "TUD_aec34e9d-a436-4631-b7ff-5563a99b73c0\n",
      "TUD_80363f35-b986-42fd-9880-e6ef96506daa\n",
      "TUD_d610163e-784c-4e55-893d-03c0b3a28fce\n",
      "TUD_5210165d-f011-4cd3-8551-a1374641065d\n",
      "TUD_256f5716-0947-48c3-946d-f47048cb3446\n",
      "TUD_28a5e49f-db97-4002-8a24-1ec9cedeaf75\n",
      "TUD_81232e85-5c72-4b02-8cfc-462b4996b633\n",
      "TUD_cff6cd3b-a587-42f2-a3ce-e735aebf87ce\n",
      "TUD_1b75db0a-60da-4a6c-8c02-6cc2550c90d0\n",
      "TUD_53c8e1e0-7a4c-43ed-9426-934c0a5a6522\n",
      "TUD_2b0bba39-b9a9-4c52-a06e-1fb20dcfd5b8\n",
      "TUD_e0704a99-6b12-4fc0-8310-c7cbabfe5a8d\n",
      "TUD_0e024a88-8c4f-4213-b1c3-67bfb552c669\n",
      "TUD_ebba15d9-71ac-42c3-89f7-221c6604828c\n",
      "TUD_dc26e6ac-c0f7-4313-8236-9d4742968441\n",
      "TUD_156aa284-e3d7-461d-ab4b-4fb4d5fc7116\n",
      "TUD_5ea21702-d6fb-484c-8fbf-15c5b8563ff1\n",
      "TUD_3175edae-5f98-46d2-b661-277370aba7fc\n",
      "TUD_db7e1836-73f2-465d-8f03-ab5b9d3d3018\n",
      "TUD_a6324bce-a3e7-4563-b8e8-2d3b43db837f\n",
      "TUD_bdf8976c-46b6-40e8-a336-e9ddfef7f565\n",
      "TUD_e1ec5f1f-a9ee-414e-9ee5-6f656fd42996\n",
      "TUD_e3b70e0e-ef65-4b17-9e28-2c29a1e40972\n",
      "TUD_7ee4b3eb-e136-4889-80f2-b440c5ff49be\n",
      "TUD_609562d5-b953-4f40-b85c-4e2506a68084\n",
      "TUD_c39ba351-4dec-4fa9-824f-6ff618a7980d\n",
      "TUD_c9d57c31-3359-47be-a8f6-98b2c1e7ebe8\n",
      "TUD_a44c7da7-2979-480f-bf8c-fb90e65df727\n",
      "TUD_cc5a729c-aa94-4b85-a4eb-16e0e781f24e\n",
      "TUD_11c817fc-334b-471d-a535-26ac8f0f448b\n",
      "TUD_4339deb2-8764-45ba-a688-a9078795f386\n",
      "TUD_3c57be43-0e51-4897-94d7-f6bcc2189a1d\n",
      "TUD_17adc7bd-5c82-4ad5-b1c8-a8b85b23db1f\n",
      "TUD_a7e5a7be-902a-4536-8247-7ce83b26ab4b\n",
      "TUD_ddbde9de-c7fe-4499-89fb-504b4241ea9c\n",
      "TUD_4b80dec5-66be-4b00-904d-e064ca35406b\n",
      "TUD_2e8d335a-4251-41d9-b1fd-1697be9fbf2e\n",
      "TUD_f724734b-eab2-4e50-b697-92671479d87e\n",
      "TUD_e7a452de-6859-42bc-97b6-3eb28815dc8b\n",
      "TUD_6b015fa3-5692-4285-815b-4d2901a6ff45\n",
      "TUD_b742f141-1351-432f-b551-6249e65ce822\n",
      "TUD_4590b1fc-6f44-4297-9e0f-04bac4c29334\n",
      "TUD_cadc9c6f-9da5-4d55-a0f2-cd48b9f0ee3b\n",
      "TUD_da1e4bbb-ce28-42d3-ae2b-f076ccc7ee91\n",
      "TUD_1b362496-af89-4ba1-b7f8-7088ccaf2a01\n",
      "TUD_cbdd109f-c692-46f4-b157-0e25bb46b2c7\n",
      "TUD_26b76962-0465-4785-ab6c-446dd6663262\n",
      "TUD_d1da3a56-14c5-4942-91a3-68e12a0e3103\n",
      "TUD_9cb25a9a-0d17-48c7-842c-3ef83ed044e5\n",
      "TUD_a775a647-83b8-43c0-978a-44bba80bb5d4\n",
      "TUD_e77bc353-149a-401d-baef-938450da160d\n",
      "TUD_e225decc-1acb-4e0d-ab28-11bbab434788\n",
      "TUD_0b5b81ae-bb45-4183-b815-6cac07f7fc3d\n",
      "TUD_f5bc5f90-97a6-46dc-839e-f5c532ae51f2\n",
      "TUD_07bc49be-571c-49de-a7a6-075cede74c54\n",
      "TUD_f150ebc6-9720-47b2-b7e7-2413eb82b957\n",
      "TUD_d16d440a-32cc-4f75-ba51-d47769c322b5\n",
      "TUD_bf259621-d718-432a-8bd3-4c003a6629a4\n",
      "TUD_2b48115c-19b0-4301-bcbe-b3e8ce6aacba\n",
      "TUD_6fef6b25-58c2-42d4-8271-b00b5ddb00b4\n",
      "TUD_59b88cba-fa73-424d-868a-864b921155c1\n",
      "TUD_e74c11d4-b9be-4b89-ae5b-b7e5bc853717\n",
      "TUD_d7afed76-8f00-4f78-bbb0-193e30d7afc0\n",
      "TUD_b0a8ae03-18b9-4a0e-9761-64ffd2851074\n",
      "TUD_6e2807a1-fdec-4fb4-9569-fd1ce2eb8264\n",
      "TUD_9276fd1a-300a-4c4c-8971-780fdf544a66\n",
      "TUD_3f903786-381a-47a4-a7a5-93cfd370361a\n",
      "TUD_56e646dc-d5c7-482b-8326-90e0de4ea419\n",
      "TUD_24d84ce3-d325-4dad-a1f1-ab35f942def7\n",
      "TUD_adef294a-56f3-4e17-a17e-61b7518a19b9\n",
      "TUD_d21d7b64-7600-47b6-b01c-3a543ca5d6d2\n",
      "TUD_29181e27-003c-4ffa-9c77-aeb53783ad91\n",
      "TUD_29f849e2-ed5f-4295-94f0-7092ead693bf\n",
      "TUD_ba7f3919-90c6-453b-a4a5-bbfd5f345dcf\n",
      "TUD_dd7b9002-03c9-4976-8bf8-6c91885608ef\n",
      "TUD_8f824b85-adf4-4576-a3d1-a2563c9e5734\n",
      "TUD_0e9a3aed-2277-43c9-b574-bb9eed326fa5\n",
      "TUD_08fc576d-7df8-4e8f-b813-52325df5d559\n",
      "TUD_6579ee97-6315-48ed-a7f1-1fca8870f130\n",
      "TUD_169e44a3-cc02-44bc-92f6-573461fe46e1\n",
      "TUD_54f7d7a3-de8a-4127-9cf3-897eb79a1860\n",
      "TUD_26955829-f8b7-4058-afc1-eea84ed4ea11\n",
      "TUD_e5a817a4-ce0a-4dd3-afd4-d70660b63d16\n",
      "TUD_1dfbb52d-761b-4cb5-9414-d858f8ab2573\n",
      "TUD_0a5a2b01-d3cd-4b8a-88ba-b02ed28a24d3\n",
      "TUD_dacac69b-b5f8-4216-a143-825a0e3bc2ea\n",
      "TUD_610cbb5e-dc40-4068-8064-6c2c5e0d1364\n",
      "TUD_57a87999-c68c-4a9e-b5db-2a2eb2aef7eb\n",
      "TUD_6083701f-09fa-4cc2-b45e-b1b74c117994\n",
      "TUD_ea90cd2c-90f9-4726-accd-1ee8d5967f26\n",
      "TUD_9266518e-2052-4f48-a13d-0e9c698125bf\n",
      "TUD_133e2ab3-185f-43fe-a2e7-e54a823d91e8\n",
      "TUD_07be4544-d0ba-4767-804b-12c313b10a0f\n",
      "TUD_b1ea7059-773d-4d48-a28b-32bb24c7f201\n",
      "TUD_6eb43736-74ad-4512-9be9-3f3c5ac9b50f\n",
      "TUD_f0bc3711-a40a-432c-b153-9e1c8b8e2935\n",
      "TUD_b157de07-e5ce-4dba-8eae-154a0002a4f5\n",
      "TUD_36851412-be13-486a-92df-09680ad98b3b\n",
      "TUD_50b4176e-21c7-4271-988a-52c13a4f2de3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TUD_5c01eb69-0d5b-4016-8b55-b450e87c24e2\n",
      "TUD_e7ed020a-ee53-4010-a14c-4db3ae31a01b\n",
      "TUD_a22c55d7-4c69-44c9-bc9f-03652b4ddfdf\n",
      "TUD_adda5ce3-0931-4282-8086-fb0cb4fc3d5d\n",
      "TUD_740995b0-2576-42ea-aa49-940c6e10a7ba\n",
      "TUD_bbed8ce5-36a8-4773-a461-33f19c5f774d\n",
      "TUD_f43422a6-ef31-4f61-ae7a-f71d3b3c645b\n",
      "TUD_9328b51a-7eb5-4206-b463-48d99b86aafd\n",
      "TUD_c0c07509-513f-4aa1-a07c-78f5c9ff5684\n",
      "TUD_b806e4df-0982-4b2e-9e63-7299ab3a8bff\n",
      "TUD_3758d2cb-d0af-45d1-9f14-93e1c86f60ed\n",
      "TUD_204b09ee-971b-4035-a4b1-616ee50afbba\n",
      "TUD_74af4924-f262-4136-8f44-29c6be14534e\n",
      "TUD_8ef36410-06c2-4372-a9fe-e0e45b264a73\n",
      "TUD_40148cf1-8b85-49df-9f5a-55ac257a04f7\n",
      "TUD_81f376a3-2c9a-4d24-b547-6b384dc0aef6\n",
      "TUD_493487ad-ac1e-4ad5-9264-48be7ebc2f74\n",
      "TUD_68ef1d61-f3c3-43e8-bacd-d2e97bd0baa1\n",
      "TUD_4fd01d95-6bf6-4864-8662-f7adf837e2f5\n",
      "TUD_f17092aa-4d78-4ef3-a80c-365825e810f1\n",
      "TUD_117aa738-99ea-4357-b880-769b58c84d97\n",
      "TUD_c3faec41-007f-4b2c-acbe-eb2eb89efd32\n",
      "TUD_d229c59f-c03f-4907-b9e8-67b83b619ac3\n",
      "TUD_29251f71-81ea-475c-8a3f-5795a6d1b341\n",
      "TUD_2eb0995d-2c58-447a-99df-061a2883aba0\n",
      "TUD_8c42378a-8769-4a48-a7fb-f5457ede0759\n",
      "TUD_6850e673-4da7-4026-a488-d80c4174170b\n",
      "TUD_becdcccc-4ef8-4eac-91df-d77571c629de\n",
      "TUD_64ee5526-8c9e-4013-9019-c63a63413ca2\n",
      "TUD_7dd09f0f-254b-42f1-a852-b61cb9f2bed3\n",
      "TUD_f7db8070-8470-484d-a7ce-ad2b81ea3318\n",
      "TUD_fa059b99-b750-4122-a31d-eee8495115a8\n",
      "TUD_d5c61e4c-9550-4c58-a069-fc8e940945ce\n",
      "TUD_8c8e1930-31b4-4e68-85cd-e6e4ba49f4f2\n",
      "TUD_0b59e5db-72ee-460e-bad6-63912f5238c0\n",
      "TUD_6d553910-2169-4c2c-a494-bdc2abd838b9\n",
      "TUD_2287ef2c-248f-409b-959a-eb1f6236e4a4\n",
      "TUD_2356d539-3500-4bb9-bfec-a2bc8d089b59\n",
      "TUD_7330511f-c22e-4ad1-a0ce-44c1c2747c05\n",
      "TUD_0b5408a7-933c-4369-8d48-1db5d3269d72\n",
      "TUD_92a3bc67-53cf-45ab-a3cd-e3c20fab9ce8\n",
      "TUD_643fd074-b051-431a-86c1-181c67497672\n",
      "TUD_409710f1-9646-40d4-a889-11c0a41a724d\n",
      "TUD_5f6beeb7-6485-4000-9894-677343c35097\n",
      "TUD_d7e33788-1ab5-4bb6-b2ec-1b31a5b85fc1\n",
      "TUD_f1c5891e-91fe-42be-bbd8-d0f5552ccf8a\n",
      "TUD_7c8f5d4c-d680-4ba2-98da-a2980b35a269\n",
      "TUD_021d1ea2-12ab-477e-b23e-77875eed6b9d\n",
      "TUD_9e8316bf-200e-4d33-9322-42f8da8233b1\n",
      "TUD_4ba70bd1-8496-4df3-9b2c-f42c700f894b\n",
      "TUD_7d9c6c44-734d-4ffd-80d8-38997ff4d747\n",
      "TUD_be0bd616-57ab-46d6-b02d-84af9dd6c717\n",
      "TUD_44dac56b-e0a5-4818-811c-81d9a408edee\n",
      "TUD_2e41c0ca-6128-4137-842a-6c0b5dab6009\n",
      "TUD_e2234250-950d-4eb5-9f2e-b5b8e67af2e5\n",
      "TUD_b56f7a38-b08c-48e1-93dd-4c0401fca76c\n",
      "TUD_b6b6915b-ac95-45ad-838d-c8fad5b8a171\n",
      "TUD_6f6b3972-c290-45eb-8b77-617129cacdac\n",
      "TUD_7421072c-e61b-47b1-87a0-d2cf85050817\n",
      "TUD_7b958de6-2087-42d9-96e6-31a6b14072fc\n",
      "TUD_bd900036-40f4-432d-bfab-425cdebc466e\n",
      "TUD_6b62bb3c-9889-421a-92cc-b0ab436c4dde\n",
      "TUD_d76287f8-0b94-4919-abab-efc9f11ff8a1\n",
      "TUD_f5fa2307-fe28-4570-a679-b4ac64d007e6\n",
      "TUD_7e70359f-7ddd-4914-b2f7-b782103708ab\n",
      "TUD_4b4a7448-0807-4a54-ac21-a7dafa7fbf92\n",
      "TUD_4060494b-3725-4abc-aefb-8235fd05871c\n",
      "TUD_3d78c977-ef85-4f9f-bcba-2e0a37c13745\n",
      "TUD_2f902ffc-2f44-45e3-b7ba-414c7420f764\n",
      "TUD_bf950f1e-db9f-4e3e-8ca1-5b00a87a1452\n",
      "TUD_322cd4ef-69bd-4dd9-98b2-539b598986b6\n",
      "TUD_e1391687-ab07-4f62-bc1a-3343dc577f23\n",
      "TUD_b7117ae8-33ee-45a6-ba70-5e9e731a56b6\n",
      "TUD_6bae6ac2-71e8-4ef0-9d61-7f3c86dfb5cc\n",
      "TUD_060fc436-07fe-47f8-b418-d53e8fb2c75c\n",
      "TUD_53fece0d-c512-4bd8-a0da-ea3311b35156\n",
      "TUD_73c6c6d2-cba7-423c-a7f0-7a07a88f5499\n",
      "TUD_cccad780-b786-4dcd-a10d-e1496d07eb6f\n",
      "TUD_5ae85b5f-ce1a-4ae3-8e8c-1991b45b08eb\n",
      "TUD_555299fd-6918-4dfb-b883-5f642a97b324\n",
      "TUD_2f31c7c5-5019-4cab-a9c5-65528f886a76\n",
      "TUD_e12ddb8d-ab56-411a-bbab-dbc942fe8b60\n",
      "TUD_d2d8c74b-9580-4101-9b72-5a5370a4bcd8\n",
      "TUD_52b586ea-6144-4b4e-a5a1-b05255ce493a\n",
      "TUD_93f9d3af-38f7-45f4-ba62-e921c386f3c9\n",
      "TUD_34a6aa6d-6038-414e-9541-8906fe6e2f5e\n",
      "TUD_1f285e6f-31c3-4173-9281-9dca204d28fe\n",
      "TUD_359825f9-ed16-43f4-a998-d2028a6c3204\n",
      "TUD_73ffa055-c46d-43f6-9a1a-747e0af16f43\n",
      "TUD_36a62709-cb33-4c4a-88b6-0ad9fba1555b\n",
      "TUD_d9b524b0-d2e1-4bde-883a-cc6313a1d8c0\n",
      "TUD_3efc8aae-e31f-47a4-a92c-b9da05917ada\n",
      "TUD_a65f49cf-5447-4816-8352-8c42846fc103\n",
      "TUD_894e4234-ed2b-411f-be90-550766f97cd5\n",
      "TUD_0a0685fc-ce0a-4580-a355-978810005b71\n",
      "TUD_35192f4a-f466-4d73-aa3c-5d77f8622709\n",
      "TUD_7e04a67d-fc3c-4d11-a1e0-67043044e637\n",
      "TUD_eb9462c4-078b-40ae-9a14-ae61ba46dfa6\n",
      "TUD_e5f90dec-7e32-411c-a228-7210c6be2ea6\n",
      "TUD_7fdb578a-a3e3-430c-b257-c85bfc45d3d9\n",
      "TUD_7b9246f5-0bfb-4eb4-8566-61415fd14cac\n",
      "TUD_d867d3a3-5534-4644-b6f4-20e7d7ae69af\n",
      "TUD_5e1bf344-f581-4ce6-a083-94573322f653\n",
      "TUD_35cd1bee-7723-43a9-b581-1cc4dc4ce07d\n",
      "TUD_cbdf86c6-9979-44c5-9f0e-c0bb96bb16d9\n",
      "TUD_53dd3e0b-5c31-43b1-b8a8-767951c10249\n",
      "TUD_ca07093c-44f1-40e6-8dc6-f01d9b7f61c8\n",
      "TUD_51b11f1c-699a-42f4-9373-b5c9697fde74\n",
      "TUD_52256ba7-e744-49d4-8bcd-a5ded6052ad8\n",
      "TUD_00f6916d-a226-44ee-bbe1-9cb03d49bd37\n",
      "TUD_d5d68d7a-1c30-4594-96bf-a03a8b4b8a8f\n",
      "TUD_3cec89f0-9a2e-4a09-a015-6a7f30ac0c96\n",
      "TUD_f672209e-162a-4ac6-8317-1f18b287b8c9\n",
      "TUD_29893270-5711-46be-9476-dc33f837a0d1\n",
      "TUD_9ecad815-9dfb-4a33-aea8-bf4b83beb8bc\n",
      "TUD_215065f7-d22d-4189-a5cb-0b2e91897cd1\n",
      "TUD_2c345e58-b4b6-426b-9f28-f730a000baf5\n",
      "TUD_f28f32fb-f420-44c1-9683-bc2a5e0ec993\n",
      "TUD_f3242bfa-1b32-4d83-ad31-c188c9b6377b\n",
      "TUD_f2fe4764-4c8c-4164-97f8-dc550dbfc73c\n",
      "TUD_eb91db81-587c-48f5-a570-321666a7dc75\n",
      "TUD_67387fb8-a443-41c3-8452-0cdcf17f0e5c\n",
      "TUD_dc4257fb-f1d4-4fa4-a289-63805bce8258\n",
      "TUD_77bb79d5-8909-40ec-8ae0-e8384594fcbf\n",
      "TUD_6e3fb29a-fcd1-4099-bb75-77eff2a03a90\n",
      "TUD_efbce68d-3ff5-4025-a602-0541af87648b\n",
      "TUD_151bc710-b9ac-437d-a43f-a429223314ee\n",
      "TUD_98a8d01e-6259-4fed-8704-5af0615304d2\n",
      "TUD_774bf8c6-8ded-4a98-9b6a-77454262a548\n",
      "TUD_07102bcb-dc4e-4bd4-9691-9de832ffa2e1\n",
      "TUD_5e14890b-d4e3-469d-bc3b-cc1f2f6df945\n",
      "TUD_a8f6363b-5362-4c9b-a928-3a4542bf4774\n",
      "TUD_89ca1773-7dd7-40ec-8abc-38bc22719b49\n",
      "TUD_74069df0-87a2-46be-9a96-0f0df06b922e\n",
      "TUD_64d94833-5cb8-4198-bc91-ac13564141d5\n",
      "TUD_9c4be69a-cb01-417c-84d5-a65ed57c72c8\n",
      "TUD_80ea30e9-11c3-4e7e-b250-7de6015f4744\n",
      "TUD_dcc2005e-c6f3-43ff-b003-4cf3bf1770f3\n",
      "TUD_f98d2c87-e564-4cf4-a9ab-aeeccceefa38\n",
      "TUD_229ece19-fd8c-4ca1-a38f-dd52c6aa3175\n",
      "TUD_63251f1a-8c42-4153-94f4-c85c92a8e950\n",
      "TUD_2eac935e-cdb1-4c0c-92d1-cbcc8dd2d867\n",
      "TUD_6ed0e1cc-7fdd-4b0a-abfc-5e353adf0279\n",
      "TUD_3c4c5c99-c866-454d-ae15-5fb62eee0ba8\n",
      "TUD_32f5e7d4-54b5-4fc1-9c46-9da21d17f0d4\n",
      "TUD_2632334a-f9ee-41b0-aaad-9f77375fc256\n",
      "TUD_8b892951-0000-431b-a313-89190f335d0c\n",
      "TUD_99424150-5e7f-43d5-a864-12841c9f31f5\n",
      "TUD_8233ed00-a892-4b18-beda-057e539a7d73\n",
      "TUD_6f85ea91-2398-4d98-ad75-cfd8ffdaa768\n",
      "TUD_f1f08f26-416e-4f50-a518-e4aae5fb14d2\n",
      "TUD_22f3bdbb-2ffa-4478-85d9-1f71eae8abf0\n",
      "TUD_a4c07b84-24ed-417a-9ad7-779bdd2422f0\n",
      "TUD_acc16325-1c25-445d-ac3c-02c3e89170d4\n",
      "TUD_cbec7bdd-4bab-4132-93cd-359587b9bf46\n",
      "TUD_a6c9be48-8f89-4165-b556-6525fe28cc11\n",
      "TUD_2fe353d4-1988-4790-aa87-bde1626b68f6\n",
      "TUD_0ed5ce33-60ac-432e-a5ff-5b3f10037a8b\n",
      "TUD_1647e1cb-84fd-46ca-b1e1-21aaf38ef30b\n",
      "TUD_ff0d73fa-339b-46c2-930c-f0e7b58625b6\n",
      "TUD_8d9193a3-8945-4881-985a-ad1a57d07477\n",
      "TUD_31ab95ab-384a-4af6-a606-26331f98e099\n",
      "TUD_f51f6f82-f660-4357-8b18-a8ef1fc0283a\n",
      "TUD_35d0f569-200b-452b-b470-34a56b803d7f\n",
      "TUD_3f677e31-aa91-4641-a7a2-db8295ffa5fd\n",
      "TUD_099bac02-8424-4dfc-9658-1fc9cdf5292d\n",
      "TUD_0ff8e807-9166-47cc-b803-1edc2803a1b8\n",
      "TUD_3137bc78-cfa1-4207-957f-6b91eae62a2f\n",
      "TUD_e141272c-80aa-40b3-8856-49ba0e7f35c4\n",
      "TUD_9e4b5949-f628-40af-b06c-c218a7a7cdb8\n",
      "TUD_20d263f3-6b9b-4fc9-ac6d-0d4baec896ce\n",
      "TUD_2bee12f5-22da-45fc-9576-c61808562a78\n",
      "TUD_06063c8a-1410-444c-bc6b-6b05e14af6f4\n",
      "TUD_8dd60526-5399-49e9-afe9-9f6e19c49ca7\n",
      "TUD_2e41be6a-5220-421c-bef8-bbb8f91c128f\n",
      "TUD_003f5644-69f1-42da-b462-2184866c1080\n",
      "TUD_8a569a87-a972-480c-bd7d-11292b3f5cdc\n",
      "TUD_bdad6dd4-c730-406a-995f-563e74c04616\n",
      "TUD_7604ccff-d585-4069-8e0e-bbaf8c90ad31\n",
      "TUD_b89345db-2993-4d27-a13f-25b2c0eb4126\n",
      "TUD_d41522e3-690a-4eb7-a352-652d39d7ac81\n",
      "TUD_c09bd3de-68a7-46ea-8a5b-b6b557429a93\n",
      "TUD_8cf81c2f-6ccb-4136-a6bc-51dfc8ffcab8\n",
      "TUD_a2fa837d-a669-4613-bec6-2e59e923e95d\n",
      "TUD_8fd22239-6559-48bd-8f55-54c33d64dd32\n",
      "TUD_932574a3-5a38-4c3d-9c34-c9ae11a161d2\n",
      "TUD_505300b0-f421-4470-b63e-cdb9154f7d54\n",
      "TUD_a304b6c7-6e5a-418e-9ac9-c2e9fb359f0b\n",
      "TUD_c9afa796-39a8-4487-8e56-2a7f020cedfe\n",
      "TUD_c049c3c2-c8d8-4a3c-86ef-222846e91291\n",
      "TUD_1922c463-91af-4c4f-8720-a89e7da4f8f6\n",
      "TUD_5d8e2c24-1262-42d6-8109-5d7b3e6f8755\n",
      "TUD_a619ad13-090c-466f-ad4f-f3ea51cfdd07\n",
      "TUD_5ef1696a-0ef8-4d4c-a807-3d0fd3247b1d\n",
      "TUD_eb91654e-87d8-4a3f-957e-93b8b94452ea\n",
      "TUD_b26fb6be-a3e1-4968-9044-01fc6a029926\n",
      "TUD_2a833efc-b55f-49bb-8965-a6c23605f621\n",
      "TUD_c9772109-7e5e-4858-8c10-6c880cbbf847\n",
      "TUD_5f7ba046-3a5c-46bb-8368-896a88f5e4a7\n",
      "TUD_2912a5b2-2cea-42ba-9ba4-1ca06b08d9d6\n",
      "TUD_2b238ade-cf76-4220-a407-4f3465d56dbc\n",
      "TUD_7d06198f-bf9f-4b3c-aa89-41c2a12d31ae\n",
      "TUD_51d9ddff-4ff1-494d-8070-c246b073e3d4\n",
      "TUD_d2a72d3f-d28b-4d9d-998a-b030c2f28aed\n",
      "TUD_07210c9d-6a0d-4329-8de9-3c7973f55da3\n",
      "TUD_f36ef6c3-0742-4372-a2f1-7f34f686ded6\n",
      "TUD_93137061-e3ef-418e-aa7a-b7539f70af50\n",
      "TUD_f2ae49ae-d01b-4374-b406-b4c1810ea364\n",
      "TUD_44f0676a-b4ee-491c-94c9-fd4c6ed29d5c\n",
      "TUD_230f983a-1119-4b08-80ee-be3b120f2d1f\n",
      "TUD_56227569-cafb-4b75-9b0a-c40199c7a894\n",
      "TUD_eca407ca-512c-4b17-82c2-dae9236a2066\n",
      "TUD_1e20f67a-d211-49dd-a32b-ea19a12b2e87\n",
      "TUD_ccdbdd3f-7871-4db1-8def-1bc9d710aed6\n",
      "TUD_90443566-8eb6-499d-8f7a-945344029c29\n",
      "TUD_a8b5fb6e-074b-4df0-97b2-6212a56552d8\n",
      "TUD_b50e0abb-f5d9-42af-88a2-6ddf31c8419a\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TUD_f444e6d3-cfa1-47dc-9fb4-9ca6978b4249\n",
      "TUD_bc53cc52-8507-43a0-9325-d4304067f677\n",
      "TUD_486154d6-b535-4a2a-b394-c2f7f3f43ac4\n",
      "TUD_b913c0ee-fa19-44a6-a7ef-41fecde7125c\n",
      "TUD_5bb2bb3c-c545-4d7f-9502-186ac0f5956e\n",
      "TUD_6a2cad07-86c3-4a16-9d04-1224ef4d900f\n",
      "TUD_1a95dc0b-64f1-42f1-9971-1f8bcb146b1f\n",
      "TUD_682a56ed-8e21-4b70-af11-0e8e9e298fa2\n",
      "TUD_218b896a-af74-4637-a0e7-fcd4e4743fed\n",
      "TUD_3e8a66e5-80f4-466d-b6bb-961f9896b800\n",
      "TUD_dd58900a-a201-474b-b02a-4feab83b25d1\n",
      "TUD_60fee9f1-7f5a-4915-8f09-2a3bcdbdd55c\n",
      "TUD_39b1653b-cde7-419b-bcd0-8549b6e34db5\n",
      "TUD_4d4e1516-3c86-4b78-8c13-bf89953af029\n",
      "TUD_ff57aedb-20f8-42cb-bc6f-513b8c8e94c5\n",
      "TUD_98015359-f99c-42a0-bb63-4e99c975decc\n",
      "TUD_a1689901-8ebe-4d2a-bab9-9e9569e3d83c\n",
      "TUD_6980f3d8-362c-412b-a560-565fd14d59c4\n",
      "TUD_29c201ec-0813-45b7-8d52-53371868150c\n",
      "TUD_5050b8ae-184b-40c6-9640-c4ba7fb875c6\n",
      "TUD_076ec1f6-d6c6-4266-afc5-dd71e24d65b2\n",
      "TUD_0eb48447-21ff-4202-85c0-25017c9fea6a\n",
      "TUD_3f184afb-241d-4d54-827d-75fcd59211a5\n",
      "TUD_e9b513c8-751b-45a7-9d99-a51177c918a2\n",
      "TUD_70ad3815-d280-4e2b-8810-43ecdcf2086e\n",
      "TUD_96cb75a3-7cf9-41a4-bce9-bc6a6ff5eb7e\n",
      "TUD_a6f87376-f084-4f33-bcfc-ced18c6c0fc4\n",
      "TUD_02156130-7e6c-445a-922d-98ed37bf87e3\n",
      "TUD_740626ef-fe0c-42b6-a177-c50270aaeca5\n",
      "TUD_e848e945-aa35-49c5-b62c-53baac9c06c9\n",
      "TUD_c4826844-77f6-4ee4-8b62-799fdbf12ee5\n",
      "TUD_de9a0da0-3d86-4a83-b25f-f93fd5bcc058\n",
      "TUD_f310b31e-aaec-43fe-9c9c-90e46076c0b9\n",
      "TUD_1af94380-1414-4497-bfc6-a67b213de050\n",
      "TUD_976110c5-9254-4994-9630-a7a03687f469\n",
      "TUD_4e36121c-da15-46ab-953e-d6a87d1d4271\n",
      "TUD_e695c2ce-e7f8-4f55-b073-e4768cf39bbe\n",
      "TUD_1a9dbc80-aeda-4564-a5d3-878eabc062b5\n",
      "TUD_b9c99dac-329c-4e57-b27c-d447cdf916bc\n",
      "TUD_acf11f8a-631b-4069-94fe-89bf746fdee1\n",
      "TUD_59d84718-b299-48f7-bdef-62c57377dfcf\n",
      "TUD_85cef785-c17c-41d7-9a58-6a183c468523\n",
      "TUD_ef93503c-ce97-41d4-baf3-c7e7c08ed355\n",
      "TUD_464eb73e-e250-4174-b792-36cc1ec15331\n",
      "TUD_ee554cc0-a7e6-49ee-939e-312bcef3d674\n",
      "TUD_d29a69f8-4410-4e74-a533-e9bf6e82f015\n",
      "TUD_f23ea108-9c84-4d4b-acdf-ecdd87188572\n",
      "TUD_cc957424-7611-4dd5-b84c-91ee9b1283d0\n",
      "TUD_4e24a08a-6b73-4c30-bcf1-ac6af8c0e1f9\n",
      "TUD_ba2c8674-878c-4f0e-814c-e93b96e7f788\n",
      "TUD_6e0e2b2b-9c26-4dc4-bca4-c177f936e90b\n",
      "TUD_66591440-00bd-4e67-a9b8-6738334909a3\n",
      "TUD_0fa98a15-e579-4542-8c9f-cfecf3b64272\n",
      "TUD_48b95ec8-a5aa-477d-8640-64a48d3d4fdf\n",
      "TUD_76f1c917-b89d-42e0-b5b3-14c3cdb11f7a\n",
      "TUD_bbe25dba-c11b-4be2-bda2-8b38fef2b6ca\n",
      "TUD_3eacd020-fc29-472b-9088-cec44de14a0a\n",
      "TUD_91601ffa-d704-45ab-a528-a3b83224fed7\n",
      "TUD_7a8aa949-fdaf-4ea9-bb0a-3a77b49557f9\n",
      "TUD_393b73c9-2943-49e6-9a72-30d0e590cb77\n",
      "TUD_69d2d334-8653-406b-b63a-fc195021a4d8\n",
      "TUD_77807364-bcad-47e1-92dd-5d3bcef92a7e\n",
      "TUD_e23175cb-d1a8-4cb6-af7b-9909f6412cf5\n",
      "TUD_80c01881-4210-4979-8820-57ed80b8995c\n",
      "TUD_7f48e4c8-22df-4b99-9028-80fed9add7ef\n",
      "TUD_0fcd09e9-0a0f-4d23-a155-fb03aa8dcbcc\n",
      "TUD_17f35cb5-61dc-47a2-a97d-d30261f62b6d\n",
      "TUD_681a1aee-f2df-4477-bf66-523c6204d5c1\n",
      "TUD_8f0cd1af-dcab-4124-aa42-50f9240c236c\n",
      "TUD_eef80801-b933-4cd2-ae0a-ca797aa74774\n",
      "TUD_66fefd5e-697a-468b-b58e-1ad3b4c25bf0\n",
      "TUD_5cc01f8f-52be-4811-b8ae-fec6e9f74f6b\n",
      "TUD_6a9e06e1-d7ba-411b-b8f9-7f373fcb9c50\n",
      "TUD_ed9b743c-a4ad-44be-9622-b51de5f39795\n",
      "TUD_bf16e165-9c76-4fb2-b1b0-719b73e60735\n",
      "TUD_b619e7d6-3df5-40ee-8108-ea7411ffeda7\n",
      "TUD_7000a1fb-695f-4bd9-980d-6481dadba321\n",
      "TUD_0cebb406-ffa4-4da7-9c66-0df77a48b1b3\n",
      "TUD_4a6bd52a-62fe-4522-8dfa-ea6f1a2195f5\n",
      "TUD_9800e588-cb61-4283-a41c-9d3d77ffc764\n",
      "TUD_adabe8f0-e85f-467b-8046-c44403e005e9\n",
      "TUD_d82de64b-b446-4df6-b335-36a3e058c8f8\n",
      "TUD_db8b269c-6f09-4a3d-9378-4e573900c7dd\n",
      "TUD_0c22a102-85cc-4d3c-9068-098a5eb5341d\n",
      "TUD_1ae97783-b8a5-41d8-a1d7-50c8817915e6\n",
      "TUD_5e1452a4-18da-49c1-b219-bb6cd5742c46\n",
      "TUD_9392f54c-14ff-4d6b-bf24-9692efde84a2\n",
      "TUD_1640ad86-fc07-4e54-9f05-1629296738c7\n",
      "TUD_e3021462-e39b-48b2-893b-59fa8ba75fb7\n",
      "TUD_dcb25701-2f87-4264-b50e-95f6ee5de5d7\n",
      "TUD_ff6b1363-019d-4891-b138-c90105e49cdf\n",
      "TUD_4f4d4450-ac86-4adf-82e4-7ccba84faa0c\n",
      "TUD_bb27eb90-fa46-4db2-a28b-7c719f0d6b7b\n",
      "TUD_04fc8fe3-d338-42b4-8bd3-6c76ea04b844\n",
      "TUD_18898596-1c6a-4c00-8d7e-40d9d80ce0ec\n",
      "TUD_ade5368d-2dc0-4bd9-8c36-48171483c4ab\n",
      "TUD_085fd330-b3d0-496f-a6cc-e8f025f73dc0\n",
      "TUD_02afa9dc-d585-4e17-a102-dc06d770a232\n",
      "TUD_16cec059-45b2-4946-8291-b92b72db8913\n",
      "TUD_015747c3-6c47-42e5-9ece-2ebdaaa2ba29\n",
      "TUD_dfb98715-941d-4a62-8743-b0acb0d701b7\n",
      "TUD_68fe8d4d-6a3d-41d9-884a-608314fc75d5\n",
      "TUD_3a01d973-d125-430f-82e2-fb83cc9239fb\n",
      "TUD_16ee713c-0651-425f-accb-b8398f09e751\n",
      "TUD_861b2cc7-8ca3-4bed-8d51-f94a359824fc\n",
      "TUD_893d1673-279e-4aa0-ab1c-efee64d4d03e\n",
      "TUD_5ee7e852-fce5-45af-8737-cf950cebd8a5\n",
      "TUD_2e8cf03e-2353-4770-be9b-686f3cf12229\n",
      "TUD_9e21bbb7-d034-4f10-8a28-3674187a0e59\n",
      "TUD_8174a80b-b048-47df-b4a6-9e48d83bae96\n",
      "TUD_e5c541f3-d5d6-4bc7-ab4b-b7b10c5c775c\n",
      "TUD_9b542aca-20fd-4bcf-a8f4-c0231e625404\n",
      "TUD_340a1ab8-c9b6-443f-9f7c-122a3b61f4e6\n",
      "TUD_9aa9ebe3-d2fa-4cf3-b22e-45326d700053\n",
      "TUD_202f2531-34f4-4b44-bbcf-a7c61aad2a2e\n",
      "TUD_22ef6407-682a-46ce-9393-ddf71286b448\n",
      "TUD_a19a6464-c9d2-47ca-a5aa-9bdcc8f51b6f\n",
      "TUD_7f3eeb52-77bd-4fdb-84a9-ea9ca0a35b94\n",
      "TUD_904db2dd-b63a-4d17-8742-1ebbf1627e01\n",
      "TUD_ec115fda-092c-469f-b8b2-450a72b3e489\n",
      "TUD_6da2ee07-99df-450d-93bd-2367725f4f70\n",
      "TUD_b859167f-6c9e-4ff0-acb8-044d55d6e69a\n",
      "TUD_9e2b32fb-278f-4f6f-9a32-91f6f49bc656\n",
      "TUD_dc8ce7c6-d13a-4ee3-9332-5294eb58d055\n",
      "TUD_76e3a56a-a8b2-478c-ae04-dba5c6a13a2e\n",
      "TUD_9e062646-1f23-48cf-bfc0-90eb58d7ab4a\n",
      "TUD_51828825-d26d-47e6-b760-6f14f2ba2f7b\n",
      "TUD_e6c72c1e-4c88-4f6a-b966-e5072bb928d2\n",
      "TUD_d4d2230a-a034-43fa-bd29-0eb6b954410b\n",
      "TUD_5b3a8ae9-a605-4933-adcb-b8f736425423\n",
      "TUD_d4655d6f-ba1b-4306-a0f1-1b495189ca9a\n",
      "TUD_3fac5a1c-c8ea-4fee-a856-9ed2afd9e96f\n",
      "TUD_5e399c00-8724-4ebf-8de3-d54efe407354\n",
      "TUD_878a94cc-5839-42f4-9ccf-ff48742526ef\n",
      "TUD_f503211b-3444-49aa-828a-ddcf0a7ea389\n",
      "TUD_2db8fe18-436c-4c76-9947-014e33ddffcf\n",
      "TUD_e7241d8a-418d-4aa7-806a-cce2055f1488\n",
      "TUD_cee5e97b-d023-4e27-8cb6-75522528e62d\n",
      "TUD_7f7b27b2-a2f9-4366-9bda-8e722d12b188\n",
      "TUD_68f91b3f-bc5d-4b1c-a927-d14d262bb4b4\n",
      "TUD_b13cd88b-099e-489c-a7e4-ae5ec0313cea\n",
      "TUD_5251d378-675c-4540-9bbd-a9e5bade7cc6\n",
      "TUD_00e26d49-a0c6-43ec-9820-bc6927948f80\n",
      "TUD_d4603eaa-055c-433c-99d2-3ce7b2704aa8\n",
      "TUD_96bde5c2-3db4-42c3-aa72-829b7eb54739\n",
      "TUD_8c63437d-ada3-4d67-8a51-b17fc4608efb\n",
      "TUD_a332673e-0fab-499e-ae08-cf6ff5ce8197\n",
      "TUD_de5b12d4-d625-4d4d-8f22-af155fd25959\n",
      "TUD_83bc5ff4-8e33-4962-9014-68cfbad8226f\n",
      "TUD_0022768b-2d96-46bf-84a3-b92067f762b9\n",
      "TUD_d059fea6-2861-49b4-ae36-5d31db109231\n",
      "TUD_2aa6252e-1175-42ec-ab5a-80731af65520\n",
      "TUD_73f81186-198d-4de9-a19a-52423b694d43\n",
      "TUD_3a53a0ff-4279-4492-8eee-5a47da9aad40\n",
      "TUD_a8516f8b-3aaa-4978-a079-85b9c2a80086\n",
      "TUD_ec871698-8a96-4836-ad74-da71c9231d29\n",
      "TUD_67f21f09-c6a0-44b9-b66a-479ba25bee66\n",
      "TUD_03860c22-0d7a-4812-b173-f285c1e87f85\n",
      "TUD_f3abc79e-6b62-4faf-a977-3845bb3ebc5c\n",
      "TUD_d8a6b83e-61ad-41ed-94c3-f090ac256ba3\n",
      "TUD_caa7e54d-1f8a-40d0-9bb1-78b202581d8f\n",
      "TUD_6d524027-0abc-46ec-a710-dc958062a38d\n",
      "TUD_33cbf224-4a0d-410e-85ad-cf8d05409f52\n",
      "TUD_6234f613-497d-4a2b-95bf-4bc52b1e3ed1\n",
      "TUD_12cb7563-45db-4824-bcf7-8f5af11ff52f\n",
      "TUD_3b7e2cc6-4810-44a9-a19a-3cc8a260d495\n",
      "TUD_700ffbee-3511-465d-ad6d-4eeee8710dbd\n",
      "TUD_c4edb267-f2c2-4085-8477-3a0d065f6278\n",
      "TUD_2b68174d-683c-4342-8e4a-4f705ab5871b\n",
      "TUD_727ec48a-1598-4cd5-91dc-0c86814abc71\n",
      "TUD_c3a4b57e-8679-4393-a78e-7a8658f62090\n",
      "TUD_4f879ecf-95d5-4482-8931-8c40abde0e79\n",
      "TUD_0475ae7f-cbf2-4722-9716-7481068de192\n",
      "TUD_59b71cd4-a7a0-430e-bba8-246745865607\n",
      "TUD_bf960362-6fa2-4879-b036-a2d662f3297b\n",
      "TUD_e17c7526-bb84-47f9-ae0a-36cacc343e1b\n",
      "TUD_5e3dfbfe-19a8-4ac4-b9b6-117aab0737bf\n",
      "TUD_3c99ceca-d210-47d7-8096-7a5d1138462f\n",
      "TUD_43dae97c-431c-4ed3-8064-35686940a32a\n",
      "TUD_5182f8c5-c9c6-4e6f-a0bd-7fbbe43cd977\n",
      "TUD_a1490c8b-35a3-4f41-a433-0e0d0899c833\n",
      "TUD_d901168b-b72e-4777-ab29-2533132b86c9\n",
      "TUD_fbe7382b-01c0-4e17-9982-755bac76064c\n",
      "TUD_4e8dd607-0447-4e33-90b0-0fa60f9db44b\n",
      "TUD_b9ad8db5-a436-4254-a80b-c8ebbc147a48\n",
      "TUD_23a4eb7f-1df4-4e9e-8a95-dee92f620086\n",
      "TUD_71a090f4-5c57-4a1a-a90e-aeed525f9fa2\n",
      "TUD_585a8d14-1ff6-4e45-ace4-fbc3d7b23002\n",
      "TUD_f66aa5a4-9f51-4846-9d9c-d064e942979e\n",
      "TUD_75480279-c222-4f13-addc-4afd52e38c24\n",
      "TUD_1454bdfe-ff41-4fca-9751-a4fef67233d5\n",
      "TUD_889a8830-9e61-4d21-bdef-624f7aec18e3\n",
      "TUD_976d6f08-9fe6-4eae-a636-fd0b66d5f645\n",
      "TUD_24352ff9-c11a-46c9-87d4-4d9d8968ed81\n",
      "TUD_54536c30-032d-4dcf-b909-2fc2dba88b4c\n",
      "TUD_6fe3398b-a45e-431c-876f-7e778885c454\n",
      "TUD_744a8044-94ea-48b7-b6db-e17f0d3a417d\n",
      "TUD_6b44e9c9-bb91-4e3c-94b8-d4ba8cd9408e\n",
      "TUD_3c23309a-5004-41e2-a2a9-fb1ab1a840ae\n",
      "TUD_fa826ec7-f53f-409d-b4bb-d037e674be3e\n",
      "TUD_c5eb65d2-cce5-44db-9e16-e790bd9e1945\n",
      "TUD_3c130adc-9e4a-4d54-93a0-74330b17599d\n",
      "TUD_fb2ad023-dff7-467d-b843-72a0f90a823a\n",
      "TUD_2dff4595-5fc9-4eb4-969b-b464ac59918e\n",
      "TUD_5e1071d4-9462-40f4-8673-ae24cb9ee475\n",
      "TUD_c6eeb03d-2681-46b6-a12a-58433eddd3a2\n",
      "TUD_b89f0d6f-2c73-4c6c-9c8f-ae759e4cad3f\n",
      "TUD_5f2a32a7-2851-4b7e-8c2d-d5fdfb1f8032\n",
      "TUD_e606b0a9-f273-425f-86ac-11840676e03d\n",
      "TUD_809eaec7-883c-47b0-9d57-8e605eaaaed1\n",
      "TUD_827c7123-231b-440e-95bd-a9c315efe27e\n",
      "TUD_850eb79f-b8de-47b5-832f-7c2c138787be\n",
      "TUD_3c7f869c-44b9-41f7-8deb-36ba284606cd\n",
      "TUD_ef8b0473-d699-4844-9521-855f3ff8fb71\n",
      "TUD_d4fac3ee-7e83-4c1e-93da-2875b2c20121\n",
      "TUD_b0daae03-f2ac-4780-8149-fc3509b814e5\n",
      "TUD_c9174083-4584-4074-be3b-2378bf8444e5\n",
      "TUD_88add684-df53-4f64-968c-2b44eba315d1\n",
      "TUD_4216cabc-ecd1-4987-bf1f-4c8cf3e96447\n",
      "TUD_fd6789f2-16b5-4dc9-8a12-536a0a04d3b2\n",
      "TUD_f3c0ade4-80bf-4f86-a07a-4f86a29f653d\n",
      "TUD_b933c6e5-7541-44f9-8262-ce949df9fbfd\n",
      "TUD_f669d142-ce4f-4faf-b028-c6c5775568b2\n",
      "TUD_ceb1e69d-dc14-44c5-8941-7ddc1d0365ba\n",
      "TUD_d9b49464-25f2-4354-9069-4e2534ed4152\n",
      "TUD_9040d3df-ae22-4537-82b2-4d084dd5b01c\n",
      "TUD_584cc65f-72ab-4fb8-bf11-3659867f3eec\n",
      "TUD_7fe80a02-fcf6-4e88-ad99-4dd3cf7ada2d\n",
      "TUD_d62f16d2-2139-4c94-bbb8-915f4c11537f\n",
      "TUD_aeb2c298-5a43-4d4a-9076-91e2f56d3e7f\n",
      "TUD_3200e567-a0cc-4021-bf9c-44d2c5586657\n",
      "TUD_6afeebc9-b574-453d-ac21-5682f57686bc\n",
      "TUD_d6e74f6e-ed5c-4641-ae0e-c50e0676e63a\n",
      "TUD_4e193978-6886-461a-8225-4d338c354b98\n",
      "TUD_58684b58-f0a6-4044-a70e-268d842ad7ec\n",
      "TUD_6dbf0e52-bf8a-4911-bf4b-1e7f361e037e\n",
      "TUD_bd83bae1-e1b6-4e02-bc75-72827e4ec798\n",
      "TUD_c7106f0b-5b8f-44fe-bca7-421653ec76c5\n",
      "TUD_21123648-f4ba-43e2-bbe1-04d1890f2d5e\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TUD_566b7b54-b40b-45fa-85c2-debaaa8098d4\n",
      "TUD_4439ccef-1079-43ec-96b5-d143eda4e23d\n",
      "TUD_c107f01a-3fb7-4dad-99b9-240cbdae2b90\n",
      "TUD_95a72bc6-2246-46ec-aa3d-9c222b11ceb0\n",
      "TUD_b242650e-9420-4cc2-b851-9bac0af62b0d\n",
      "TUD_eab4f47a-17c6-40c7-b55a-da09a5813cf8\n",
      "TUD_78eed80f-2d9e-4c9a-b806-40b8adb9ccda\n",
      "TUD_a5157424-63cf-4770-9e07-20c35971a84c\n",
      "TUD_7e5ccc35-774e-41d6-b12a-a502b1960256\n",
      "TUD_7bcddd81-b8de-4bcc-a069-7080bef1bdf6\n",
      "TUD_8fb5b6f6-93a6-4aed-a446-013125541ff9\n",
      "TUD_136cd4d6-396a-4520-98bf-d9dc9782f807\n",
      "TUD_bd7ab4e8-602d-4b24-a9ce-63313e47e6b0\n",
      "TUD_be7f6eee-e1e7-40d4-b2f8-5922139153b2\n",
      "TUD_20170a8c-7e1e-434f-b5d6-9ba933e2ab6e\n",
      "TUD_a3bea0c0-fc6d-47a1-9b94-bf39df4aff8f\n",
      "TUD_4f9e5af1-bbda-4685-9045-2014171fb928\n",
      "TUD_fdaf7d6a-1d45-41a5-8157-15586fa3480f\n",
      "TUD_93fba141-d466-468d-96d5-6991e5b658cf\n",
      "TUD_ae14fcf5-707e-4666-8b34-f3a91fd25f78\n",
      "TUD_db5f3d18-ce49-496a-8328-ec9203136949\n",
      "TUD_43902b9c-ed6b-4743-b00b-d8f803b111e0\n",
      "TUD_3a4b2767-6172-4eeb-a6b4-1ddae7b531c5\n",
      "TUD_01bfcad0-9a56-454b-a30e-dc2fd3394f12\n",
      "TUD_fd98efb8-8225-4eea-bf6c-5f56a2d022f5\n",
      "TUD_8b5b5120-c99b-4183-bec1-918e44860bfa\n",
      "TUD_d2bfe438-36f4-4fb6-8b51-0480f17449c2\n",
      "TUD_62da8ff0-fbfc-483a-8177-0f7b50b16c89\n",
      "TUD_5c84b35a-39d0-4c19-a905-a7bf60239138\n",
      "TUD_73541a63-93e9-44ff-a913-2fd16239bddc\n",
      "TUD_769ccdf5-5f90-4f66-a1ba-fa3a09c38ccf\n",
      "TUD_6e23606d-c5a6-4635-8f8d-ea3b2b630763\n",
      "TUD_61f7f022-60a4-4935-af13-6b5438d89c04\n",
      "TUD_0925832a-1afd-4ee3-9a3c-1c35fcad2ad5\n",
      "TUD_313c929c-9543-4037-9fa7-06645e654cf3\n",
      "TUD_f6903dcf-87e4-45a1-8a17-60699a478ab0\n",
      "TUD_3387006b-dd3c-430e-979e-f8a87e57ca11\n",
      "TUD_b8d4403e-574a-4e71-ab0d-93fd24717a66\n",
      "TUD_3e6fe824-5d96-4f1b-890d-8f9218cf41ea\n",
      "TUD_c372f6e8-92f2-4793-9b14-e4b445b1a6c8\n",
      "TUD_dc6b61d6-40fc-42d1-8205-bdfc5e1072f2\n",
      "TUD_467fe2ad-db79-4d4a-9a4b-83e7c49d9106\n",
      "TUD_1cdeaa76-3b5a-4231-9ff5-8ba7b283ba35\n",
      "TUD_3d088772-b776-49b8-ac7e-fc196783180e\n",
      "TUD_328930cc-d48e-44ae-9cfb-03aa3d0dc429\n",
      "TUD_0bd2fbdf-bdde-4c6f-8a96-c42077bb2d49\n",
      "TUD_a03b025b-1fd5-4cd1-9423-a5c1bd85fefe\n",
      "TUD_f10f3598-c47b-4f61-8827-34febd6afb45\n",
      "TUD_c45e3307-2e1c-442d-8451-8511927707de\n",
      "TUD_b6ae4f9e-89c5-4d55-87d8-aeba045b8cb3\n",
      "TUD_61b8d02d-2413-4565-bcba-d7822b0a9476\n",
      "TUD_4b956b17-a5da-4b87-99a9-aeefb2c938fe\n",
      "TUD_6ff865cc-1f70-46ad-abd6-08a9b6dccd8b\n",
      "TUD_7a248cd8-7749-4bd5-a938-ed56a7bd9992\n",
      "TUD_9e92acb7-8191-4714-90cc-6c3f2ad78df3\n",
      "TUD_36954cd6-385b-45ce-8d49-26463cfa5766\n",
      "TUD_c71c85bc-d742-449b-88e7-33e172392ec2\n",
      "TUD_de3e099b-e6d0-4a8b-aaab-aefbc1ffd707\n",
      "TUD_06d80709-ad0b-4f9b-a1cc-65634cc42d00\n",
      "TUD_16c43d66-80e3-47ce-83dc-3b8a5877368e\n",
      "TUD_2a145431-37c4-413c-9a02-c514a69695e6\n",
      "TUD_1b30ea32-f6a1-45f4-80e2-b4b6f29f6666\n",
      "TUD_650ec0d0-4613-4dae-96b1-1f685dff0e60\n",
      "TUD_c0a208a8-d012-450e-9c07-f23a39910855\n",
      "TUD_d063b265-3c59-48ac-bb4e-53dab8866cc3\n",
      "TUD_d564cdae-0a85-42aa-9251-addeb1173620\n",
      "TUD_f4ac1938-d7d8-4fa4-84ef-31e0404186f7\n",
      "TUD_6d58c3c1-51c1-4282-a9f8-888d50040755\n",
      "TUD_6bd8fee2-806b-4a95-ac66-056348f8c36c\n",
      "TUD_9b192f84-f3f7-483e-9b67-99b1480e29c7\n",
      "TUD_64ce9217-e5e5-495f-ae0a-858cc1ef5886\n",
      "TUD_bf3325ce-f246-4977-91bc-785f877347b8\n",
      "TUD_103551c5-8e23-4dd0-8b25-6e54f77b2dba\n",
      "TUD_6741902f-fc75-4bc2-91d5-5d204218c6d6\n",
      "TUD_9b2db13b-5948-4928-b3a4-c6df1ae1e9f1\n",
      "TUD_6be235a6-5a78-4481-971d-0d4b39073361\n",
      "TUD_d0b5c0b9-356f-4852-ba98-f3703831daf6\n",
      "TUD_ea39f04d-eec3-4957-a009-ab9a0a00a935\n",
      "TUD_7624a97f-cf52-4925-a6eb-6dbe112de305\n",
      "TUD_25d80828-efc2-4255-8719-1fd297380109\n",
      "TUD_a4371a0a-6176-4da6-819c-2be96222a5f1\n",
      "TUD_968f224e-3a30-4c6b-8dc6-9a4e1e43b2ec\n",
      "TUD_2e19140c-9a84-451b-a641-46c4c40ad9ac\n",
      "TUD_1c2f845a-1afe-4488-9446-97e9b4252b60\n",
      "TUD_58ad76d8-4552-461e-aa61-54299d021bd1\n",
      "TUD_2f16fbcc-1a61-4144-bcd2-72effdfc2b9d\n",
      "TUD_a93e7843-552e-432c-8133-f17b5ef6de89\n",
      "TUD_2b95ac2a-09d7-4f14-ab08-864d33e61fea\n",
      "TUD_01f0fe35-f0a8-4fec-85d1-b6c3fe2f099a\n"
     ]
    }
   ],
   "source": [
    "for doc in extracted_publications[0]:\n",
    "    _id = doc['_id']\n",
    "    uuid = _id[4:]\n",
    "    mentors = []\n",
    "    promotors = []\n",
    "    try:\n",
    "        for c in items[uuid]['contributor']:\n",
    "            if 'mentor' in c:\n",
    "                mentors.append(c)\n",
    "                publications_collection.update_one({'_id': _id}, {'$set' : {'mentors' : mentors}})\n",
    "            if 'promotor' in c:\n",
    "                promotors.append(c)\n",
    "                publications_collection.update_one({'_id': _id}, {'$set' : {'promotors' : promotors}})\n",
    "    except KeyError:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "for doc in extracted_publications[0]:\n",
    "    _id = doc['_id']\n",
    "    uuid = _id[4:]\n",
    "    publications_collection.update_one({'_id': _id}, {'$set' : {'journal' : 'tudelft'}})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'master thesis'"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "items[uuid]['type'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Indexing and ES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Got 5 Hits:\n",
      "TUD_20170a8c-7e1e-434f-b5d6-9ba933e2ab6e A machine learning approach for optimisation in railway planning\n",
      "TUD_d4d2230a-a034-43fa-bd29-0eb6b954410b Scalable machine learning algorithms on a big data infrastructure\n",
      "TUD_25aea2bb-cd25-499f-9fe6-8de69462e49d Modelling and optimization of a permanent-magnet machine in a flywheel\n",
      "TUD_33c6e093-a129-47ec-a663-3ce808343a93 Design of a high speed induction starter/generator for aerospace applications: Comparison with PM machine\n",
      "TUD_98a8d01e-6259-4fed-8704-5af0615304d2 Development of the Machine Learning-Based Approach to Access Occupancy Information Through Indoor CO2 Data: for Demand-Driven Building Heating Energy Management\n"
     ]
    }
   ],
   "source": [
    "res = es.search(index = \"ir_tud\", body = {\"query\": {\"match\": {\"title\" : \"machine\"}}}, size = 20)\n",
    "\n",
    "print(\"Got %d Hits:\" % res['hits']['total'])\n",
    "for hit in res['hits']['hits']:\n",
    "    print(hit['_id'], hit['_source']['title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import pymongo\n",
    "import elasticsearch\n",
    "from elasticsearch import helpers\n",
    "import nltk\n",
    "import sys\n",
    "import os\n",
    "import config as cfg\n",
    "\n",
    "sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "client = pymongo.MongoClient('localhost:' + str(cfg.mongoDB_Port))\n",
    "publications_collection = client.TU_Delft_Library.publications\n",
    "es = elasticsearch.Elasticsearch([{'host': 'localhost', 'port': 9200}],\n",
    "                                 timeout=30, max_retries=10, retry_on_timeout=True)\n",
    "es.cluster.health(wait_for_status='yellow', request_timeout=1)\n",
    "\n",
    "def extract_metadata(documents):\n",
    "    list_of_docs = []\n",
    "    for i, r in enumerate(documents):\n",
    "        extracted = {\n",
    "                \"_id\": \"\",\n",
    "                \"title\": \"\",\n",
    "                \"publication\": \"\",\n",
    "                \"year\": \"\",\n",
    "                \"content\": \"\",\n",
    "                \"abstract\": \"\",\n",
    "                \"authors\": [],\n",
    "                \"references\": []}\n",
    "        extracted['_id'] = r['_id']\n",
    "        extracted['title'] = r['title']\n",
    "        extracted['faculty'] = r['faculty']\n",
    "        extracted['year'] = r['year']\n",
    "        extracted['type'] = r['type']\n",
    "        extracted['authors'] = r['authors']\n",
    "        extracted['content'] = r['content']['fulltext']\n",
    "\n",
    "        try:\n",
    "            extracted['mentors'] = r['mentors']\n",
    "        except KeyError:\n",
    "            pass\n",
    "        \n",
    "        try:\n",
    "            extracted['promotors'] = r['promotors']\n",
    "        except KeyError:\n",
    "            pass  \n",
    "        \n",
    "        try:\n",
    "            extracted['abstract'] = r['content']['abstract']\n",
    "        except KeyError:\n",
    "            extracted['abstract'] = ''\n",
    "            \n",
    "        try:\n",
    "            extracted['references'] = r['content']['references']\n",
    "        except KeyError:\n",
    "            extracted['references'] = ''\n",
    "        \n",
    "        list_of_docs.append(extracted)\n",
    "    return list_of_docs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "extracted_publications = []\n",
    "query = {}\n",
    "results = publications_collection.find(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hybrid Interconnect Design for Heterogeneous Hardware Accelerators\n",
      "Modeling and Inversion of Scattered Surface waves\n",
      "Hardware Acceleration of Bioinformatics Sequence Alignment Applications\n",
      "Low-Power Receive-Electronics for a Miniature 3D Ultrasound Probe\n",
      "Removing roadblocks for mobile phone sensing\n",
      "Analyzing, Designing & Implementing a Secure, Centralized, Generic, Extensible, Input-Sanitizing, Intuitive Configuration Management Station\n",
      "Efficient numerical methods for the instationary solution of laminar reacting gas flow problems\n",
      "60 GHz Beamforming Transmitter Design for Pulse Doppler Radar\n",
      "Task Scheduling for Adaptive Reconfigurable VLIW Multicore Processors\n",
      "Implementing and evaluating a simplified transistor model for timing analysis of integrated circuits\n",
      "Food Benefit-Risk Assessment with Bayesian Belief Networks and Multivariable Exposure-Response\n",
      "Reducing Free Riding in Peer-to-Peer Systems with Supervised Teaming\n",
      "?-VEX: A reconfigurable and extensible VLIW processor\n",
      "Modelling context in automatic speech recognition\n",
      "Sampling with strict energy constraints\n",
      "Carbon nanotubes as vertical interconnects in 3D integrated circuits\n",
      "The Molen compiler for reconfigurable architectures\n",
      "Churn prediction in telecommunication\n",
      "Scheduling in Multi-Radio Sensor System Connecting ZigBee with Bluetooth Low Energy\n",
      "A flexible electronic paper with integrated display driver using single grain TFT technology\n",
      "Monitoring of and traf?c generators for networks on chip using the aethereal platform\n",
      "Automated Software Testing of JavaScript Web Applications\n",
      "Analyses and deployement of the BitTorrent protocol for community ad hoc networks\n",
      "Bioparticle Separation in Microfluidic Devices for in-Line Application\n",
      "Non-parametric extreme quantile estimation for the common shaped tail model: Forecasting extreme precipitation by post-processing precipitation from a numerical weather prediction model\n",
      "Counting sequences, Gray codes and lexicodes\n",
      "ANTARES: Automatic Diagnosis of Software/Hardware Systems\n",
      "Enabling Big Data Analytics For MATLAB Programs Using High Performance Compute Methods\n",
      "Hardware and Software Design of the Microthrust Power and Control System\n",
      "Asynchronous Logic as Counter Measure against Power Analysis Attacks\n",
      "Low-Power Low-Noise CMOS Imager Design: In Micro-Digital Sun Sensor Application\n",
      "Random graphs: From static to dynamic\n",
      "Using Dynamic Bayesian Networks for Posed versus Spontaneous Facial Expression Recognition\n",
      "Bayesian estimation for ionospheric calibration in radio astronomy\n",
      "WideBand PLL as a clock multiplier\n",
      "Receiver Windowing Design for Narrowband Interference Mitigation in MB-OFDM UWB System\n",
      "U-Sem, a platform for augmented user modelling\n",
      "Estimating Long-Term Uncertainties of Economic Variables\n",
      "Technology Aware Network-on-Chip Connectivity and Synchronization Design\n",
      "Computing the Spectrum of the Confined Hydrogen Atom\n",
      "No-Reference Image Quality Assessment using Deep Convolutional Neural Networks\n",
      "Accelerated HaplotypeCaller DNA Analysis Application on FPGAs Using CAPI\n",
      "Arithmetic soft-core accelerators\n",
      "Low-Noise CMOS Image Sensors for Radio-Molecular Imaging\n",
      "Ultra Linear Low-loss Varactors & Circuits for Adaptive RF Systems\n",
      "3DTV Rendering from Multiple Cameras\n",
      "Interactive Visualization of Functional Brain Connectivity\n",
      "Reaching a higher sense of presence in VR through 3D architectural visualization\n",
      "Edit Distance-Based Classification of Symbol Sequences\n",
      "Design and Characterization of the Time Division Multiplexing concept on a dual clock Imaging DSP\n",
      "Comparing Integrators on Perceived Realism and Quality\n",
      "Unmanned aircraft systems HMI and automation; tackling control, integrity and integration issues\n",
      "Characterization of complex networks: Application to robustness analysis\n",
      "Aluminum Nitride and Chromium Nitride Thin Films for Strain Gauge Application\n",
      "Acceleration of the Multi Level Fast Multipole Algorithm on a Graphics Processing Unit\n",
      "Design of a high speed induction starter/generator for aerospace applications: Comparison with PM machine\n",
      "Why and How JavaScript Developers Use Linters\n",
      "Pauli Frames for Quantum Computer Architectures\n",
      "Circuit and interconnect design for high bit-rate applications\n",
      "Detection of Fetal Copy Number Aberrations by Shallow Sequencing of Maternal Blood Samples\n",
      "Substrate Crosstalk Suppression Using Wafer-Level Packaging: Metalized Through-Substrate Trench Approach\n",
      "Sensor management for surveillance and tracking: An operational perspective\n",
      "An Exploratory Study on Faults in Web API Integration\n",
      "Tracking known security vulnerabilities in third-party components\n",
      "Improving compression of DNA sequencing data by clustering on quality score alphabet\n",
      "Designing Procedurally Generated Levels\n",
      "Reliability study of the Floating Gate Based Embedded Non?Volatile Memory (eNVM)\n",
      "Fairness and resource allocation in device-to-device wireless regional area network\n",
      "Offloading mobile data traffic to Wi-Fi through MASS-ANET: Mobile assisted ad-hoc networking\n",
      "GPU-based simulation of brain neuron models\n",
      "Porting GCC to a Clustered VLIW Processor\n",
      "Identification of Room Boundaries for Sound Field Estimation\n",
      "OS evaluation for real time image processing on a multicore architecture\n",
      "Regularized Least Squares Imaging for High Resolution Ultrasound\n",
      "A Highly Selective, Very Linear Low Noise Transconductance Amplifier Capable of Large-Signal Handling for Current-Mode Receivers Front-End\n",
      "Libswift-PPSPP Information Centric Router: SHA1 Accelerator\n",
      "Stochastic integration in Banach spaces and applications to parabolic evolution equations\n",
      "Applying Architecture and Ontology to the Splitting and Allying of Enterprises\n",
      "Electricity load modelling using computational intelligence\n",
      "Quantitative Application Data Flow Characterization for Heterogeneous Multicore Architectures\n",
      "8uW to 1mW Input Power Management IC Design for RF Energy Harvester\n",
      "Experimental and Computational Study of a Wind Powered Reverse Osmosis System\n",
      "Pearson codes design with error-control capabilities\n",
      "Modularization and Specification of Service-Oriented Systems\n",
      "Sensorless control of Permanent Magnet Synchronous Motor at low speed\n",
      "Population graph for Mycobacterium Tuberculosis\n",
      "Performance Evaluation of Distributed SQL Query Engines and Query Time Predictors\n",
      "A hardware Accelerator for the OpenFOAM Sparse Matrix-Vector Product\n",
      "On the Central Path of Redundant Klee-Minty Problems\n",
      "Hardware implementation of digital signal processing algorithms for long distance pipeline inspections\n",
      "High-speed parallel processing on CUDA-enabled Graphics Processing Units\n",
      "Level-Crossing ADCs and Their Applications in Biomedical Readout Systems\n",
      "Detecting Code Smells in Android Applications\n",
      "Code Integrity Check targetting RISC Processors\n",
      "Abstractions for Asynchronous User Interfaces in Web Applications\n",
      "Scheduling the Spark Framework under the Mesos Resource Manager\n",
      "On the influences of personality traits on employees engagement with gamified enterprise tools\n",
      "A probabilistic model for oil spill volume in tanker collisions and groundings\n",
      "Sparsity-Aware Wireless Networks: Localization and Sensor Selection\n",
      "Multi-Atlas Based Image Segmentation Applied on Pulmonary CT Scan Data\n",
      "Image reconstruction in low-field MRI: A super-resolution approach\n",
      "Silicon doping techniques using chemical vapor dopant deposition\n",
      "Efficient Runtime Management of Reconfigurable Hardware Resources\n",
      "An enterprise ontology based approach to service specification\n",
      "SRAM power reduction: An ultra-low-power SRAM architecture in 45nm technology\n",
      "Hardware acceleration of short-read mapping with the Burrows-Wheeler Aligner\n",
      "Energy-Efficient Self-Timed Zero-Crossing-Based Incremental Delta-Sigma ADC\n",
      "Full-Newton step interior-point methods for conic optimization\n",
      "Modelling and Analysis of Execution Traces from Real-time Applications\n",
      "Basora Enterprise Search: A new system for enterprise information retrieval\n",
      "Towards high-level synthesis of neural models on FPGAs\n",
      "VHDL to SystemC: The Design of a Translator\n",
      "Implicit Parallelization of the Nested Relational Calculus for Semi-Structured Data\n",
      "Private Computing and Mobile Code Systems\n",
      "Small-Animal PET Detector Module Based on Multichannel Digital Silicon Photomultipliers\n",
      "Porting the GCC Compiler to a VLIW Vector Processor\n",
      "A 12-bit 500MS/s Pipeline Split-ADC\n",
      "A Generic and Automatic Test Strategy for Compiler Testing\n",
      "Designs and algorithms for packet and content inspection\n",
      "Acceleration of read alignment with coherent attached FPGA coprocessors\n",
      "Investigations into Informed Phase Restoration of Amplitude Spectra for Audio Signals\n",
      "Improving the scalability of multicore systems with a focus on H.264 video decoding\n",
      "Facility Location Problems: Approximation Algorithms and Polyhedral Results\n",
      "Wireless LANs; Protocols, Security and Deployment\n",
      "Evaluation of DNA scaffolding techniques using PacBio long reads\n",
      "Parallel parsing\n",
      "Grace in Spoofax\n",
      "Electrical characterisation of matched pairs for evaluation of integrated circuit technologies\n",
      "A Low-Complexity CMOS Receiver for UWB signals\n",
      "Interfacing the ?-VEX with the µBlaze processor\n",
      "Optimization and Energy Management of a Microgrid Based on Frequency Communications\n",
      "Renovation of idiomatic crosscutting concerns in embedded systems\n",
      "Mark formation model for optical rewritable recording\n",
      "The Design and Implementation of a Virtual Reality Exposure Therapy System for Patients Suffer from Paranoia\n",
      "Context-Aware Recommender Systems in the E-commerce Domain\n",
      "The efficient computation of the mutual coupling in large, non-periodic array antennas\n",
      "Modal wavefront correctors based on nematic liquid crystals\n",
      "A Custom FFT Hardware Accelerator for Wave Fields Synthesis\n",
      "Reference integration architecture for mobile enterprise solutions\n",
      "Accelerating Software Pipelines Using Streaming Caches\n",
      "Constructing a Test Code Quality Model and Empirically Assessing its Relation to Issue Handling Performance\n",
      "Scalable Video Coding\n",
      "Generic tools, specific languages\n",
      "Energy-Efficient Capacitive-Sensor Interfaces\n",
      "Designing a privacy aware infrastructure for an Inclusive Enterprise at IBM\n",
      "A fault tolerant memory architecture for a RISC-V softcore\n",
      "Solving the Overproduction Problem in Reactive Programming using Feedback Control\n",
      "Higher order elliptic problems and positivity\n",
      "Design of a Baseband Section for LTE-Advanced Mobile Communiation\n",
      "Characterization of CMOS Image Sensor\n",
      "Quantitative analysis and visualization of memory access patterns\n",
      "Full-Duplex CMOS Duplexer: Passive self-interference cancellation\n",
      "Design, Modeling and Simulation of a 52MHz MEMS Gyroscope Device in 1.5um SOI\n",
      "Credit Portfolio Losses\n",
      "Ultra Low-Power Analog Integrated Circuits for Extracellular Action Potential Detection\n",
      "A Generative Approach for Data Synchronization between Web and Mobile Applications\n",
      "Validation of Performance Estimation, Channel Sizing and Automatic Loop Transformations of High Level Specifications for Polyhedral Process Network\n",
      "Experimental Analysis on ECC Schemes for Fault-Tolerant Hybrid Memories\n",
      "A Client/Server Framework for Interactive Remote 3D Visualization of Histological Data\n",
      "Error subspace filtering for atmospheric chemistry data assimilation modeling\n",
      "Population graph of Mycobacterium Tuberculosis\n",
      "Procedural Vessel Hull Generation\n",
      "High Data-rate Communication via Near-field EM transfer between Integrated Loop Antennas\n",
      "Computational Assessment of Single-Molecule Protein Sequencing\n",
      "Geometric Distortion in Image and Video Watermarking. Robustness and Perceptual Quality Impact\n",
      "Sybil-resistant trust mechanisms in distributed systems\n",
      "On the Effect of Code Quality on Agile Effort Estimations: The Case of Shell\n",
      "A Cache-Based Hardware Accelerator for Memory Data Movements\n",
      "Fundamental Properties of Wireless Mobile Ad-hoc Networks\n",
      "Field Programmable Gate Arrays with Hardwired Networks on Chip\n",
      "Model-driven evolution of software architectures\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hierarchically semi-separable representation and its applications\n",
      "Random Subspace Method for One-Class Classifiers\n",
      "DotFETs: MOSFETs strained by a Single SiGE dot in a Low-Temperature ELA Technology\n",
      "Advanced Retrieval Models for Web Image Search\n",
      "First approach for a decentralized resource allocation in ad hoc grids\n",
      "K-loops: Loop Transformations for Reconfigurable Architectures\n",
      "Optimization of on-site PD measurements and evaluation of diagnostic parameters for assessing condition of distribution cable system\n",
      "Ferroelectric Materials and Artificial Dielectric Layer Structures for Microwave Integrated Circuit Technologies\n",
      "Modeling and optimization of Nb tuning circuits for Nb/AlOx/Nb SIS mixers in the 600-720 GHz range\n",
      "A Universal Fault Injector for Structural Simulation Environments\n",
      "Highly Linear LNA Design for Base Station Applications\n",
      "Integration of existing optimisation techniques with the DWARV C-to-VHDL compiler\n",
      "Enterprise Expertise Characterization\n",
      "Exploiting the Reconfigurability of ρ-VEX Processor for Real-Time Robotic Applications\n",
      "Dynamic improvement of robustness of power transmission grids in decentralized and distributed environments\n",
      "Free-riding Resilient Video Streaming in Peer-to-Peer Networks\n",
      "Composable and Predictable Power Management\n",
      "Particle-based Visualization of Soft-Tissue: Deformation in the Human Orbit\n",
      "Wireless communication systems using signal space diversity\n",
      "Probabilistic Simulation of Residential Household’s Load Profiles\n",
      "Deterministic execution of multithreaded applications for reliability of multicore systems\n",
      "Connection of large-scale wind power generation to the Dutch electrical power system and its impact on dynamic behaviour\n",
      "Shared Perspectives on Perceptual Features for Query by Example in E-Commerce Systems\n",
      "Identifying shared functionality using latent semantic indexing\n",
      "A noise subspace approach for localization\n",
      "SIMD Floating Point Extension for Ray Tracing\n",
      "Distributed G-MAC: A flexible MAC protocol for servicing Gossip Algorithms\n",
      "High-Aspect-Ratio CMOS add-on modules for RF passive components\n",
      "Flexible Power Transfer Network\n",
      "Detecting Breaking Changes in JavaScript APIs\n",
      "High-order Material Point Method\n",
      "Residual Current Protection of a Meshed DC Distribution Grid with Multiple Grounding Points\n",
      "A SoC solution for fingerprint minutiae extraction\n",
      "Mean-variance optimization for life-cycle pension portfolios\n",
      "Assuring the Quality of Unit Testing in a Continuous Delivery Environment\n",
      "Scalable Multi-core Architectures for Data Mining Applications\n",
      "Efficient solution methods for the incompressible Stokes problem\n",
      "Interactive visualisation techniques for large time-dependent data sets\n",
      "A Selection Method for Model-Driven Development Tools\n",
      "Load Side Detection of Series Arcs in DC Microgrids by Simulation and Experimental Validation\n",
      "Low Temperature Fabrication of SiO2 Films Using Liquid Silicon\n",
      "Wavefront Reconstruction from Intensity Based Images for High Resolution Microscopy\n",
      "Automated security testing of AJAX web widgets\n",
      "MePoEfAr: Memory & Power Efficient Architecture for Embedded Microcontrollers\n",
      "Identifying and Managing Technical Debt in Complex Distributed Systems\n",
      "A Domain Specific Language for Computation-in-Memory\n",
      "Incentivizing Seeding In BitTorrent - Indirect Interaction as an Incentive to Seed\n",
      "On the determination of approximations of first integrals for few-body gravitational problems: With applications to capture trajectories\n",
      "Design and Simulation of Single Electron Tunneling Circuits for Brownian Motion Based Logic and Arithmetic Computation\n",
      "Wireless Transceiver Design for High Velocity Scenarios\n",
      "A step forward in the theory and practice of ICT management simulation\n",
      "An architecture for task execution in adverse environments\n",
      "Optimized Permanent Magnet Generator Topologies for Direct-Drive Wind Turbines\n",
      "Wafer Level Packaging With Nano Metal Paste Interconnects For UV-C LEDs\n",
      "Use of high-throughput DNA data for discovery of unknown genes in Zebrafish\n",
      "Towards Dependable Network-on-Chip Architectures\n",
      "Security in RFID Systems: Theory and practice for enhancing security in RFID systems\n",
      "On Achieving the Minimum Energy for Sending One Single Bit with Feedback\n",
      "Static Code Analysis Tools: Effects on Development of Open Source Software\n",
      "Prototyping of a Stretchable Microelectrode Array for Cardiotoxicity Drug Screening\n",
      "3D Information Extraction Based on GPU\n",
      "Optical analysis of c-Si quantum dots embedded in a silica matrix\n",
      "Coordinating autonomous planning and scheduling\n",
      "Development of a workload set for multi-core architectures\n",
      "Studies in Physical Optics: Coherence Theory and Surface Plasmons\n",
      "Task Scheduling Methods for Composable and Predictable MPSoC\n",
      "Prediction of Team Cohesion, by examining prosodic mimicry in small group meetings\n",
      "Tile-Based rasterization on an embedded Tile-based MPSoC\n",
      "Parallelization of Ensemble Kalman Filter (EnKF) for Oil Reservoirs\n",
      "Visual Instance-Based Modeling of Procedure Specifications for Architectural Structures\n",
      "Adding Cycle Scavenging Support to the Koala Grid Resource Manager\n",
      "Analyzing the Evolution of WSDL Interfaces using Metrics\n",
      "Composable Virtual Platforms for Mixed-Criticality Embedded Systems\n",
      "Pricing multi-asset options with sparse grids\n",
      "Employing Mechanism Design for the Online Allocation of Items\n",
      "Optimizing Greenhouse Heat Production in Lansingerland: Using Simulated Annealing and Simulation\n",
      "Visualize local neighborhood for supporting debugging\n",
      "Wind Speed Modeling\n",
      "Improving packaging and increasing the level of integration in power electronics\n",
      "Avoiding conversion and rearrangement overhead in SIMD architecures\n",
      "Monolithic Integration of Rectifiers and Drivers for low power SSL applications on a Rigid to Flex Substrate\n",
      "Advanced Breakdown Modeling for Solid-State Circuit Design\n",
      "High-Level Power Estimation and Optimization of DRAMs\n",
      "The clamped elastic grid, a fourth order equation on a domain with corner\n",
      "Recursive variable expansion: A transformation for reconfigurable computing\n",
      "A texture analysis of 3D GPR images\n",
      "Ontology driven Enterprise Information Systems Engineering\n",
      "Landmark Influence Visualization in Active Shape Models\n",
      "Microchip electrophoresis for UVC-induced DNA damage assessment\n",
      "Detecting Multiple-Accounts Cheating in MOOCs\n",
      "RNS support for RSA cryptography\n",
      "Analysis and recording of multimodal data\n",
      "A study to end-winding induced iron losses in high-speed electrical machines with concentrated windings\n",
      "High Efficiency RF Power Amplifier Architectures\n",
      "Exploring suitable Adder Designs for Biomedical Implants: A gracefully-degradable, fault-tolerant, and highly resource-constrained adder for SiMS\n",
      "Linked open government data\n",
      "Detection of Duplicate Content on Twitter\n",
      "Protocol conversions for the Aethereal Networks-on-Chip\n",
      "Scaling and Modelling Photoluminescence of Si Nanoparticles Embedded in Silicon Oxide\n",
      "Excitation and readout of a thermally driven time-domain optical coherence tomography system\n",
      "Towards a Systematic Exploration of the Optimization Space for Many-Core Processors\n",
      "Coordination for Agents with Freedom of Choice\n",
      "Integration of stochastic generation in power systems\n",
      "Fractals and moving plumes\n",
      "Eliciting and Representing Joint Distributions From Experts: Quantification of a Human Performance Model for Risk Analysis\n",
      "High Performance OpenCL Implementation of Medical Image Processing Algorithms [CP]\n",
      "Performance Analysis of Chainsaw-based Live P2P Video Streaming\n",
      "Electronic DC transformer with high power density\n",
      "Social Temperature Sensing\n",
      "Enhancer-gene networks for identification of cancer driver genes affected by enhancer mutations\n",
      "Orientation-and-Location Controlled Single-Grain TFTs on Glass Substrate\n",
      "Computational approaches for the MINLP waste water network model\n",
      "Frequency References Based on the Thermal Diffusivity of Silicon and Silicon Dioxide\n",
      "Voltage calibration of BJT-based temperature sensors\n",
      "Design and development of active baseband loadpull system\n",
      "A Low-Cost Universal Integrated Interface for Capacitive Sensors\n",
      "Next-generation neuromodulator for epilepsy prevention\n",
      "Design of an Energy-Efficient Interface Circuit for a MEMS-based Capacitive Pressure Sensor\n",
      "Design of a Portable and Customizable Microprocessor for Rapid System Prototyping\n",
      "DEMO applied to Quality Management Systems\n",
      "Visualization of SysML Project Meta-Model Architecture and Evolution\n",
      "The Essence of Reactive Programming: A Theoretical Approach\n",
      "Example-Based Approach For Real-Time Rigged Character Animation\n",
      "Individual Cell Analysis of Micromorph Tandem Solar Cells\n",
      "MEP-MAS: A Message Passing Multiprocessor Array for Streaming Applications\n",
      "Intelligent methods for automated video surveillance\n",
      "Learning to recognise: A study on one-class classification and active learning\n",
      "Energy reduction techniques for caches and multiprocessors\n",
      "Silicon Technology for Integrating High-Performance Low-Energy Electron Photodiode Detectors\n",
      "Optimal Role Assignment in Heterogeneous Juror Panels\n",
      "Visual Analysis for Sports Sensor Data\n",
      "A Novel Concurrent Validation Scheme for Hardware Transactional Memory\n",
      "Runtime Support for Heterogeneous Multi-core Systems\n",
      "The Impact of Test Code Summary to Understand the System Behaviour\n",
      "Methodology for Deploying Secure Applications in the Cloud\n",
      "Measuring the internet\n",
      "On aspects of boundary damping for cables and vertical beams\n",
      "Cost Effective Modular Adders for RNS-based Processors\n",
      "Profiling symmetric encryption algorithms for implantable medical devices\n",
      "Constraint-based feature validation\n",
      "Understanding and Improving the Performance Consistency of Distributed Computing Systems\n",
      "Noise in sub-micron CMOS image sensors\n",
      "Supporting forensic investigators on exploring web forums through Interactive Topic Modeling\n",
      "Inferring the influence of cultivation parameters on transcriptional regulation\n",
      "Splash: Data synchronization in unmanaged, untrusted peer-to-peer networks\n",
      "Reliability Assessment and Test Methods for Anti-counterfeiting Technology\n",
      "New method for future transmission system bottleneck indentification for interconnected power systems\n",
      "3D Transesophageal echocardiography using a fast-rotating transducer\n",
      "FPGA accelerated Facial Recognition\n",
      "IWEX: Mapping on current high performance architectures\n",
      "LC Ladder Based Orthonormal Filter for Impulse-Radio UWB Pulse Generation.\n",
      "Automated Implant-Processor Design\n",
      "Efficient High Performance Computing on Heterogeneous Platforms\n",
      "Effcient Simulation and Valuation of Embedded Options using Monte Carlo Simulations\n",
      "Customizing and hardwiring on-chip interconnects in FPGAs\n",
      "Image Reconstuction in MRI: The Possibilities of Portable Low-cost MRI Scanners\n",
      "A Scalable Mextram Model for Advanced Bipolar Circuit Design\n",
      "Ray Tracing NURBS Surfaces using CUDA\n",
      "Constructing snake-in-the-box codes and families of such codes covering the hypercube\n",
      "Market risk calculations in stock- and bond prices: a garch-copula approach\n",
      "Accelerated Mean Shift for static and streaming environments\n",
      "Trace-based performance analysis of scheduling bags of tasks in grids\n",
      "A Framework For Coping With Intra-Node Adjacent Channel Interference In Multi- Channel Multi-Radio Mesh Networks\n",
      "Quantitative hardware prediction modeling for hardware/software co-design\n",
      "A Study of Synchronization Issues of Wavelet Packet based Multicarrier Modulation\n",
      "Design of a Distributed Clock Generator for Multiple Power Domain System-on-Chip Integrated Circuits\n",
      "Vector-Based Semantic Expansion Approach: An Application to Patent Retrieval\n",
      "Arbitrage-free methods to price European options under the SABR model\n",
      "Sparse Matrix Vector Processing Formats\n",
      "Analytical cost estimation for embedded systems\n",
      "Archived film analysis and restoration\n",
      "Design and Prototype of the Range-Trie IP: Lookup in the HTX reconfigurable platform\n",
      "Hardware Algorithms For Tile-Based Real-Time Rendering\n",
      "Advances in DFT-Based Single-Microphone Speech Enhancement\n",
      "Electro magnetic compatibility of cabling and wiring in buildings and installations\n",
      "Introducing Enterprise Architecture at TNT Post\n",
      "Analyzing the Change-Proneness of APIs and web APIs\n",
      "OFDM transmissions over rapidly changing channels\n",
      "The electrical, thermal and spatial integration of a converter in a power electronic module\n",
      "The Furn: Organizing furniture throughout the web\n",
      "Power-aware architecting for data-dominated applications\n",
      "Comparison of optical inter-domain routing protocols in the presence of wavelength converters\n",
      "Ontology visualization and instance matches confirmation\n",
      "Scalability of Bioinformatics Applications for Multicore Architectures\n",
      "Multivariable feedback control of a Dividing Wall Column\n",
      "A comparison of supervised gene set searching algorithms for outcome prediction of breast cancer\n",
      "Towards Quantitative Spatio-Temporal Gene Expression Measurements in Mice\n",
      "Collaborative Robot Agents\n",
      "Development of the Machine Learning-Based Approach to Access Occupancy Information Through Indoor CO2 Data: for Demand-Driven Building Heating Energy Management\n",
      "Dynamic and Transient study in Mid Voltage networks in relation to Distributed Generations\n",
      "A finite volume solver for the simulation of transport processes\n",
      "Modeling SRAM Start-up Characteristics For Physical Unclonable Functions\n",
      "PRAGMA: A Partial-Reconfigurable Audio Platform\n",
      "Energy-efficient readout of resonant sensors\n",
      "An Ultra Low Power Fully Integrated Sensor Interface IC for Pacemakers\n",
      "Scheduling in high performance buffered crossbar switches\n",
      "A Double-mode Linear Imaging System for IC-compatible Microspectrometers in Visible Light\n",
      "Charge Domain Interlacing CMOS Image Sensor Design\n",
      "A Study of Joint Network Coding and Power Control in Energy Harvesting Networks\n",
      "Fault Detection for the Delfi Nanosatellite Programme\n",
      "Effective reverse conversion in residue number system processors\n",
      "Resource Allocation in Time-Division-Multiplexed Networks on Chip\n",
      "Markov processes for maintenance optimization of civil infrastructure in the Netherlands\n",
      "Spectrum Sensing Issues for Cognitive Radio\n",
      "Combining and evaluating regression methods for galaxyredshift estimates\n",
      "Low Power Evaluation for Arbitration and MPSoC\n",
      "Efficient pre-filtering techniques for packet inspection\n",
      "Performance analysis of SCISM organization applied to the IA-32 architecture\n",
      "Mobility-based Time References for Wireless Sensor Networks\n",
      "Using the Multiple Instance Learning framework to address differential regulation\n",
      "A Probabilistic Approach to Symbolic Performance Modeling of Parallel Systems\n",
      "Testability and Fault Tolerance for Emerging Nanoelectronic Memories\n",
      "Trellis-based source and channel coding\n",
      "A tailored fMRI image processing pipeline based on ITK\n",
      "Integration of 3D tracking systems for Interaction in Spatial Augmented Reality\n",
      "Exploiting Embedding in Content-Based Recommender systems\n",
      "Finite Element Analysis of Human Bone Structures on the Cell Broadband Engine\n",
      "Timing-Driven chip design\n",
      "BibMix: Enrichment of Citation Metadata Based on Integration of Bibliographic Data\n",
      "A workload model for MapReduce\n",
      "Design space exploration for a Local Object Store\n",
      "Security of the European Electronic Patient Record\n",
      "Performance estimation technique for optimizing and integrating IPs in MPSoCs\n",
      "Impairment-Aware QoS Routing in Translucent Optical Networks\n",
      "Parameters identification of brushless doubly-fed induction generators\n",
      "Hierarchical Memory Diagnosis Approach\n",
      "Identification and Elucidation of Expression Quantitative Trait Loci (eQTL) and their regulating mechanisms using Decodive Deep Learning\n",
      "Rate-Distortion Optimal Time-Frequency Decompositions for MDCT-based Audio Coding\n",
      "Exploring an alternative concept to cope with unknown terrain database integrity in an SVS\n",
      "Analysis and Test Development for Parasitic Fails in Deep Sub-Micron Memory Devices\n",
      "Integration of large scale wind power generation in the Dutch electricity system with interconnections to neighboring power systems\n",
      "Feature conversion for concurrent engineering\n",
      "Dead code elimination for web applications written in dynamic languages\n",
      "Using the marginal structural model for finding the best moment to treat: Applied to a dataset consisting of dialysis patients treated with Erythropoiesis-Stimulating Agents\n",
      "A New Test Paradigm for Semiconductor Memories in the Nano-Era\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Algorithms for Simple Temporal Reasoning\n",
      "Flash memory device: Electrical modeling and simulation\n",
      "90 nm VLSI Design of an 8-bits Microcontroller\n",
      "Comparing Feature Sets and Classifiers for Sentiment Analysis of Opinionated Free Text\n",
      "What Does Passive Learning Bring To Adyen?\n",
      "Performance Validation of Networks on Chip\n",
      "SmArT solving: Tools and techniques for satisfiability solvers\n",
      "Application of Monte Carlo Simulation for risk-based decision making in MV distribution networks\n",
      "Kafkalytics: Anti-pattern detection in a Kafka message bus\n",
      "Ecological approach to develop a shared mental model for collaboration at a distance during a crisis situation\n",
      "PPU: A Protocol Parsing Unit in Hardware\n",
      "Testing of modern semiconductor memory structures\n",
      "Performance Analysis of WebRTC-based Video Conferencing\n",
      "Shai: Scenario authoring for simulation games\n",
      "Evaluation Framework for Task Scheduling Algorithms in Distributed Reconfigurable Systems\n",
      "The impact of event-triggered control on the energy consumption of a legged robot\n",
      "Dielectric Relaxation Analysis to Assess the Integrity of High Voltage DC Cables of the Mass Impregnated Type\n",
      "High-precision Sigma-Delta Modulator with improved linearity\n",
      "Co-simulation of Demand-side Management and Load Flow Analysis tools in a LV network: Mitigating Over & Under Voltages\n",
      "Design, Analysis and Experimental Evaluation of a Distributed Community Detection Algorithm\n",
      "Stability analysis of transmission system with high penetration of distributed generation\n",
      "A comprehensive study of Dynamic Memory Management in OpenCL kernels\n",
      "LTE Performance Measurement In Trial Network & Validation Of LTE Performance Estimation Models\n",
      "An Off-Chip Bridge for On-Chip Network-Based Systems Supporting Traffic Quality of Service\n",
      "Implementing Texture Feature Extraction Algorithms on FPGA\n",
      "Characterization of Human Blockage in 60 GHz Communication\n",
      "Fault-Tolerant On-Board Computer Software for the Delfi-n3Xt Nanosatellite\n",
      "Single Pick-Up Dual Frequency IPT Systems\n",
      "Interactive learning for video content analysis\n",
      "Rewarding Good Behavior in Peer-to-Peer Networks\n",
      "Field Of Threads: Master’s Thesis on a tool for clarifying thread behavior\n",
      "System-Level Design Space Exploration of Reconfigurable Architectures\n",
      "DRAM fault analysis and test generation\n",
      "Scientific visualization in virtual reality: Interaction techniques and application development\n",
      "Design and Characterization of the Time Division Multiplexing concept on a dual clock Imaging DSP\n",
      "Modeling of traffic on fairways: Optimizing the allocation of traffic\n",
      "Assessment of detection limits in viral diversity studies using 454 amplicon sequencing\n",
      "Semantic navigation in video games\n",
      "Direct and comparative visualization techniques for HARDI Data\n",
      "Analyzing web applications by visualizing their state-flow graphs\n",
      "Ultra-Low-Power Event-Driven Radio Design\n",
      "Influence of Auxiliary Features in Factorization-based Collaborative Filtering\n",
      "Advancing sustainable transportation by charging EVs with PV power at the workplace: An optimal charging strategy\n",
      "Fine-grained Performance Evaluation of Large-scale Graph Processing Systems\n",
      "Enhanced visualization for improved lesion diagnosis in virtual colonoscopy\n",
      "Enhanced Question Classification with Optimal Combination of Features\n",
      "Handoff Management in Radio over Fiber 60 GHz Indoor Networks\n",
      "Modelling Robust Design Problems via Conic Optimization\n",
      "Jordan algebraic approach to symmetric optimization\n",
      "Abstract reduction operation models in the LIME programming model\n",
      "Consolidated Deep Actor Critic Networks\n",
      "A Co-processor for a Secure Implantable Medical Device\n",
      "Scaling Distributed Virtual Environments using Socially Aware Load Partitioning and Interest Management\n",
      "Nonblocking Distributed State-Tree-Structures\n",
      "Finding Errors in Massive Payment Log Data\n",
      "Exploring Convolutional Neural Networks on the ρ-VEX architecture\n",
      "Super-resolution Algorithms for Joint Range-Azimuth-Doppler Estimation in Automotive Radars\n",
      "Implementing Virtual Address Hardware Support on the ?-VEX Platform\n",
      "Quality of Service Routing in the Internet. Theory, Complexity and Algorithms\n",
      "Implementation of Bio-Informatics Applications on Various GPU Platforms\n",
      "Usable and adaptable business software\n",
      "Congestion Management with feedback queue\n",
      "Managing clones using dynamic change tracking and resolution: Helping developers to cope with changing clone fragments\n",
      "Automated detection of performance regressions in web applications using association rule mining\n",
      "Design & Modeling of a MEMS Contact-mode Piezoresistive Detector for Pull-in Acceleration Sensing\n",
      "An Accelerator based on the ?-VEX Processor: An Exploration using OpenCL\n",
      "Optimizing truck driver schedules with dependent working shifts, drivers' legislation, and multiple time windows\n",
      "Investigation on Capacitive Sensor Interface with Improved Immunity to External Interference\n",
      "Quantization effects in PDMM: A first study for synchronous distributed averaging\n",
      "Permanent pixels: Building blocks for the longevity of digital surrogates of historical photographs\n",
      "The Strength of Preference\n",
      "Energy Efficient and Compact RF High-Power Amplifiers\n",
      "Mechanisms for Self-organizing Ad Hoc Grids\n",
      "A Qualitative Comparative Study between Posh and GOAL for Programming UT2004 Bots\n",
      "Round-trip Business Process Driven SOA modelling between ARIS and Cordys\n",
      "Performance-Oriented Fault Tolerance in Computing Systems\n",
      "Social Web Data Analytics: Relevance, Redundancy, Diversity\n",
      "Design of 2.25 MW, permanent magnet direct drive generator for wind energy applications with concentrated windings and reduction of eddy current losses in the rotor back iron\n",
      "Performance Analysis of Middleware for Component Based Robot Control Software\n",
      "Developing a project architecture and evaluating the Enterprise Architecture initiative at Tata Steel\n",
      "Inductively powered implant for monitoring and application of telemetric metronomic photodynamic therapy\n",
      "Performance Modeling of Stochastic Diagnosis Engines\n",
      "On the effective permittivity of finite inhomogeneous objects\n",
      "Pricing Methods in a LIBOR Market Model with Stochastic Volatility\n",
      "Hydroelastic analysis of very large floating structures\n",
      "Extracting information from collections of musical performances using Dynamic Time Warping\n",
      "Single grain Si TFT's and circuits based on the µ-Czochralski process\n",
      "FMCW radar receiver front-end design\n",
      "End User Involvement in Exploratory Test Automation for Web Applications\n",
      "Advanced GPR data processing algorithms for detection of anti-personnel landmines\n",
      "CMOS bandgap references and temperature sensors and their applications\n",
      "Modeling management of research and education networks\n",
      "A Low-Power CMOS Wind Sensor with Corner Heaters\n",
      "Design Partitioning for Custom Hardware Emulation\n",
      "Custom Architecture for Immersive-Audio Applications\n",
      "New Primal-dual Interior-point Methods Based on Kernel Functions\n",
      "A Quantitative Model for Hardware/Software Partitioning\n",
      "The Unified Anatomical Human: Model-based Visualization of Heterogeneous Anatomy Data\n",
      "Detecting a risk mismatch between actual investment portfolio and its strategic asset allocation\n",
      "A 10-bit 25Msps Pipeline ADC for Companding Baseband Processing in Wireless Application\n",
      "Exploring Characteristics of Code Churn\n",
      "Constructing a Confidential, Authenticated, Forward Secure and Offline Logging Scheme\n",
      "Planar elliptically shaped dipole antenna for UWB Impulse Radio\n",
      "Communication Service and Dynamic Memory Management for hardware tasks executed on Dynamic and Partial Reconfigurable resources\n",
      "Stochastic Scheduling of Train Maintenance Projects\n",
      "Interpretable classification of tumours through multiple instance learning and somatic mutations\n",
      "Chireix's / LINC Power Amplifier for Base Station Applications Using GaN Devices with Load Compensation\n",
      "On-line testing of routers in networks-on-chip\n",
      "A fully differential switched capacitor wavelet filter\n",
      "A Framework to assess Data Quality in university web portals\n",
      "Opportunistic Routing for Indoor Energy Harvesting Wireless Sensor Networks\n",
      "Antennas for Frequency Reconfigurable Phased Arrays\n",
      "Run-Time Partial Reconfiguration on the Virtex-II Pro\n",
      "Attack-resilient media using phone-to-phone networking\n",
      "Throughput-constrained Voltage and Frequency Scaling for Real-time heterogeneous multiprocessors\n",
      "Thermal management of High-Current Bidirectional Isolated DC-DC Converters\n",
      "Modelling information diffusion based on Epinion\n",
      "KOALA-C: A Scheduler for Integrated Multi-Cluster and Multi-Cloud Environments\n",
      "Multithreading for Embedded Reconfigurable Multicore Systems\n",
      "Context Based Human Activity Classification, Speaking or Nonspeaking\n",
      "Modelling and Mining of Data mining Workflows in Data Science Scripts\n",
      "Processor Design with DSP Enhancements for Wireless Applications\n",
      "Thermal Modelling of Existing Residential Buildings in North Western Europe\n",
      "Integrating a Neuron Network application into a ZYBO Zynq-7000 development board with an AXI-Bus interface\n",
      "Fourier Analysis of Iterative Methods for the Helmholtz Problem\n",
      "Model Predictive Control of a Wave Energy Converter: Archimedes Wave Swing\n",
      "Extraction and Visualization of Dimensions from a Geometric Model on the basis of its Medial Axis\n",
      "3D chromatin loops measured with Hi-C bring together SNP-SNP pairs engaging in epistatic interactions in GWAS data\n",
      "Co-design of Smart Lighting and Communication for Visible Light Networks\n",
      "Towards On-line Angle Instability Detection in Electrical Networks\n",
      "Detection of Botnet Command and Control Traffic in Enterprise Networks\n",
      "Guaranteed Quality ECG Signal Compression Algorithm\n",
      "General Purpose Computing with Reconfigurable Acceleration\n",
      "Digital Image Watermarking Robustness: A Comparative Study\n",
      "Operating characteristics for the design and optimisation of classification systems\n",
      "Supporting Developers' Teamwork from within the IDE\n",
      "Reliable Transport for Wireless Sensor Networks\n",
      "A patient-specific poroelastic model of a brain with a subdural hematoma\n",
      "Surface-Micromachined Thermal Conductivity Gas Sensors For Hydrogen Detection\n",
      "Determining Performance Boundaries and Automatic Loop Optimization of High-Level System Specifications\n",
      "Random sampling methods for two-view geometry estimation\n",
      "On 3D Stacked IC Yield Improvement and 3D-DfT Test Architecture\n",
      "The Effect of Generated Comments for Integration Tests on Code Comprehension\n",
      "Preserving privacy through cryptography in online behavioural advertising\n",
      "Automatically fused instructions: algorithms for the customization of the instruction, set of a recon?gurable architecture\n",
      "Modeling of nonlinear medical diagnostic ultrasound\n",
      "Compiler Assisted Runtime Adaptation\n",
      "Separation of Concerns in Web User Interface Design\n",
      "Quality Assurance in Agile Distributed Software Teams\n",
      "Multi-Domain SystemC Model of a Neural Interface\n",
      "Ethernet Circuit Failure Detection over Aggregated Trunks\n",
      "C-V profiling of ultra-shallow junctions using a buried layer with stepped doping\n",
      "MetaC - Embedded Software specific extensions for the C programming language\n",
      "Energy Efficient Branch Prediction for the Cell BE SPU\n",
      "Automatic Discovery of Distributed Algorithms for Large-Scale Systems\n",
      "Time-of-Flight 3D Imaging based on a SPAD-TDC Pixel Array in Standard 65 nm CMOS Technology\n",
      "Accurate Electromagnetic Modeling Methods for Integrated Circuits\n",
      "Reflecting walls and dissipating tiles: Billiards and fractal percolation\n",
      "Accelerating Diffusion-Weighted Chemical Shift Imaging using Compressed Sensing with Parameter Mapping\n",
      "A Sub-GHz UWB Pulse Generator for Wireless Biomedical Communication\n",
      "Accelerating Graph Analysis with Heterogeneous Systems\n",
      "Characterization of pure boron depositions integrated in silicon diodes for nanometer-deep junction applications\n",
      "Measurement Study of Multi-point Videoconferencing\n",
      "Customizable Register Files for Multidimensional SIMD Architectures\n",
      "Single electron tunneling based arithmetic computation\n",
      "Hardware Support for Dynamic Partial Reconfiguration\n",
      "Incorporating Leveled Homomorphic Encryption-based Private Information Retrieval in Federated eID Schemes to Enhance User Privacy\n",
      "FPGA Hardware acceleration of co-occurring aberrations in aCGH data\n",
      "Rendezvous on an interval and a search game on a star\n",
      "Hydrodynamic Limit for the Symmetric Inclusion Process\n",
      "Interactive Remote Rendering\n",
      "Scalable machine learning algorithms on a big data infrastructure\n",
      "Hardware and algorithm study for a fingerprint verification system\n",
      "The TM3270 Media-processor\n",
      "Delft’s history revisited: Semantic Web applications in the cultural heritage domain\n",
      "Simulation of single-electron tunnelling circuits using SPICE\n",
      "The Riesz transform on a complete Riemannian manifold with Ricci curvature bounded from below\n",
      "Desynchronization Methods for Scheduled Circuits\n",
      "Mobile crowdsourcing in an enterprise environment\n",
      "Expanding thermal plasma deposition of hydrogenated amorphous silicon for solar cells\n",
      "Towards a fault tolerant RISC-V softcore\n",
      "Opportunistic Spectrum Access: Designing Link and Transport layer\n",
      "LudoGraph: A Sampling Capable Cloud-Based System for Large-Scale Graph Processing: Based on the Pregel programming model\n",
      "Development of High Efficiency SHJ/Poly-Si Passivating Contact Hybrid Solar Cells\n",
      "CoopViz: Real Time Visualization of BitTorrent Swarm Dynamics\n",
      "Acceleration of the 2D Helmholtz model HARES\n",
      "Modular RT-Motion USB Software Framework\n",
      "Energy neutral neighbourhoods\n",
      "How the DEMO methodology could be a valuable addition to a testing framework\n",
      "Application-level Network Behavior Analysis and Anomaly Detection using Density Based Clustering\n",
      "Experimental and Industrial Evaluation of Variability Resilient Schemes\n",
      "Characterization of thin-film silicon materials and solar cells through numerical modeling\n",
      "Traffic load balancing to improve urban air quality\n",
      "Density Tapering for Sparse Planar Spiral Antenna Arrays\n",
      "Silicon MEMS for Detection of Liquid and Solid Fronts\n",
      "Super-regenerative Receiver for UWB-FM\n",
      "Silicon MEMS Micro-Evaporator\n",
      "Risk based decision support for new air traffic operations with reduced aircraft separation\n",
      "Fault-Tolerant Platform for Intra-Spacecraft Modular Wireless Sensor Network\n",
      "Auto-completion Algorithms for Geocoding Systems\n",
      "Design, Optimization, and Evaluation of a Path Query System\n",
      "JRET: A tool for the reconstruction of sequence diagrams from program executions\n",
      "RegistrationShop: An interactive 3D medical volume registration tool\n",
      "An Economic Framework for Resource Allocation in Ad-hoc Grids\n",
      "Simdization transformation strategies\n",
      "Longest Prefix Match and Incremental Updates for Range Tries\n",
      "Comparing vehicle routing approaches with artificial intelligence approaches on solving instances of the logistics planning problem domain\n",
      "4T CMOS Active Pixel Sensors under Ionizing Radiation\n",
      "Search-Based Test Data Generation for SQL Queries\n",
      "Multiple-view feature modelling with model adjustment\n",
      "Performing data assimilation experiments with hydrodynamic models: A Java Sea case\n",
      "Multicore Architectures for Bioinformatics Applications\n",
      "Performance improvement of motion control applications using GPPs and ASIPs on FPGA\n",
      "Characterization and Design of Complex Networks\n",
      "Open-Source Software-Based SRAM-PUF for Secure Data and Key Storage Using Off-The-Shelf SRAM\n",
      "Accelerating Computational Electromagnetic Diffraction Model on Programmable Graphics Processors\n",
      "An Adaptive Defect-Tolerant Multiprocessor Array Architecture\n",
      "Front-End for Composable Resource Sharing Using Latency-Rate Servers\n",
      "Modular RT-Motion USB\n",
      "Providing End-to-end Bandwidth Guarantees with OpenFlow\n",
      "Cross-Platform Expertise Characterization for Question Routing\n",
      "Transport modelling in coastal waters using stochastic differential equations\n",
      "Edgeless Silicon Sensors\n",
      "Design and Implementation of a Range Trie for Address Lookup\n",
      "Simulating Hearing Loss Using a Transmission Line Cochlea Model\n",
      "Hybrid Modeling for inter-turn voltage computation in layer type distribution transformers\n",
      "SOA principles applied in current methodologies: Describing guidelines for using RUP and AIM for implementing SOA solutions\n",
      "Convolutional Models for Landmine Identification with Ground Penetrating Radar\n",
      "Integrated field equations methods for the computation of electromagnetic fields in strongly inhomogeneous media\n",
      "Better than Worst-Case Design for Streaming Applications under Process Variation\n",
      "SparkJNI: A Reference Design for a Heterogeneous Apache Spark Framework\n",
      "Design of a high-performance buffered crossbar switch fabric using network on chip\n",
      "Creating a Mood Database for automated affect analysis\n",
      "Reordering DRAM Requests for Improved Bandwidth Utilization\n",
      "Free-space Emulator of 60GHz Directional Antenna Equipment\n",
      "ASIC FFT processor for MB-OFDM UWB system\n",
      "Signal processing of FMCW Synthetic Aperture Radar data\n",
      "A Multi Junction Solar Cell for Water Splitting: Development of an a-Si:H/nc-Si:H/SHJ triple junction solar cell\n",
      "Deterministic Task Transfer in Network-on-Chip Based Multi-Core Processors\n",
      "Fault tolerance on the ?-VEX processor\n",
      "Experimental analysis of design-for-testability techniques in SRAMs\n",
      "Mission-Driven sensor management analysis, design, implementation and simulation\n",
      "Taxi ride scheduling and pricing using historical data\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Multi Bit-Rate Video on Demand for P2P networks\n",
      "Research, design, simulation and implementation of an automatic flight control system for a real-time flight simulator\n",
      "Superlinearly Convergent Unstructured Staggered Schemes for Compressible and Incompressible Flows\n",
      "Quantitative analysis of trade-offs and model input sensitivities in public health\n",
      "Analysis of infinite dimensional diffusions\n",
      "Designing the Real-Valued Gene-pool Optimal Mixing Evolutionary Algorithm and Applying it to Substantially Improve the Efficiency of Multi-Objective Deformable Image Registration\n",
      "Validating perceptual objective listening quality assessment methods on the tonal language Igbo\n",
      "Suitability of tile-based rendering for low-power 3d graphics accelerators\n",
      "Sampling time error calibration in Time-interleaved ADCs\n",
      "Credits in BitTorrent: designing prospecting and investment functions\n",
      "A Semantic Web based approach to expertise finding at KPMG\n",
      "Run-time Adaptable VLIW Processors: Resources, Performance, Power Consumption, and Reliability Trade-offs\n",
      "Multi-core architecture for anonymous Internet streaming\n",
      "Ultra thin films for sensing and heating of microprobes\n",
      "Relative localization for Mobile Nodes using Visible Light\n",
      "Condition assessement of electrical contact of disconnectors\n",
      "Persuasive technology to motivate elderly individuals to walk: A case study\n",
      "System performance analysis and fixed-point architecture of a gradient-based optical flow algorithm\n",
      "Power Balancing in autonomous renewable energy systems: The design and construction of a renewable energy laboratory DENlab\n",
      "Design of electromagnetically compatible electronics\n",
      "An Integrated System to Manage Crosscutting Concerns in Source Code\n",
      "User-in-the-loop smart lighting control system\n",
      "Transformation of DEMO models into exchangeable format\n",
      "Interactive 3D Segmentation of CT-scans of the Hip Joint, Using Active Learning to Select Delineation Planes\n",
      "Enhancing BCI Based Navigation by Adding Intention Detection and a Virtual Assistant\n",
      "Classification of playing styles in football: The use of ball action data\n",
      "Frequency Based Cellular Microgrid Control: Demand Side Management and Supply Side Management using Grid Frequency in a Cellular Microgrid\n",
      "Techniques for Memory Mapping on Multi-Core Automotive Embedded Systems\n",
      "Exploring Tile-Based Rasterization Alternatives for Mobile Devices\n",
      "Parameter Identififcation of Dynamic Equivalents Using Heuristic Optimization Techniques\n",
      "Reliable sensor networks: A case study in commuter trains\n",
      "Understanding Ajax Applications by using Trace Analysis\n",
      "45nm Extraction and verification flow with SPACE\n",
      "Application of State Space Hidden Markov Models to the approximation of (embedded) option prices\n",
      "Hybrid NEMS-CMOS Architectures for Ultra Low Power Smart Systems: Architectures for Ultra Low Power Smart Systems\n",
      "The Molen Polymorphic Media Processor\n",
      "Exploring and implementing a User Datagram Protocol/Internet Protocol stack within a small Field Programmable Gate Array\n",
      "SPIN's Promela to Java Compiler: With help from Stratego\n",
      "Adaptive Clock Scheduling for Pipelined Structures\n",
      "A Resiliency-First Approach to Distributed DAG Computations\n",
      "Identifying options for the multi-terminal expansion of the COBRAcable\n",
      "Specification and Implementation of a DMA Controller in an Embedded System\n",
      "On the pricing of Bermudan swaptions in the multi-curve LIBOR Market Model\n",
      "Discontinuous Galerkin Methods: Linear Systems and Hidden Accuracy\n",
      "A model to measure Quality of Experience (QoE) of Online Social Gaming\n",
      "Hydrogenated Indium Oxide (IO:H) TCO for Thin Film Solar Cell\n",
      "Reconfigurable network processing platforms\n",
      "System Level Support for Dynamic Partial Reconfiguration\n",
      "Architecture-Level Fault-Tolerance Techniques for Biomedical Implants\n",
      "Using semantic relatedness to improve the evaluation of multi-label classifiers\n",
      "Improving the linear solver used in the interactive wave model of a real-time ship simulator\n",
      "Ultra low-power biomedical signal processing: An analog wavelet filter approach for pacemakers\n",
      "Impact of Offshore Wind Power on the Stability of the Dutch Electrical Grid\n",
      "A machine learning approach for optimisation in railway planning\n",
      "SSA Back-Translation: Faster Results with Edge Splitting and Post Optimization\n",
      "Automated test generation for microsoft DSL tools\n",
      "Understanding Operation and User Behavior in Peer-to-Peer Systems\n",
      "Toward seamless networking in indoor environments in millimeter wave bands\n",
      "Millimetre wave propagation and scattering phenomena for automotive radar\n",
      "Techniques and software architectures for medical visualisation and image processing\n",
      "Flexible international exchanges: A possible solution for large-scale wind power integration within the future power system\n",
      "Decomposing and Interactively Solving a High Dimensional Problem; Scheduling Trains on Shunting Yards\n",
      "Contributions to the financial mathematics of energy markets\n",
      "Black-Box assessment of Web systems security\n",
      "Elliptic regions in porous media flow: Effect of relative permeability on loss of strict hyperbolicity in three-phase porous media flow\n",
      "Interconnect Testing for 3D Stacked Memories\n",
      "Robust Communication Framework for Unmanned Aerial Systems\n",
      "Data streams in spreadsheets with reactive extensions\n",
      "Semi-custom VLSI Design and Realization of DC-DC Converters in UMC90\n",
      "Space charge accumulation in polymeric high voltage DC cable systems\n",
      "Design of a Readout Scheme for a MEMS Microphone\n",
      "A Domain-Specific Language for Internal Site Search\n",
      "Pattern transfer on vertical cavity sidewalls using SU8\n",
      "Schema-Based Partitioning for Optimized Querying of XML data\n",
      "Structured Electronic Design of High-Pass ∆Σ Converters: And their application to cardiac signal acquisition\n",
      "Anti-Pattern Scanner: An Approach to Detect Anti-Patterns and Design Violations\n",
      "The current interruption process in vacuum analysis of the currents and voltages of current-zero measurements\n",
      "A study on TCP-SYN attacks and their effects on a network infrastructure\n",
      "Hala ρ-VEX: Highly-Programmable Dynamically-Reconfigurable FPGA-based Streaming Platform for Image Processing\n",
      "Modeling of turbulent two-phase stratified flow\n",
      "Accelerating Computational Finance Simulations; with OpenCL: a case study\n",
      "Software To Hardware: Alternatives For Reducing Design Time Of Optimized FPGA Implementations In Medical Devices [CP]\n",
      "Modeling and topology investigation of modular machines for wind generator systems\n",
      "Aging Assessment and Reliability Aware Computing Platforms\n",
      "Leveraging blockchains to establish cooperation\n",
      "User contribution in peer-to-peer communities\n",
      "A Real-Time Hybrid Neuron Network for Highly Parallel Cognitive Systems\n",
      "Multiple-description lattice vector quantization\n",
      "Knowledge driven facial modelling\n",
      "Improving the Visualization of 3D Simulations using Computer Game technology\n",
      "Virtual Reality and Social Phobia: Recreating a social situation in virtual reality\n",
      "New architecture for remote powered wireless transmitter\n",
      "Concurrent Cube-and-Conquer\n",
      "An optical measurement system for the analysis of bacteria in drain fluid\n",
      "Electrothermal Behavior of High-Frequency Silicon-On-Glass Transistors\n",
      "Detecting malicious behaviour using system calls\n",
      "Fine-Grain Runtime Fault Diagnosis for Reconfigurable Logic Blocks\n",
      "WatchDog For IntelliJ: An IDE Plugin To Analyze Software Testing Practices\n",
      "Passive Localization with Visible Light\n",
      "Analyzing P2PTV Traffic Via Measurement\n",
      "Adaptive OFDM and CDMA Algorithms for SISO and MIMO Channels\n",
      "Automatic Hardware Generation for Reconfigurable Architectures\n",
      "Towards Natural Language Understanding using Multimodal Deep Learning\n",
      "Simulating Interaction Between Tracked Real-World Objects and User-Controlled Virtual Objects\n",
      "A combined BEM/FEM method for IC substrate modeling\n",
      "Energy Allocation Strategies for Micro-Grids\n",
      "LLVM-based ?-VEX compiler\n",
      "Heuristic Search for Defect Tolerant Multiprocessor Arrays\n",
      "Phasor Measurement Unit (PMU) based power system analysis of MV distribution grid\n",
      "Hardware Acceleration of BWA-MEM Genome Mapping Application\n",
      "What Programmers Know About Debugging And How They Use Their IDE Debuggers\n",
      "Visual Analysis of Lung Cancer Imaging Data Using Multidimensional Projections Techniques\n",
      "A framework for the study of grid inter-operation mechanisms\n",
      "Securing Social Media: A Network Structure Approach\n",
      "The design and implementation of the KOALA grid resource management system\n",
      "On the Equivalence of Nested Relational Languages\n",
      "Hardware Acceleration of Monte-Carlo Integration in Finance\n",
      "Distributed Regular Path Query Matching and Optimization for Graph Database based on Spark\n",
      "Ultrasound 3D positioning system for surgical instruments\n",
      "Studying Late Propagations Using Software Repository Mining\n",
      "Aging Mitigation Schemes for Embedded Memories\n",
      "Procedural Generation of Traffic Signs\n",
      "Throughput and Power optimization of the Range Trie design for IP lookups\n",
      "Design & development of public-key based authentication architecture for IoT devices using PUF\n",
      "Compressive Wideband Spectrum Sensing for Cognitive Radio Applications\n",
      "A Genetic Approach to Programming Large-Scale Systems\n",
      "Hardware design and implementation of a network-on-chip based high performance crossbar switch Fabric\n",
      "Strategies for Trading Agents in a Double Auction\n",
      "Banking 2.0: Developing a Reference Architecture for Financial Services in The Cloud\n",
      "Sensing Locations: Occupancy Sensor Data Driven Automatic Determination of Sensor Locations in Buildings\n",
      "Declarative specification of template-based textual editors\n",
      "New Algorithms for the Simple Temporal Problem\n",
      "Evaluation of Behavior-Driven Development\n",
      "Sparse multi-class prediction based on the Group Lasso in multinomial logistic regression\n",
      "Low Overhead Indoor Localisation on a Navel Ship\n",
      "Managing Software Design Erosion with Design Conformance Checking\n",
      "Improving the Performance of the Variant Calling Workﬂow for DNA Sequencing\n",
      "A Hamiltonian Reduction Method for Nonlinear Dynamics\n",
      "Tool-assisted Spreadsheet Refactoring and Parsing Spreadsheet Formulas\n",
      "The Lifetime Prediction of LED Drivers and Lamps\n",
      "Delamination of Bonded Repairs: A Damage Tolerance Approach\n",
      "Cost-Effectiveness of test-driven development\n",
      "Combining source code and test coverage changes in pull requests\n",
      "Computational Intelligence. Mortality Models for the Actuary\n",
      "An Inside to Outside approach: Focussing on parametric pattern generation for spatial organization, satisfying experiential demands\n",
      "Re-engineering Web Applications from MSO to SSO\n",
      "Heat Transfer in the Thick Thermoset Composites\n",
      "Qualitative and Quantitative Imaging in Electromagnetic Inverse Scattering Theory\n",
      "Building applications for mobile devices that will support business processes involving SAP\n",
      "South Chicago Lakeside Health City Medical Library\n",
      "Designing Linked Data Applications\n",
      "Investigation of the impact of cohesion on the change-proneness of Java interfaces\n",
      "On asymptotic approximations of first integrals for a class of nonlinear oscillators\n",
      "A new model for the pre-sliding regime of friction\n",
      "Isogeometric analysis for thin-walled composite structures\n",
      "Designing DAWN: A Data Analytics Workflow Notation\n",
      "R3_Rehabilitate*Reintegrate*Reshape\n",
      "VSC-HVDC based Network Reinforcement\n",
      "Networked Indoor Lighting Controls with Visible Light Communication\n",
      "Context-Based Spelling Correction for the Dutch Language: Applied on spelling errors extracted from the Dutch Wikipedia revision history\n",
      "Passive and Active Inspection of Wireless Sensor Networks: A Practical Use Evaluation\n",
      "A System Dynamics Exploration of Port-City Development: The Case of Tema, Ghana\n",
      "Analyzing the State of Static Analysis: A Large-Scale Evaluation in Open Source Software\n",
      "Conceptual Design of a Composite Main Fitting of a Cantilever Type Nose Landing Gear\n",
      "OWL Formal Ontology for the Social Structure in the OperA Meta-model\n",
      "Empirical Software Linguistics: An Investigation of Code Reviews, Recommendations and Faults\n",
      "A Koiter-Newton arclength method for buckling-sensitive structures\n",
      "Evaluating meta-governance as guiding principle for municipalities to develop a climate-proof urban area and deal with extreme rainfall\n",
      "The Mid-city's New Apple Center\n",
      "Enterprise Ontology, does it care?\n",
      "Efficient approximate leave-one-out cross-validation for ridge and lasso\n",
      "Workflow Mining: A stepwise Approach for extracting Event logs from Corporate Network Data\n",
      "DIOS: Disaster Interagency Orchestration System: Developing and Evaluating a Network-Centric Crisis Information Management System for ensuring Information and System Quality in Crisis Situations\n",
      "Link Prediction and the Evolution of Communities on Twitter\n",
      "Midway Convention Center | Chicago\n",
      "Lamb Wave Propagation in Sandwich Composite Structures for the Detection of Impact Damage\n",
      "Selecting Bug-prone Components to Study the Effectiveness of Reengineering and Unit Testing\n",
      "Participation and Interaction in Projects: A Game-Theoretic Analysis\n",
      "Capturing and Predicting the Integration Process of an Embedded Software Company\n",
      "Operational Scheduling in a Multi-Actor Environment using Multiagent Systems\n",
      "Feature-Oriented Evolution of Variant-rich software systems\n",
      "Plan repair in decoupled Simple Temporal Problems\n",
      "Suggesting Queries using Query-Flow Graphs to find Dutch Content with Curated Tags\n",
      "Testing Advanced Web Interfaces: Tool-assisted user interface testing of AJAX-based Web applications\n",
      "Performance Optimization of Multi-Tenant Software Systems\n",
      "Chicago - embraced\n",
      "The In?uence of Software Maintainability on Issue Handling\n",
      "Towards the agile enterprise: A method to come from a DEMO model to a Normalized System, applied to Government Subsidy Schemes\n",
      "The Delft-Java Engine\n",
      "Code Smells in the Mobile Applications Domain\n",
      "A Method for Enterprise Ontology based Design of Enterprise Information Systems\n",
      "Robust Downstream Communication and Storage for Computational RFIDs\n",
      "A Study of Pair Correlation Functions Using Classical DFT\n",
      "Using Github Profiles in Software Developer Recruitment and Hiring\n",
      "Farming Guangming: Integrating agricultural landscape and new town development for the “Green City” Guangming in Shenzhen\n",
      "DenCity Lab: Productive Working Living\n",
      "Phishing as a Service: Designing an ethical way of mimicking targeted phishing attacks to train employees\n",
      "Regional Health Information Exchange (RHIE)\n",
      "Run-time Reconfiguration in Wireless Sensor Networks\n",
      "Gesture-Based OLAP Navigation on Touchscreen Devices\n",
      "Incorporating Crowd Perspectives into Multimedia Retrieval Systems\n",
      "Dynamic User Interface Supporting Type Analysis for RDFGears\n",
      "IMS messaging gateway in the Cloud\n",
      "Smart-grid management using Java-based Complex Event Processing\n",
      "Single person pose recognition and tracking\n",
      "Hardware acceleration of simulations of distributed systems\n",
      "Predicting the Structural Performance of Composite Structures Under Cyclic Loading\n",
      "fine-GRAPE: Fine-Grained APi usage Extractor An Approach and Dataset to Investigate API Usage\n",
      "Data assimilation in atmospheric chemistry models using Kalman filtering\n",
      "Remove and Expose: Chicago's Public Soil Treatment Facility\n",
      "Language-parametric Techniques for Language-Specific Editors\n",
      "Context-Aware Multi-Stage Route Planning\n",
      "DEMO and Security\n",
      "Ontology Driven RDF Data Creation for the Universal Information Adapter\n",
      "Component diagram recovery with dynamic analysis\n",
      "Computational architecture: Focusing on the structural and layout complexity\n",
      "The Added Value of Enterprise Architecture\n",
      "Obama Presidential Museum in Garfield, Chicago\n",
      "Full-Wave Analysis of Electrically Large Reflector Antennas\n",
      "Identity & Access Management: Get in control: IT Governance, people, permission and technical challenges.\n",
      "The feasibility of the energy-only market in a highly renewable European power system\n",
      "The Place of all\n",
      "Thermoplastic Composite Wind Turbine Blades: An Integrated Design Approach\n",
      "Throughput rates of Simple Operations when Scaling-out with RDBMS and NoSQL Databases\n",
      "Focusing on spatial composition and influence of building envelope on daylight aspects in an art center\n",
      "Orchestration of (chain-) processes in subsidy agencies: Developing a generic subsidy orchestration system\n",
      "Topology optimization of 3D linkages with application to morphing winglets\n",
      "Community Vocational Schools _ South Learns\n",
      "Web Service Growing Pains: Understanding Services and Their Clients\n",
      "Compiler Strategies for Transport Triggered Architectures\n",
      "Wireless Sensor Platform for Sporting Applications\n",
      "Chicago: Social Dwelling Typologies\n",
      "Integrating landmarks in partial order planners\n",
      "Authorship Identification and Verification of JavaScript Source Code: An Evaluation of Techniques\n",
      "Online deployment of dependent tasks onto networked systems\n",
      "Sensor-Assisted Movement Identification and Prediction for Beamformed 60 GHz Links\n",
      "Relations in Role-Based Data Modeling and Navigation\n",
      "Media Platform Chicago South Works\n",
      "Graphic statics in funicular design: Calculating force equilibrium through complementary energy\n",
      "Influence of test parameters on the on-load tap changer's dynamic resistance measurement\n",
      "Hearsay: Suppressing spam using trust in mobile social gossiping networks\n",
      "Identifying Anomalies in a Continuous Running Software System through Log Data Extraction\n",
      "Energy-based Aeroelastic Analysis and Optimisation of Morphing Wings\n",
      "Extract Knowledge from Social Networks\n",
      "A single comprehensive computational building model\n",
      "TMM-Based Process Improvement in Small Global Organization\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "User Modeling and Personalization in the Microblogging Sphere\n",
      "Transparency in actions on requirements\n",
      "Evaluation of innovative land tools in sub-Saharan Africa: Three cases from a peri-urban context\n",
      "The world according to MARP\n",
      "A Variational Formulation for Thin Membranes\n",
      "Perspectives on Cyber Security: Managerial perspectives on cyber security and the role of end user awareness\n",
      "Personalized Microblog Search on Twitter\n",
      "Architecture-Driven Integration of Modeling Languages for the Design of Software-Intensive Systems\n",
      "Computational Architecture: Focusing on perception and functionality aspects of urban intervention\n",
      "A mathematical analysis of a belt system with a low and time-varying velocity\n",
      "Performance Evaluation of Polybenzimidazole for Potential Aerospace Applications\n",
      "Modelling and optimization of a permanent-magnet machine in a flywheel\n",
      "An Art Museum Hub for Den Haag\n",
      "Ultrasonic Plastic Welding of Carbon Fiber Reinforced Polyamide 6 to Aluminium and Steel\n",
      "Taming Technology: The narrative anchor reconciling time, territory and technology in geoinformation infrastructures\n",
      "Evaluating the Lifespan of Code Smells in a Software System using Software Repository Mining\n",
      "2.5D Playgrounds for Bogota: Stimulate Playful Actions in Public Spaces by Redesigning Ground Surfaces into 2.5D Architectural Compositions\n",
      "Opportunistic Communication in Extreme Wireless Sensor Networks: A step back towards the smart dust dream\n",
      "Evolving Danwei housing: An alternative way to develop former public housing in Shenzhen, China\n",
      "Supporting users with credibility assessments of health Tweets\n",
      "Metric-based Evaluation of Implemented Software Architectures\n",
      "Adaptability in dynamic wireless networks\n",
      "Building Blocks for Language Workbenches\n",
      "Development of a Rapid Thermoplastic Impregnation Device\n",
      "Optimization of stiffened panels using a combination of FEM and a predictor-corrector interior point method\n",
      "Design of a coaching support system for evaluating sport tactics\n",
      "Measuring time distribution of engineering test and production code\n",
      "Dependency-based anomaly detection\n",
      "Weak signal detection on twitter datasets: A non-accumulated approach for non-famous events\n",
      "Gamification in Enterprise Crowdsourcing\n",
      "A Case for Deep Learning in Mining Software Repositories\n",
      "Massivizing Networked Virtual Environments on Clouds\n",
      "ArchWiki: Using Web 2.0 for Architecture Knowledge Management\n",
      "The Market for Owner-Occupied Dwellings After the Crisis: Institutional Change and Shortages of Owner-Occupied Dwellings in the Netherlands\n",
      "South Works Creative Incubator\n",
      "Arens de la ciudad\n",
      "Declarative Access Control for WebDSL\n",
      "Analysis and Testing of Ajax-based Single-page Web Applications\n",
      "Efficient Execution of Video Applications on Heterogeneous Multi- and Many-Core Processors\n",
      "Resolving Disruptions in Simple Temporal Problems\n",
      "Improving software flexibility in a smart business network: Designing an ontology-driven software architecture for the Internet Business Learning Community\n",
      "Temperature and Load effects in Modelling and Experimental Verification of Acoustic Emission Signals for Structural Health Monitoring Applications\n",
      "Separate Compilation as a Separate Concern: A Framework for Language-Independent Selective Recompilation\n",
      "Focusing Automatic Code Inspections\n",
      "Design for values in smart grid systems: An exploratory empirical investigation of the ethical values and their interrelationship that play a role in the British public debate on smart grid systems\n",
      "Scalability and fault-tolerance in the groupware domain\n",
      "Combining Design and Engineering Methodology for Organizations with the Rational Unified Process\n",
      "Offline Data and Synchronization for a Mobile Backend as a Service system\n",
      "Thermoplastic Composite Wind Turbine Blades: Kinetics and Processability\n",
      "Robust scheduling in an uncertain environment\n",
      "Towards a Functional Interface for Temporal Navigation of News Archives\n",
      "Streaming Fraud Detection on Session Based Data\n",
      "Improving the Quality of Haptic Feedback Yields Only Marginal Improvements in Teleoperated Task Performance\n",
      "RDF Gears, a data integration framework for the Semantic Web\n",
      "Simulating the flexibility potential of demand response with heat pumps in the Netherlands\n",
      "The adoption of DEMO in practice\n",
      "Ordinal multi-class Molecular Classification\n",
      "Analysis of high-volume traffic using Complex Event Processing and a Domain Specific Language\n",
      "In Dependencies We Trust: How vulnerable are dependencies in software modules?\n",
      "Performative architecture as a guide line for transformation of the defence line of Amsterdam\n"
     ]
    }
   ],
   "source": [
    "for r in results:\n",
    "    print(r['title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from elasticsearch import Elasticsearch\n",
    "import pandas as pd\n",
    "from nltk.corpus import wordnet\n",
    "import operator, requests\n",
    "from pymongo import MongoClient\n",
    "from xml.etree import ElementTree\n",
    "import operator\n",
    "\n",
    "client = MongoClient('localhost:4321')\n",
    "db = client.TU_Delft_Library\n",
    "\n",
    "es = Elasticsearch([{'host': 'localhost', 'port': 9200}])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "query_input = 'Lofi'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Valle Torre, Manuel (author)']\n",
      "['Lofi, Christoph (mentor)']\n",
      "Shared Perspectives on Perceptual Features for Query by Example in E-Commerce Systems\n",
      "['Zegers, Jan (author)']\n",
      "['Lofi, Christoph (mentor)']\n",
      "Modelling and Mining of Data mining Workflows in Data Science Scripts\n"
     ]
    }
   ],
   "source": [
    "# _query_author = {\n",
    "#     'query': {\n",
    "#         'match_phrase': {\n",
    "#             'authors': query_input        }\n",
    "#     }\n",
    "# }\n",
    "\n",
    "_query_author = {\n",
    "                  \"query\": {\n",
    "                    \"bool\": {\n",
    "                      \"should\": [\n",
    "                        { \"match\": { \"authors\":  query_input }},\n",
    "                        { \"match\": { \"supervisors\": query_input   }}\n",
    "                      ]\n",
    "                    }\n",
    "                  }\n",
    "                }\n",
    "\n",
    "# _query_author = {\"query\":\n",
    "#     {\"match\": {\n",
    "#         \"authors\": {\n",
    "#             \"query\": string,\n",
    "#             \"operator\": \"and\"\n",
    "#         }\n",
    "#     }\n",
    "#     }\n",
    "# }\n",
    "\n",
    "_search_author = es.search(index=\"ir_tud\", doc_type=\"publications\", body=_query_author)\n",
    "for doc in _search_author['hits']['hits']:\n",
    "#     title_list.append(doc['_source']['title'])\n",
    "#     journal_list.append(doc['_source']['journal'])\n",
    "#     year_list.append(doc['_source']['year'])\n",
    "#     authors_list.append(doc['_source']['authors'])\n",
    "#     id_list.append(doc['_id'])\n",
    "    print(doc['_source']['authors'])\n",
    "    print(doc['_source']['supervisors'])\n",
    "    print(doc['_source']['title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "_string = 'machine learning'\n",
    "\n",
    "_query_all = {\n",
    "    'query': {\n",
    "        \"query_string\": {\n",
    "            \"default_field\": \"content\",\n",
    "            \"query\": _string\n",
    "        }\n",
    "    }\n",
    "}\n",
    "\n",
    "_search = es.search(index=\"ir_tud\", doc_type=\"publications\", body=_query_all, size = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "id_list = []\n",
    "for doc in _search['hits']['hits']: \n",
    "#     print(doc['_source']['title'])\n",
    "    id_list.append(doc['_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def popular_upcoming_entities(paper_id_list):\n",
    "    terms_in_results = []\n",
    "    terms_labels = {}\n",
    "    entity_occurrences = {}\n",
    "    upcoming_occurrences = {}\n",
    "\n",
    "    for paper in paper_id_list:\n",
    "        _query_terms = {\n",
    "            \"query\": {\n",
    "                \"bool\": {\n",
    "                    \"must\": [{\"match\": {'paper_id': paper}}],\n",
    "                    \"should\": [{\"match\": {\"annotator\": 'method'}},\n",
    "                               {\"match\": {\"annotator\": 'dataset'}}]\n",
    "                }\n",
    "            }\n",
    "        }\n",
    "\n",
    "        _query_terms = es.search(index=\"entities_tud\", doc_type=\"entities\", body=_query_terms)\n",
    "\n",
    "        for hit in _query_terms['hits']['hits']:\n",
    "            entity = hit['_source']['lower']\n",
    "            terms_in_results.append(entity)\n",
    "            terms_labels[entity] = hit\n",
    "\n",
    "    terms_in_results = list(set(terms_in_results))\n",
    "\n",
    "    for entity in terms_in_results:\n",
    "        _query_occurrences = {\n",
    "            'query': {\n",
    "                'match_phrase': {\n",
    "                    'content': entity\n",
    "                }\n",
    "            }\n",
    "        }\n",
    "\n",
    "        _query_occurrences = es.search(index=\"ir_tud\", doc_type=\"publications\", body=_query_occurrences)\n",
    "        years = []\n",
    "        entity_occurrences[entity] = _query_occurrences['hits']['total']\n",
    "\n",
    "        for doc in _query_occurrences['hits']['hits']:\n",
    "            years.append(doc['_source']['year'])\n",
    "        if years:\n",
    "            if int(min(years)) > 2010:\n",
    "                upcoming_occurrences[entity] = _query_occurrences['hits']['total']\n",
    "\n",
    "    sorted_occurrences = sorted(entity_occurrences.items(), key=operator.itemgetter(1), reverse=True)\n",
    "    sorted_upcoming = sorted(upcoming_occurrences.items(), key=operator.itemgetter(1), reverse=True)\n",
    "\n",
    "    triples = []\n",
    "    for pair in sorted_occurrences:\n",
    "        amount = pair[1]\n",
    "        term = pair[0]\n",
    "\n",
    "        if terms_labels[term]['_source']['annotator'] in ['method', 'dataset']:\n",
    "            actual_label = terms_labels[term]['_source']['annotator']\n",
    "        elif terms_labels[term]['_source']['annotator'] in ['other', 'noise', 'software']:\n",
    "            continue\n",
    "        elif terms_labels[term]['_source']['mt_similarity'] > terms_labels[term]['_source']['ds_similarity']:\n",
    "            actual_label = 'method'\n",
    "        else:\n",
    "            actual_label = 'dataset'\n",
    "        triples.append([term, actual_label, amount])\n",
    "\n",
    "    return sorted_occurrences, sorted_upcoming, terms_labels, triples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "popular, upcoming, entity_labels, entity_triples = popular_upcoming_entities(id_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mapreduce\n",
      "whereas\n",
      "whenever\n",
      "big data processing\n",
      "big data analytics\n",
      "scalability\n",
      "stack\n",
      "big data processing\n",
      "bigbench\n",
      "converted\n",
      "matlab\n",
      "whenever\n"
     ]
    }
   ],
   "source": [
    "for paper in id_list[:2]:\n",
    "    _query_terms = {\n",
    "        \"query\": {\n",
    "            \"bool\": {\n",
    "                \"must\": [{\"match\": {'paper_id': paper}}],\n",
    "                \"should\": [{\"match\": {\"annotator\": 'method'}},\n",
    "                           {\"match\": {\"annotator\": 'dataset'}}]\n",
    "            }\n",
    "        }\n",
    "    }\n",
    "\n",
    "    _query_terms = es.search(index=\"entities_tud\", doc_type=\"entities\", body=_query_terms)\n",
    "\n",
    "    for hit in _query_terms['hits']['hits']:\n",
    "        print(hit['_source']['lower'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'_id': 'TUD_98a8d01e-6259-4fed-8704-5af0615304d2138187',\n",
       " '_index': 'entities_tud',\n",
       " '_score': 46.910866,\n",
       " '_source': {'PMIdata': 0.6049066545457461,\n",
       "  'PMImethod': 2.100680896118057,\n",
       "  'annotator': 'undefined',\n",
       "  'clean': 'whenever',\n",
       "  'ds_sim_50': 0,\n",
       "  'ds_sim_60': 0,\n",
       "  'ds_sim_70': 0,\n",
       "  'ds_sim_80': 0,\n",
       "  'ds_sim_90': 0,\n",
       "  'ds_similarity': 0.470866,\n",
       "  'filteredWord': 'whenever',\n",
       "  'inwordNet': 0,\n",
       "  'journal': 'tudelft',\n",
       "  'label': 'dataset',\n",
       "  'lower': 'whenever',\n",
       "  'mt_sim_50': 1,\n",
       "  'mt_sim_60': 0,\n",
       "  'mt_sim_70': 0,\n",
       "  'mt_sim_80': 0,\n",
       "  'mt_sim_90': 0,\n",
       "  'mt_similarity': 0.474401,\n",
       "  'no_punkt': 'whenever',\n",
       "  'paper_id': 'TUD_98a8d01e-6259-4fed-8704-5af0615304d2',\n",
       "  'title': 'Development of the Machine Learning-Based Approach to Access Occupancy Information Through Indoor CO2 Data: for Demand-Driven Building Heating Energy Management',\n",
       "  'word': 'whenever',\n",
       "  'year': '2017'},\n",
       " '_type': 'entities'}"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hit"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Data Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import pymongo\n",
    "import elasticsearch\n",
    "import nltk\n",
    "import string\n",
    "import sys\n",
    "import re\n",
    "import os\n",
    "\n",
    "import config as cfg\n",
    "\n",
    "sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "client = pymongo.MongoClient('localhost:' + str(cfg.mongoDB_Port))\n",
    "publications_collection = client.TU_Delft_Library\n",
    "es = elasticsearch.Elasticsearch([{'host': 'localhost', 'port': 9200}],\n",
    "                                 timeout=30, max_retries=10, retry_on_timeout=True)\n",
    "es.cluster.health(wait_for_status='yellow', request_timeout=1)\n",
    "\n",
    "\n",
    "def return_names(mongo_string_search, db):\n",
    "    results = db.publications.find(mongo_string_search)\n",
    "    list_of_docs = list()\n",
    "    extracted = {\n",
    "        \"_id\": \"\",\n",
    "    }\n",
    "    for i, r in enumerate(results):\n",
    "        extracted['_id'] = r['_id']\n",
    "        list_of_docs.append(extracted)\n",
    "        extracted = {\n",
    "            \"_id\": \"\",\n",
    "        }\n",
    "    return list_of_docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "extracted_publications = []\n",
    "query = {}\n",
    "extracted_publications.append(return_names(query, publications_collection))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[{'_id': 'TUD_862e5b58-b9d1-462a-90b0-6f94a054632e'},\n",
       "  {'_id': 'TUD_960eb3cd-080d-4423-b9cb-f82417e9e13c'},\n",
       "  {'_id': 'TUD_c35fa6e0-e632-4f17-bf84-0f1cc8f98c0c'},\n",
       "  {'_id': 'TUD_f116361b-8ff7-4995-a8e5-e5354f2902b9'},\n",
       "  {'_id': 'TUD_46081d76-17fc-41f6-bdc4-fbdc6d7bb939'},\n",
       "  {'_id': 'TUD_ed609f8d-6cc5-489f-aa45-669b797f677a'},\n",
       "  {'_id': 'TUD_f514548b-63c6-4015-a824-c86c9a63477c'},\n",
       "  {'_id': 'TUD_beca2029-d274-4d14-9ea0-a59490019bc1'},\n",
       "  {'_id': 'TUD_fc4aa0e6-1d20-41e9-a660-237a6873bb3c'},\n",
       "  {'_id': 'TUD_84ff0f8e-ec0a-4b77-8fa9-9df7eaef6b60'},\n",
       "  {'_id': 'TUD_9ced4cb2-9809-4b58-af25-34e458e8ea23'},\n",
       "  {'_id': 'TUD_b0c2a859-4351-4b6a-b66b-a7bbfed76ecb'},\n",
       "  {'_id': 'TUD_280fe4d8-d1de-4a46-ad99-be8129d85dab'},\n",
       "  {'_id': 'TUD_78bf003b-e784-40ca-888c-48e0246b3883'},\n",
       "  {'_id': 'TUD_ba867fef-5421-4692-b955-6c2c468653cc'},\n",
       "  {'_id': 'TUD_b6ea3435-825a-4306-a4d4-4edee26f233c'},\n",
       "  {'_id': 'TUD_8150156e-e633-4319-8685-ccac7b083434'},\n",
       "  {'_id': 'TUD_c061a03a-84a5-4ac3-ba22-c5b315f08edd'},\n",
       "  {'_id': 'TUD_32789615-a48a-4b0f-ba27-d11992a201d1'},\n",
       "  {'_id': 'TUD_187e5e62-db64-4f62-8b08-c5dddfc174a3'},\n",
       "  {'_id': 'TUD_22662036-5abb-4a5a-9f1a-a4a9a68e0997'},\n",
       "  {'_id': 'TUD_d8f95cd0-f243-4989-89f3-a8a262b1bfd6'},\n",
       "  {'_id': 'TUD_8bd86467-7565-481c-beec-4d76423f1bff'},\n",
       "  {'_id': 'TUD_167d7879-7f61-4c5e-9390-7d1435e9eda1'},\n",
       "  {'_id': 'TUD_a69af610-68f9-447d-9c82-cdae18994f50'},\n",
       "  {'_id': 'TUD_975a4a47-7935-4f76-9503-6d4e36b674a3'},\n",
       "  {'_id': 'TUD_c281056f-a646-4736-9f4b-bb53ecaa5d72'},\n",
       "  {'_id': 'TUD_88ece5de-f233-4c13-a940-f5c862a9b154'},\n",
       "  {'_id': 'TUD_011a9efc-4813-4e4f-8e7f-a5f7c2e27f1b'},\n",
       "  {'_id': 'TUD_84f0e48c-a6f8-4708-92f6-047e849f565a'},\n",
       "  {'_id': 'TUD_e57d0dc4-14aa-4223-9042-4d46c6f0ee8c'},\n",
       "  {'_id': 'TUD_5e522da0-6712-42c7-b90a-55a7f80d64b5'},\n",
       "  {'_id': 'TUD_17de3262-b7a3-400b-8fbc-1b0a25d4d957'},\n",
       "  {'_id': 'TUD_36f8bedc-2fb1-461d-a979-9bfdb9520930'},\n",
       "  {'_id': 'TUD_026c9807-3632-4272-b69f-a3b6f5452607'},\n",
       "  {'_id': 'TUD_d5b89811-d405-469e-a508-7f062d13558f'},\n",
       "  {'_id': 'TUD_7cd6c046-956b-47bf-bd21-d631a1677e98'},\n",
       "  {'_id': 'TUD_f2723dd7-d3f4-4ee0-a59c-3394562c4150'},\n",
       "  {'_id': 'TUD_7ddd648a-f5b2-4d18-8cee-75a7c7210b57'},\n",
       "  {'_id': 'TUD_ed582f9d-2d93-48d1-8b81-3711fd22742c'},\n",
       "  {'_id': 'TUD_e8fd5e0c-4f2d-47b0-9952-06cea4fe0c98'},\n",
       "  {'_id': 'TUD_6efb78a4-5221-46f4-851b-fa776cb174de'},\n",
       "  {'_id': 'TUD_6547d4c2-0e5f-430a-aed4-bc67c8bf7744'},\n",
       "  {'_id': 'TUD_38d9e021-b5ec-48f8-8aea-93a4393a0478'},\n",
       "  {'_id': 'TUD_427a64d8-6ba1-4418-a04f-e5f092c9ff60'},\n",
       "  {'_id': 'TUD_2a54b619-6f90-474b-83a3-95656874206b'},\n",
       "  {'_id': 'TUD_9f7a367d-100d-4ff1-a02e-3532d96689ee'},\n",
       "  {'_id': 'TUD_b38bc8f5-3e86-4ef7-83df-c292193bbea2'},\n",
       "  {'_id': 'TUD_38efa276-e4e9-40b2-bd04-b7f9c5e2c492'},\n",
       "  {'_id': 'TUD_4600d978-3988-4daf-b6a4-4534e54634f6'},\n",
       "  {'_id': 'TUD_1306f207-66c9-460c-8c73-e41c4af74ef4'},\n",
       "  {'_id': 'TUD_4eb03123-3847-43ce-8f8f-214c382b5698'},\n",
       "  {'_id': 'TUD_1ad87d96-5c2d-4463-957e-e104495b05cb'},\n",
       "  {'_id': 'TUD_a5639b34-9df5-4be8-bd49-7c3e972046be'},\n",
       "  {'_id': 'TUD_7348938d-c490-4b13-a5e8-e771a28b71f2'},\n",
       "  {'_id': 'TUD_33c6e093-a129-47ec-a663-3ce808343a93'},\n",
       "  {'_id': 'TUD_73783036-0241-4dff-bae8-e56ad155ce00'},\n",
       "  {'_id': 'TUD_ab5d5b57-daed-4f7f-96fa-f0f9bb2fd83e'},\n",
       "  {'_id': 'TUD_97097039-766a-4784-810e-79447d5b036c'},\n",
       "  {'_id': 'TUD_27d7627f-8163-4cb9-aa3e-591ecdd2370e'},\n",
       "  {'_id': 'TUD_215714d3-63dc-48e4-ac11-39484cb120ac'},\n",
       "  {'_id': 'TUD_c290585a-8e2d-448a-9b31-52540877195e'},\n",
       "  {'_id': 'TUD_48a8afda-774f-4658-a283-3170bbe9ef73'},\n",
       "  {'_id': 'TUD_504b4d73-c4ab-4e5e-bcaf-ca6d2ff7347b'},\n",
       "  {'_id': 'TUD_4c9abeb7-6b98-4f07-9cbb-ed27bbdade06'},\n",
       "  {'_id': 'TUD_1e24b455-728d-42c0-82c5-e55f64956161'},\n",
       "  {'_id': 'TUD_8cd415c3-1ca3-4974-982f-40b5508bd8e1'},\n",
       "  {'_id': 'TUD_db220b8a-c131-4cb2-bde0-604a269703ef'},\n",
       "  {'_id': 'TUD_73f1f696-0349-4d79-a7a2-75c38c137473'},\n",
       "  {'_id': 'TUD_59d5a00d-6bb7-490d-9887-32e9530eb33e'},\n",
       "  {'_id': 'TUD_49a3bfaa-012f-4e3b-ac34-b8d9a760c4fc'},\n",
       "  {'_id': 'TUD_4f520286-ec98-49fb-a005-f18023c1ef9b'},\n",
       "  {'_id': 'TUD_e36d3aba-e35b-457e-8dae-5a6fcdd4d86b'},\n",
       "  {'_id': 'TUD_70d91e29-e24b-4e31-b701-b2357ffc3a8e'},\n",
       "  {'_id': 'TUD_a9711610-96bf-4741-af78-b3d393f0b875'},\n",
       "  {'_id': 'TUD_1d235433-6f66-42a6-87b1-ef2843f0db38'},\n",
       "  {'_id': 'TUD_a9f7c378-9896-4ab7-ad3d-b8c093c36ffc'},\n",
       "  {'_id': 'TUD_0edd0472-39df-4296-b692-e9916e79fb1e'},\n",
       "  {'_id': 'TUD_b0671ce6-355a-4c8a-a525-81931b0f92a0'},\n",
       "  {'_id': 'TUD_5acbd4cd-ab5f-486a-8268-61cd7416550d'},\n",
       "  {'_id': 'TUD_46a0d7b7-5f30-481b-a48c-de32513f69d3'},\n",
       "  {'_id': 'TUD_3dbdac61-5e13-432a-b05a-cb66f5f55b7d'},\n",
       "  {'_id': 'TUD_5d7ae484-2b60-40f4-b7b8-1ca0a32425b3'},\n",
       "  {'_id': 'TUD_1390209c-a35a-49af-a082-1c855663d124'},\n",
       "  {'_id': 'TUD_42501e33-6bdf-467a-8788-2d49e5098638'},\n",
       "  {'_id': 'TUD_11f0e772-3564-4e22-b4d7-fe94c6b69987'},\n",
       "  {'_id': 'TUD_62bee645-b8a0-40c6-be48-28ebbb541c42'},\n",
       "  {'_id': 'TUD_ce583533-45ea-4237-b18d-fe31272ea1ee'},\n",
       "  {'_id': 'TUD_008b12b6-5115-4c48-9dd0-f88999ccb34b'},\n",
       "  {'_id': 'TUD_34c62679-c826-4169-a84f-f101d012478e'},\n",
       "  {'_id': 'TUD_7ab16201-272c-4f18-899c-98a7fb527093'},\n",
       "  {'_id': 'TUD_69f7fbd3-4922-4526-b92d-0f3a6bbe6ba8'},\n",
       "  {'_id': 'TUD_bab69ac3-07b7-4dae-8b80-670443af2faa'},\n",
       "  {'_id': 'TUD_4bada907-7f8c-4dfb-bb80-b1c687bb5b7f'},\n",
       "  {'_id': 'TUD_355f297a-bc22-445e-a489-934582d1d1d2'},\n",
       "  {'_id': 'TUD_aafcf8fe-4292-406b-9c58-bc368fe55994'},\n",
       "  {'_id': 'TUD_9203ba22-72e0-4da5-bb50-f9e88339cc9a'},\n",
       "  {'_id': 'TUD_1eab77a9-5693-4fd1-abf7-0306233f4388'},\n",
       "  {'_id': 'TUD_9df0aca7-a587-413e-b18f-07bcd7e608b9'},\n",
       "  {'_id': 'TUD_9a3f74a3-6049-4c24-8b52-5ae044e46201'},\n",
       "  {'_id': 'TUD_437e234e-bed8-45e3-a9b4-1378abd1dcda'},\n",
       "  {'_id': 'TUD_9acfce4e-0cab-4d85-b1a3-ce9ed0cdb748'},\n",
       "  {'_id': 'TUD_4d437aa6-1593-423c-bc3a-d12d46806675'},\n",
       "  {'_id': 'TUD_f96e4cae-a5ea-408e-b520-9a1cc3c42f8d'},\n",
       "  {'_id': 'TUD_6bf48a02-371d-47fe-b642-a88f5c917a1d'},\n",
       "  {'_id': 'TUD_b3daf825-2d23-4161-ae1a-a1577614c33c'},\n",
       "  {'_id': 'TUD_077f4577-c9b8-4be7-91a7-49bf751e73ef'},\n",
       "  {'_id': 'TUD_4cd6734c-1976-4cd9-8d90-bd7145e5d2a6'},\n",
       "  {'_id': 'TUD_2bcc9188-aaaf-4232-8c85-4a0997a3935a'},\n",
       "  {'_id': 'TUD_a8128b6f-c8a1-4739-a845-8bdb3de536ce'},\n",
       "  {'_id': 'TUD_e7cf3e77-7b81-4028-8c7a-c2a62e94261b'},\n",
       "  {'_id': 'TUD_f2ecf719-d8d4-4197-be07-1822e84f6968'},\n",
       "  {'_id': 'TUD_5e0bf5b4-f361-4d36-8558-7cb76337fb0c'},\n",
       "  {'_id': 'TUD_64c8efc7-7a36-4acf-a493-147d5a1051d5'},\n",
       "  {'_id': 'TUD_333007e2-0fe5-43da-892c-d86b0a80d9ef'},\n",
       "  {'_id': 'TUD_6d74b3b5-cb4e-462c-be1d-91af20a8e057'},\n",
       "  {'_id': 'TUD_dea2ad58-69f6-46db-bea7-831a0cca7e46'},\n",
       "  {'_id': 'TUD_e91fb785-f2b8-459c-a5c7-ab926da5d9cf'},\n",
       "  {'_id': 'TUD_f4bb7503-1c62-4a0a-a77c-e119381e4307'},\n",
       "  {'_id': 'TUD_282aa7de-cba5-422c-98df-7ad24e89924b'},\n",
       "  {'_id': 'TUD_85873447-3211-43cb-a139-935f804e36ac'},\n",
       "  {'_id': 'TUD_2ad8a0a4-4536-4a3d-88ec-2734509868da'},\n",
       "  {'_id': 'TUD_6e483ba3-052d-4711-aeb1-a03e84ba2c08'},\n",
       "  {'_id': 'TUD_4c8919ec-534c-4da7-8e48-273485c2b9f2'},\n",
       "  {'_id': 'TUD_ef7d8499-ff37-4ebc-97d6-ee9804938f5c'},\n",
       "  {'_id': 'TUD_7fef8c52-6adc-45d5-8913-6d3fe044139b'},\n",
       "  {'_id': 'TUD_76ab40b5-ccdb-4db1-95ef-db71c01e0c7f'},\n",
       "  {'_id': 'TUD_827cfa3f-0027-45cd-9cf2-6fd43551db07'},\n",
       "  {'_id': 'TUD_633f3846-b8ea-4bf5-84d4-21752c584f06'},\n",
       "  {'_id': 'TUD_52874750-4eab-4974-8a42-8da5b700adff'},\n",
       "  {'_id': 'TUD_31002d86-b88d-46e2-9462-6c214a552543'},\n",
       "  {'_id': 'TUD_576da4a7-781c-4950-a36e-2366211022a7'},\n",
       "  {'_id': 'TUD_9bc1729f-6471-4dd6-b61b-8e2a16f3c951'},\n",
       "  {'_id': 'TUD_aec34e9d-a436-4631-b7ff-5563a99b73c0'},\n",
       "  {'_id': 'TUD_80363f35-b986-42fd-9880-e6ef96506daa'},\n",
       "  {'_id': 'TUD_d610163e-784c-4e55-893d-03c0b3a28fce'},\n",
       "  {'_id': 'TUD_5210165d-f011-4cd3-8551-a1374641065d'},\n",
       "  {'_id': 'TUD_256f5716-0947-48c3-946d-f47048cb3446'},\n",
       "  {'_id': 'TUD_28a5e49f-db97-4002-8a24-1ec9cedeaf75'},\n",
       "  {'_id': 'TUD_81232e85-5c72-4b02-8cfc-462b4996b633'},\n",
       "  {'_id': 'TUD_cff6cd3b-a587-42f2-a3ce-e735aebf87ce'},\n",
       "  {'_id': 'TUD_1b75db0a-60da-4a6c-8c02-6cc2550c90d0'},\n",
       "  {'_id': 'TUD_53c8e1e0-7a4c-43ed-9426-934c0a5a6522'},\n",
       "  {'_id': 'TUD_2b0bba39-b9a9-4c52-a06e-1fb20dcfd5b8'},\n",
       "  {'_id': 'TUD_e0704a99-6b12-4fc0-8310-c7cbabfe5a8d'},\n",
       "  {'_id': 'TUD_0e024a88-8c4f-4213-b1c3-67bfb552c669'},\n",
       "  {'_id': 'TUD_ebba15d9-71ac-42c3-89f7-221c6604828c'},\n",
       "  {'_id': 'TUD_dc26e6ac-c0f7-4313-8236-9d4742968441'},\n",
       "  {'_id': 'TUD_156aa284-e3d7-461d-ab4b-4fb4d5fc7116'},\n",
       "  {'_id': 'TUD_5ea21702-d6fb-484c-8fbf-15c5b8563ff1'},\n",
       "  {'_id': 'TUD_3175edae-5f98-46d2-b661-277370aba7fc'},\n",
       "  {'_id': 'TUD_db7e1836-73f2-465d-8f03-ab5b9d3d3018'},\n",
       "  {'_id': 'TUD_a6324bce-a3e7-4563-b8e8-2d3b43db837f'},\n",
       "  {'_id': 'TUD_bdf8976c-46b6-40e8-a336-e9ddfef7f565'},\n",
       "  {'_id': 'TUD_e1ec5f1f-a9ee-414e-9ee5-6f656fd42996'},\n",
       "  {'_id': 'TUD_e3b70e0e-ef65-4b17-9e28-2c29a1e40972'},\n",
       "  {'_id': 'TUD_7ee4b3eb-e136-4889-80f2-b440c5ff49be'},\n",
       "  {'_id': 'TUD_609562d5-b953-4f40-b85c-4e2506a68084'},\n",
       "  {'_id': 'TUD_c39ba351-4dec-4fa9-824f-6ff618a7980d'},\n",
       "  {'_id': 'TUD_c9d57c31-3359-47be-a8f6-98b2c1e7ebe8'},\n",
       "  {'_id': 'TUD_a44c7da7-2979-480f-bf8c-fb90e65df727'},\n",
       "  {'_id': 'TUD_cc5a729c-aa94-4b85-a4eb-16e0e781f24e'},\n",
       "  {'_id': 'TUD_11c817fc-334b-471d-a535-26ac8f0f448b'},\n",
       "  {'_id': 'TUD_4339deb2-8764-45ba-a688-a9078795f386'},\n",
       "  {'_id': 'TUD_3c57be43-0e51-4897-94d7-f6bcc2189a1d'},\n",
       "  {'_id': 'TUD_17adc7bd-5c82-4ad5-b1c8-a8b85b23db1f'},\n",
       "  {'_id': 'TUD_a7e5a7be-902a-4536-8247-7ce83b26ab4b'},\n",
       "  {'_id': 'TUD_ddbde9de-c7fe-4499-89fb-504b4241ea9c'},\n",
       "  {'_id': 'TUD_4b80dec5-66be-4b00-904d-e064ca35406b'},\n",
       "  {'_id': 'TUD_2e8d335a-4251-41d9-b1fd-1697be9fbf2e'},\n",
       "  {'_id': 'TUD_f724734b-eab2-4e50-b697-92671479d87e'},\n",
       "  {'_id': 'TUD_e7a452de-6859-42bc-97b6-3eb28815dc8b'},\n",
       "  {'_id': 'TUD_6b015fa3-5692-4285-815b-4d2901a6ff45'},\n",
       "  {'_id': 'TUD_b742f141-1351-432f-b551-6249e65ce822'},\n",
       "  {'_id': 'TUD_4590b1fc-6f44-4297-9e0f-04bac4c29334'},\n",
       "  {'_id': 'TUD_cadc9c6f-9da5-4d55-a0f2-cd48b9f0ee3b'},\n",
       "  {'_id': 'TUD_da1e4bbb-ce28-42d3-ae2b-f076ccc7ee91'},\n",
       "  {'_id': 'TUD_1b362496-af89-4ba1-b7f8-7088ccaf2a01'},\n",
       "  {'_id': 'TUD_cbdd109f-c692-46f4-b157-0e25bb46b2c7'},\n",
       "  {'_id': 'TUD_26b76962-0465-4785-ab6c-446dd6663262'},\n",
       "  {'_id': 'TUD_d1da3a56-14c5-4942-91a3-68e12a0e3103'},\n",
       "  {'_id': 'TUD_9cb25a9a-0d17-48c7-842c-3ef83ed044e5'},\n",
       "  {'_id': 'TUD_a775a647-83b8-43c0-978a-44bba80bb5d4'},\n",
       "  {'_id': 'TUD_e77bc353-149a-401d-baef-938450da160d'},\n",
       "  {'_id': 'TUD_e225decc-1acb-4e0d-ab28-11bbab434788'},\n",
       "  {'_id': 'TUD_0b5b81ae-bb45-4183-b815-6cac07f7fc3d'},\n",
       "  {'_id': 'TUD_f5bc5f90-97a6-46dc-839e-f5c532ae51f2'},\n",
       "  {'_id': 'TUD_07bc49be-571c-49de-a7a6-075cede74c54'},\n",
       "  {'_id': 'TUD_f150ebc6-9720-47b2-b7e7-2413eb82b957'},\n",
       "  {'_id': 'TUD_d16d440a-32cc-4f75-ba51-d47769c322b5'},\n",
       "  {'_id': 'TUD_bf259621-d718-432a-8bd3-4c003a6629a4'},\n",
       "  {'_id': 'TUD_2b48115c-19b0-4301-bcbe-b3e8ce6aacba'},\n",
       "  {'_id': 'TUD_6fef6b25-58c2-42d4-8271-b00b5ddb00b4'},\n",
       "  {'_id': 'TUD_59b88cba-fa73-424d-868a-864b921155c1'},\n",
       "  {'_id': 'TUD_e74c11d4-b9be-4b89-ae5b-b7e5bc853717'},\n",
       "  {'_id': 'TUD_d7afed76-8f00-4f78-bbb0-193e30d7afc0'},\n",
       "  {'_id': 'TUD_b0a8ae03-18b9-4a0e-9761-64ffd2851074'},\n",
       "  {'_id': 'TUD_6e2807a1-fdec-4fb4-9569-fd1ce2eb8264'},\n",
       "  {'_id': 'TUD_9276fd1a-300a-4c4c-8971-780fdf544a66'},\n",
       "  {'_id': 'TUD_3f903786-381a-47a4-a7a5-93cfd370361a'},\n",
       "  {'_id': 'TUD_56e646dc-d5c7-482b-8326-90e0de4ea419'},\n",
       "  {'_id': 'TUD_24d84ce3-d325-4dad-a1f1-ab35f942def7'},\n",
       "  {'_id': 'TUD_adef294a-56f3-4e17-a17e-61b7518a19b9'},\n",
       "  {'_id': 'TUD_d21d7b64-7600-47b6-b01c-3a543ca5d6d2'},\n",
       "  {'_id': 'TUD_29181e27-003c-4ffa-9c77-aeb53783ad91'},\n",
       "  {'_id': 'TUD_29f849e2-ed5f-4295-94f0-7092ead693bf'},\n",
       "  {'_id': 'TUD_ba7f3919-90c6-453b-a4a5-bbfd5f345dcf'},\n",
       "  {'_id': 'TUD_dd7b9002-03c9-4976-8bf8-6c91885608ef'},\n",
       "  {'_id': 'TUD_8f824b85-adf4-4576-a3d1-a2563c9e5734'},\n",
       "  {'_id': 'TUD_0e9a3aed-2277-43c9-b574-bb9eed326fa5'},\n",
       "  {'_id': 'TUD_08fc576d-7df8-4e8f-b813-52325df5d559'},\n",
       "  {'_id': 'TUD_6579ee97-6315-48ed-a7f1-1fca8870f130'},\n",
       "  {'_id': 'TUD_169e44a3-cc02-44bc-92f6-573461fe46e1'},\n",
       "  {'_id': 'TUD_54f7d7a3-de8a-4127-9cf3-897eb79a1860'},\n",
       "  {'_id': 'TUD_26955829-f8b7-4058-afc1-eea84ed4ea11'},\n",
       "  {'_id': 'TUD_e5a817a4-ce0a-4dd3-afd4-d70660b63d16'},\n",
       "  {'_id': 'TUD_1dfbb52d-761b-4cb5-9414-d858f8ab2573'},\n",
       "  {'_id': 'TUD_0a5a2b01-d3cd-4b8a-88ba-b02ed28a24d3'},\n",
       "  {'_id': 'TUD_dacac69b-b5f8-4216-a143-825a0e3bc2ea'},\n",
       "  {'_id': 'TUD_610cbb5e-dc40-4068-8064-6c2c5e0d1364'},\n",
       "  {'_id': 'TUD_57a87999-c68c-4a9e-b5db-2a2eb2aef7eb'},\n",
       "  {'_id': 'TUD_6083701f-09fa-4cc2-b45e-b1b74c117994'},\n",
       "  {'_id': 'TUD_ea90cd2c-90f9-4726-accd-1ee8d5967f26'},\n",
       "  {'_id': 'TUD_9266518e-2052-4f48-a13d-0e9c698125bf'},\n",
       "  {'_id': 'TUD_133e2ab3-185f-43fe-a2e7-e54a823d91e8'},\n",
       "  {'_id': 'TUD_07be4544-d0ba-4767-804b-12c313b10a0f'},\n",
       "  {'_id': 'TUD_b1ea7059-773d-4d48-a28b-32bb24c7f201'},\n",
       "  {'_id': 'TUD_6eb43736-74ad-4512-9be9-3f3c5ac9b50f'},\n",
       "  {'_id': 'TUD_f0bc3711-a40a-432c-b153-9e1c8b8e2935'},\n",
       "  {'_id': 'TUD_b157de07-e5ce-4dba-8eae-154a0002a4f5'},\n",
       "  {'_id': 'TUD_36851412-be13-486a-92df-09680ad98b3b'},\n",
       "  {'_id': 'TUD_50b4176e-21c7-4271-988a-52c13a4f2de3'},\n",
       "  {'_id': 'TUD_5c01eb69-0d5b-4016-8b55-b450e87c24e2'},\n",
       "  {'_id': 'TUD_e7ed020a-ee53-4010-a14c-4db3ae31a01b'},\n",
       "  {'_id': 'TUD_a22c55d7-4c69-44c9-bc9f-03652b4ddfdf'},\n",
       "  {'_id': 'TUD_adda5ce3-0931-4282-8086-fb0cb4fc3d5d'},\n",
       "  {'_id': 'TUD_740995b0-2576-42ea-aa49-940c6e10a7ba'},\n",
       "  {'_id': 'TUD_bbed8ce5-36a8-4773-a461-33f19c5f774d'},\n",
       "  {'_id': 'TUD_f43422a6-ef31-4f61-ae7a-f71d3b3c645b'},\n",
       "  {'_id': 'TUD_9328b51a-7eb5-4206-b463-48d99b86aafd'},\n",
       "  {'_id': 'TUD_c0c07509-513f-4aa1-a07c-78f5c9ff5684'},\n",
       "  {'_id': 'TUD_b806e4df-0982-4b2e-9e63-7299ab3a8bff'},\n",
       "  {'_id': 'TUD_3758d2cb-d0af-45d1-9f14-93e1c86f60ed'},\n",
       "  {'_id': 'TUD_204b09ee-971b-4035-a4b1-616ee50afbba'},\n",
       "  {'_id': 'TUD_74af4924-f262-4136-8f44-29c6be14534e'},\n",
       "  {'_id': 'TUD_8ef36410-06c2-4372-a9fe-e0e45b264a73'},\n",
       "  {'_id': 'TUD_40148cf1-8b85-49df-9f5a-55ac257a04f7'},\n",
       "  {'_id': 'TUD_81f376a3-2c9a-4d24-b547-6b384dc0aef6'},\n",
       "  {'_id': 'TUD_493487ad-ac1e-4ad5-9264-48be7ebc2f74'},\n",
       "  {'_id': 'TUD_68ef1d61-f3c3-43e8-bacd-d2e97bd0baa1'},\n",
       "  {'_id': 'TUD_4fd01d95-6bf6-4864-8662-f7adf837e2f5'},\n",
       "  {'_id': 'TUD_f17092aa-4d78-4ef3-a80c-365825e810f1'},\n",
       "  {'_id': 'TUD_117aa738-99ea-4357-b880-769b58c84d97'},\n",
       "  {'_id': 'TUD_c3faec41-007f-4b2c-acbe-eb2eb89efd32'},\n",
       "  {'_id': 'TUD_d229c59f-c03f-4907-b9e8-67b83b619ac3'},\n",
       "  {'_id': 'TUD_29251f71-81ea-475c-8a3f-5795a6d1b341'},\n",
       "  {'_id': 'TUD_2eb0995d-2c58-447a-99df-061a2883aba0'},\n",
       "  {'_id': 'TUD_8c42378a-8769-4a48-a7fb-f5457ede0759'},\n",
       "  {'_id': 'TUD_6850e673-4da7-4026-a488-d80c4174170b'},\n",
       "  {'_id': 'TUD_becdcccc-4ef8-4eac-91df-d77571c629de'},\n",
       "  {'_id': 'TUD_64ee5526-8c9e-4013-9019-c63a63413ca2'},\n",
       "  {'_id': 'TUD_7dd09f0f-254b-42f1-a852-b61cb9f2bed3'},\n",
       "  {'_id': 'TUD_f7db8070-8470-484d-a7ce-ad2b81ea3318'},\n",
       "  {'_id': 'TUD_fa059b99-b750-4122-a31d-eee8495115a8'},\n",
       "  {'_id': 'TUD_d5c61e4c-9550-4c58-a069-fc8e940945ce'},\n",
       "  {'_id': 'TUD_8c8e1930-31b4-4e68-85cd-e6e4ba49f4f2'},\n",
       "  {'_id': 'TUD_0b59e5db-72ee-460e-bad6-63912f5238c0'},\n",
       "  {'_id': 'TUD_6d553910-2169-4c2c-a494-bdc2abd838b9'},\n",
       "  {'_id': 'TUD_2287ef2c-248f-409b-959a-eb1f6236e4a4'},\n",
       "  {'_id': 'TUD_2356d539-3500-4bb9-bfec-a2bc8d089b59'},\n",
       "  {'_id': 'TUD_7330511f-c22e-4ad1-a0ce-44c1c2747c05'},\n",
       "  {'_id': 'TUD_0b5408a7-933c-4369-8d48-1db5d3269d72'},\n",
       "  {'_id': 'TUD_92a3bc67-53cf-45ab-a3cd-e3c20fab9ce8'},\n",
       "  {'_id': 'TUD_643fd074-b051-431a-86c1-181c67497672'},\n",
       "  {'_id': 'TUD_409710f1-9646-40d4-a889-11c0a41a724d'},\n",
       "  {'_id': 'TUD_5f6beeb7-6485-4000-9894-677343c35097'},\n",
       "  {'_id': 'TUD_d7e33788-1ab5-4bb6-b2ec-1b31a5b85fc1'},\n",
       "  {'_id': 'TUD_f1c5891e-91fe-42be-bbd8-d0f5552ccf8a'},\n",
       "  {'_id': 'TUD_7c8f5d4c-d680-4ba2-98da-a2980b35a269'},\n",
       "  {'_id': 'TUD_021d1ea2-12ab-477e-b23e-77875eed6b9d'},\n",
       "  {'_id': 'TUD_9e8316bf-200e-4d33-9322-42f8da8233b1'},\n",
       "  {'_id': 'TUD_4ba70bd1-8496-4df3-9b2c-f42c700f894b'},\n",
       "  {'_id': 'TUD_7d9c6c44-734d-4ffd-80d8-38997ff4d747'},\n",
       "  {'_id': 'TUD_be0bd616-57ab-46d6-b02d-84af9dd6c717'},\n",
       "  {'_id': 'TUD_44dac56b-e0a5-4818-811c-81d9a408edee'},\n",
       "  {'_id': 'TUD_2e41c0ca-6128-4137-842a-6c0b5dab6009'},\n",
       "  {'_id': 'TUD_e2234250-950d-4eb5-9f2e-b5b8e67af2e5'},\n",
       "  {'_id': 'TUD_b56f7a38-b08c-48e1-93dd-4c0401fca76c'},\n",
       "  {'_id': 'TUD_b6b6915b-ac95-45ad-838d-c8fad5b8a171'},\n",
       "  {'_id': 'TUD_6f6b3972-c290-45eb-8b77-617129cacdac'},\n",
       "  {'_id': 'TUD_7421072c-e61b-47b1-87a0-d2cf85050817'},\n",
       "  {'_id': 'TUD_7b958de6-2087-42d9-96e6-31a6b14072fc'},\n",
       "  {'_id': 'TUD_bd900036-40f4-432d-bfab-425cdebc466e'},\n",
       "  {'_id': 'TUD_6b62bb3c-9889-421a-92cc-b0ab436c4dde'},\n",
       "  {'_id': 'TUD_d76287f8-0b94-4919-abab-efc9f11ff8a1'},\n",
       "  {'_id': 'TUD_f5fa2307-fe28-4570-a679-b4ac64d007e6'},\n",
       "  {'_id': 'TUD_7e70359f-7ddd-4914-b2f7-b782103708ab'},\n",
       "  {'_id': 'TUD_4b4a7448-0807-4a54-ac21-a7dafa7fbf92'},\n",
       "  {'_id': 'TUD_4060494b-3725-4abc-aefb-8235fd05871c'},\n",
       "  {'_id': 'TUD_3d78c977-ef85-4f9f-bcba-2e0a37c13745'},\n",
       "  {'_id': 'TUD_2f902ffc-2f44-45e3-b7ba-414c7420f764'},\n",
       "  {'_id': 'TUD_bf950f1e-db9f-4e3e-8ca1-5b00a87a1452'},\n",
       "  {'_id': 'TUD_322cd4ef-69bd-4dd9-98b2-539b598986b6'},\n",
       "  {'_id': 'TUD_e1391687-ab07-4f62-bc1a-3343dc577f23'},\n",
       "  {'_id': 'TUD_b7117ae8-33ee-45a6-ba70-5e9e731a56b6'},\n",
       "  {'_id': 'TUD_6bae6ac2-71e8-4ef0-9d61-7f3c86dfb5cc'},\n",
       "  {'_id': 'TUD_060fc436-07fe-47f8-b418-d53e8fb2c75c'},\n",
       "  {'_id': 'TUD_53fece0d-c512-4bd8-a0da-ea3311b35156'},\n",
       "  {'_id': 'TUD_73c6c6d2-cba7-423c-a7f0-7a07a88f5499'},\n",
       "  {'_id': 'TUD_cccad780-b786-4dcd-a10d-e1496d07eb6f'},\n",
       "  {'_id': 'TUD_5ae85b5f-ce1a-4ae3-8e8c-1991b45b08eb'},\n",
       "  {'_id': 'TUD_555299fd-6918-4dfb-b883-5f642a97b324'},\n",
       "  {'_id': 'TUD_2f31c7c5-5019-4cab-a9c5-65528f886a76'},\n",
       "  {'_id': 'TUD_e12ddb8d-ab56-411a-bbab-dbc942fe8b60'},\n",
       "  {'_id': 'TUD_d2d8c74b-9580-4101-9b72-5a5370a4bcd8'},\n",
       "  {'_id': 'TUD_52b586ea-6144-4b4e-a5a1-b05255ce493a'},\n",
       "  {'_id': 'TUD_93f9d3af-38f7-45f4-ba62-e921c386f3c9'},\n",
       "  {'_id': 'TUD_34a6aa6d-6038-414e-9541-8906fe6e2f5e'},\n",
       "  {'_id': 'TUD_1f285e6f-31c3-4173-9281-9dca204d28fe'},\n",
       "  {'_id': 'TUD_359825f9-ed16-43f4-a998-d2028a6c3204'},\n",
       "  {'_id': 'TUD_73ffa055-c46d-43f6-9a1a-747e0af16f43'},\n",
       "  {'_id': 'TUD_36a62709-cb33-4c4a-88b6-0ad9fba1555b'},\n",
       "  {'_id': 'TUD_d9b524b0-d2e1-4bde-883a-cc6313a1d8c0'},\n",
       "  {'_id': 'TUD_3efc8aae-e31f-47a4-a92c-b9da05917ada'},\n",
       "  {'_id': 'TUD_a65f49cf-5447-4816-8352-8c42846fc103'},\n",
       "  {'_id': 'TUD_894e4234-ed2b-411f-be90-550766f97cd5'},\n",
       "  {'_id': 'TUD_0a0685fc-ce0a-4580-a355-978810005b71'},\n",
       "  {'_id': 'TUD_35192f4a-f466-4d73-aa3c-5d77f8622709'},\n",
       "  {'_id': 'TUD_7e04a67d-fc3c-4d11-a1e0-67043044e637'},\n",
       "  {'_id': 'TUD_eb9462c4-078b-40ae-9a14-ae61ba46dfa6'},\n",
       "  {'_id': 'TUD_e5f90dec-7e32-411c-a228-7210c6be2ea6'},\n",
       "  {'_id': 'TUD_7fdb578a-a3e3-430c-b257-c85bfc45d3d9'},\n",
       "  {'_id': 'TUD_7b9246f5-0bfb-4eb4-8566-61415fd14cac'},\n",
       "  {'_id': 'TUD_d867d3a3-5534-4644-b6f4-20e7d7ae69af'},\n",
       "  {'_id': 'TUD_5e1bf344-f581-4ce6-a083-94573322f653'},\n",
       "  {'_id': 'TUD_35cd1bee-7723-43a9-b581-1cc4dc4ce07d'},\n",
       "  {'_id': 'TUD_cbdf86c6-9979-44c5-9f0e-c0bb96bb16d9'},\n",
       "  {'_id': 'TUD_53dd3e0b-5c31-43b1-b8a8-767951c10249'},\n",
       "  {'_id': 'TUD_ca07093c-44f1-40e6-8dc6-f01d9b7f61c8'},\n",
       "  {'_id': 'TUD_51b11f1c-699a-42f4-9373-b5c9697fde74'},\n",
       "  {'_id': 'TUD_52256ba7-e744-49d4-8bcd-a5ded6052ad8'},\n",
       "  {'_id': 'TUD_00f6916d-a226-44ee-bbe1-9cb03d49bd37'},\n",
       "  {'_id': 'TUD_d5d68d7a-1c30-4594-96bf-a03a8b4b8a8f'},\n",
       "  {'_id': 'TUD_3cec89f0-9a2e-4a09-a015-6a7f30ac0c96'},\n",
       "  {'_id': 'TUD_f672209e-162a-4ac6-8317-1f18b287b8c9'},\n",
       "  {'_id': 'TUD_29893270-5711-46be-9476-dc33f837a0d1'},\n",
       "  {'_id': 'TUD_9ecad815-9dfb-4a33-aea8-bf4b83beb8bc'},\n",
       "  {'_id': 'TUD_215065f7-d22d-4189-a5cb-0b2e91897cd1'},\n",
       "  {'_id': 'TUD_2c345e58-b4b6-426b-9f28-f730a000baf5'},\n",
       "  {'_id': 'TUD_f28f32fb-f420-44c1-9683-bc2a5e0ec993'},\n",
       "  {'_id': 'TUD_f3242bfa-1b32-4d83-ad31-c188c9b6377b'},\n",
       "  {'_id': 'TUD_f2fe4764-4c8c-4164-97f8-dc550dbfc73c'},\n",
       "  {'_id': 'TUD_eb91db81-587c-48f5-a570-321666a7dc75'},\n",
       "  {'_id': 'TUD_67387fb8-a443-41c3-8452-0cdcf17f0e5c'},\n",
       "  {'_id': 'TUD_dc4257fb-f1d4-4fa4-a289-63805bce8258'},\n",
       "  {'_id': 'TUD_77bb79d5-8909-40ec-8ae0-e8384594fcbf'},\n",
       "  {'_id': 'TUD_6e3fb29a-fcd1-4099-bb75-77eff2a03a90'},\n",
       "  {'_id': 'TUD_efbce68d-3ff5-4025-a602-0541af87648b'},\n",
       "  {'_id': 'TUD_151bc710-b9ac-437d-a43f-a429223314ee'},\n",
       "  {'_id': 'TUD_98a8d01e-6259-4fed-8704-5af0615304d2'},\n",
       "  {'_id': 'TUD_774bf8c6-8ded-4a98-9b6a-77454262a548'},\n",
       "  {'_id': 'TUD_07102bcb-dc4e-4bd4-9691-9de832ffa2e1'},\n",
       "  {'_id': 'TUD_5e14890b-d4e3-469d-bc3b-cc1f2f6df945'},\n",
       "  {'_id': 'TUD_a8f6363b-5362-4c9b-a928-3a4542bf4774'},\n",
       "  {'_id': 'TUD_89ca1773-7dd7-40ec-8abc-38bc22719b49'},\n",
       "  {'_id': 'TUD_74069df0-87a2-46be-9a96-0f0df06b922e'},\n",
       "  {'_id': 'TUD_64d94833-5cb8-4198-bc91-ac13564141d5'},\n",
       "  {'_id': 'TUD_9c4be69a-cb01-417c-84d5-a65ed57c72c8'},\n",
       "  {'_id': 'TUD_80ea30e9-11c3-4e7e-b250-7de6015f4744'},\n",
       "  {'_id': 'TUD_dcc2005e-c6f3-43ff-b003-4cf3bf1770f3'},\n",
       "  {'_id': 'TUD_f98d2c87-e564-4cf4-a9ab-aeeccceefa38'},\n",
       "  {'_id': 'TUD_229ece19-fd8c-4ca1-a38f-dd52c6aa3175'},\n",
       "  {'_id': 'TUD_63251f1a-8c42-4153-94f4-c85c92a8e950'},\n",
       "  {'_id': 'TUD_2eac935e-cdb1-4c0c-92d1-cbcc8dd2d867'},\n",
       "  {'_id': 'TUD_6ed0e1cc-7fdd-4b0a-abfc-5e353adf0279'},\n",
       "  {'_id': 'TUD_3c4c5c99-c866-454d-ae15-5fb62eee0ba8'},\n",
       "  {'_id': 'TUD_32f5e7d4-54b5-4fc1-9c46-9da21d17f0d4'},\n",
       "  {'_id': 'TUD_2632334a-f9ee-41b0-aaad-9f77375fc256'},\n",
       "  {'_id': 'TUD_8b892951-0000-431b-a313-89190f335d0c'},\n",
       "  {'_id': 'TUD_99424150-5e7f-43d5-a864-12841c9f31f5'},\n",
       "  {'_id': 'TUD_8233ed00-a892-4b18-beda-057e539a7d73'},\n",
       "  {'_id': 'TUD_6f85ea91-2398-4d98-ad75-cfd8ffdaa768'},\n",
       "  {'_id': 'TUD_f1f08f26-416e-4f50-a518-e4aae5fb14d2'},\n",
       "  {'_id': 'TUD_22f3bdbb-2ffa-4478-85d9-1f71eae8abf0'},\n",
       "  {'_id': 'TUD_a4c07b84-24ed-417a-9ad7-779bdd2422f0'},\n",
       "  {'_id': 'TUD_acc16325-1c25-445d-ac3c-02c3e89170d4'},\n",
       "  {'_id': 'TUD_cbec7bdd-4bab-4132-93cd-359587b9bf46'},\n",
       "  {'_id': 'TUD_a6c9be48-8f89-4165-b556-6525fe28cc11'},\n",
       "  {'_id': 'TUD_2fe353d4-1988-4790-aa87-bde1626b68f6'},\n",
       "  {'_id': 'TUD_0ed5ce33-60ac-432e-a5ff-5b3f10037a8b'},\n",
       "  {'_id': 'TUD_1647e1cb-84fd-46ca-b1e1-21aaf38ef30b'},\n",
       "  {'_id': 'TUD_ff0d73fa-339b-46c2-930c-f0e7b58625b6'},\n",
       "  {'_id': 'TUD_8d9193a3-8945-4881-985a-ad1a57d07477'},\n",
       "  {'_id': 'TUD_31ab95ab-384a-4af6-a606-26331f98e099'},\n",
       "  {'_id': 'TUD_f51f6f82-f660-4357-8b18-a8ef1fc0283a'},\n",
       "  {'_id': 'TUD_35d0f569-200b-452b-b470-34a56b803d7f'},\n",
       "  {'_id': 'TUD_3f677e31-aa91-4641-a7a2-db8295ffa5fd'},\n",
       "  {'_id': 'TUD_099bac02-8424-4dfc-9658-1fc9cdf5292d'},\n",
       "  {'_id': 'TUD_0ff8e807-9166-47cc-b803-1edc2803a1b8'},\n",
       "  {'_id': 'TUD_3137bc78-cfa1-4207-957f-6b91eae62a2f'},\n",
       "  {'_id': 'TUD_e141272c-80aa-40b3-8856-49ba0e7f35c4'},\n",
       "  {'_id': 'TUD_9e4b5949-f628-40af-b06c-c218a7a7cdb8'},\n",
       "  {'_id': 'TUD_20d263f3-6b9b-4fc9-ac6d-0d4baec896ce'},\n",
       "  {'_id': 'TUD_2bee12f5-22da-45fc-9576-c61808562a78'},\n",
       "  {'_id': 'TUD_06063c8a-1410-444c-bc6b-6b05e14af6f4'},\n",
       "  {'_id': 'TUD_8dd60526-5399-49e9-afe9-9f6e19c49ca7'},\n",
       "  {'_id': 'TUD_2e41be6a-5220-421c-bef8-bbb8f91c128f'},\n",
       "  {'_id': 'TUD_003f5644-69f1-42da-b462-2184866c1080'},\n",
       "  {'_id': 'TUD_8a569a87-a972-480c-bd7d-11292b3f5cdc'},\n",
       "  {'_id': 'TUD_bdad6dd4-c730-406a-995f-563e74c04616'},\n",
       "  {'_id': 'TUD_7604ccff-d585-4069-8e0e-bbaf8c90ad31'},\n",
       "  {'_id': 'TUD_b89345db-2993-4d27-a13f-25b2c0eb4126'},\n",
       "  {'_id': 'TUD_d41522e3-690a-4eb7-a352-652d39d7ac81'},\n",
       "  {'_id': 'TUD_c09bd3de-68a7-46ea-8a5b-b6b557429a93'},\n",
       "  {'_id': 'TUD_8cf81c2f-6ccb-4136-a6bc-51dfc8ffcab8'},\n",
       "  {'_id': 'TUD_a2fa837d-a669-4613-bec6-2e59e923e95d'},\n",
       "  {'_id': 'TUD_8fd22239-6559-48bd-8f55-54c33d64dd32'},\n",
       "  {'_id': 'TUD_932574a3-5a38-4c3d-9c34-c9ae11a161d2'},\n",
       "  {'_id': 'TUD_505300b0-f421-4470-b63e-cdb9154f7d54'},\n",
       "  {'_id': 'TUD_a304b6c7-6e5a-418e-9ac9-c2e9fb359f0b'},\n",
       "  {'_id': 'TUD_c9afa796-39a8-4487-8e56-2a7f020cedfe'},\n",
       "  {'_id': 'TUD_c049c3c2-c8d8-4a3c-86ef-222846e91291'},\n",
       "  {'_id': 'TUD_1922c463-91af-4c4f-8720-a89e7da4f8f6'},\n",
       "  {'_id': 'TUD_5d8e2c24-1262-42d6-8109-5d7b3e6f8755'},\n",
       "  {'_id': 'TUD_a619ad13-090c-466f-ad4f-f3ea51cfdd07'},\n",
       "  {'_id': 'TUD_5ef1696a-0ef8-4d4c-a807-3d0fd3247b1d'},\n",
       "  {'_id': 'TUD_eb91654e-87d8-4a3f-957e-93b8b94452ea'},\n",
       "  {'_id': 'TUD_b26fb6be-a3e1-4968-9044-01fc6a029926'},\n",
       "  {'_id': 'TUD_2a833efc-b55f-49bb-8965-a6c23605f621'},\n",
       "  {'_id': 'TUD_c9772109-7e5e-4858-8c10-6c880cbbf847'},\n",
       "  {'_id': 'TUD_5f7ba046-3a5c-46bb-8368-896a88f5e4a7'},\n",
       "  {'_id': 'TUD_2912a5b2-2cea-42ba-9ba4-1ca06b08d9d6'},\n",
       "  {'_id': 'TUD_2b238ade-cf76-4220-a407-4f3465d56dbc'},\n",
       "  {'_id': 'TUD_7d06198f-bf9f-4b3c-aa89-41c2a12d31ae'},\n",
       "  {'_id': 'TUD_51d9ddff-4ff1-494d-8070-c246b073e3d4'},\n",
       "  {'_id': 'TUD_d2a72d3f-d28b-4d9d-998a-b030c2f28aed'},\n",
       "  {'_id': 'TUD_07210c9d-6a0d-4329-8de9-3c7973f55da3'},\n",
       "  {'_id': 'TUD_f36ef6c3-0742-4372-a2f1-7f34f686ded6'},\n",
       "  {'_id': 'TUD_93137061-e3ef-418e-aa7a-b7539f70af50'},\n",
       "  {'_id': 'TUD_f2ae49ae-d01b-4374-b406-b4c1810ea364'},\n",
       "  {'_id': 'TUD_44f0676a-b4ee-491c-94c9-fd4c6ed29d5c'},\n",
       "  {'_id': 'TUD_230f983a-1119-4b08-80ee-be3b120f2d1f'},\n",
       "  {'_id': 'TUD_56227569-cafb-4b75-9b0a-c40199c7a894'},\n",
       "  {'_id': 'TUD_eca407ca-512c-4b17-82c2-dae9236a2066'},\n",
       "  {'_id': 'TUD_1e20f67a-d211-49dd-a32b-ea19a12b2e87'},\n",
       "  {'_id': 'TUD_ccdbdd3f-7871-4db1-8def-1bc9d710aed6'},\n",
       "  {'_id': 'TUD_90443566-8eb6-499d-8f7a-945344029c29'},\n",
       "  {'_id': 'TUD_a8b5fb6e-074b-4df0-97b2-6212a56552d8'},\n",
       "  {'_id': 'TUD_b50e0abb-f5d9-42af-88a2-6ddf31c8419a'},\n",
       "  {'_id': 'TUD_f444e6d3-cfa1-47dc-9fb4-9ca6978b4249'},\n",
       "  {'_id': 'TUD_bc53cc52-8507-43a0-9325-d4304067f677'},\n",
       "  {'_id': 'TUD_486154d6-b535-4a2a-b394-c2f7f3f43ac4'},\n",
       "  {'_id': 'TUD_b913c0ee-fa19-44a6-a7ef-41fecde7125c'},\n",
       "  {'_id': 'TUD_5bb2bb3c-c545-4d7f-9502-186ac0f5956e'},\n",
       "  {'_id': 'TUD_6a2cad07-86c3-4a16-9d04-1224ef4d900f'},\n",
       "  {'_id': 'TUD_1a95dc0b-64f1-42f1-9971-1f8bcb146b1f'},\n",
       "  {'_id': 'TUD_682a56ed-8e21-4b70-af11-0e8e9e298fa2'},\n",
       "  {'_id': 'TUD_218b896a-af74-4637-a0e7-fcd4e4743fed'},\n",
       "  {'_id': 'TUD_3e8a66e5-80f4-466d-b6bb-961f9896b800'},\n",
       "  {'_id': 'TUD_dd58900a-a201-474b-b02a-4feab83b25d1'},\n",
       "  {'_id': 'TUD_60fee9f1-7f5a-4915-8f09-2a3bcdbdd55c'},\n",
       "  {'_id': 'TUD_39b1653b-cde7-419b-bcd0-8549b6e34db5'},\n",
       "  {'_id': 'TUD_4d4e1516-3c86-4b78-8c13-bf89953af029'},\n",
       "  {'_id': 'TUD_ff57aedb-20f8-42cb-bc6f-513b8c8e94c5'},\n",
       "  {'_id': 'TUD_98015359-f99c-42a0-bb63-4e99c975decc'},\n",
       "  {'_id': 'TUD_a1689901-8ebe-4d2a-bab9-9e9569e3d83c'},\n",
       "  {'_id': 'TUD_6980f3d8-362c-412b-a560-565fd14d59c4'},\n",
       "  {'_id': 'TUD_29c201ec-0813-45b7-8d52-53371868150c'},\n",
       "  {'_id': 'TUD_5050b8ae-184b-40c6-9640-c4ba7fb875c6'},\n",
       "  {'_id': 'TUD_076ec1f6-d6c6-4266-afc5-dd71e24d65b2'},\n",
       "  {'_id': 'TUD_0eb48447-21ff-4202-85c0-25017c9fea6a'},\n",
       "  {'_id': 'TUD_3f184afb-241d-4d54-827d-75fcd59211a5'},\n",
       "  {'_id': 'TUD_e9b513c8-751b-45a7-9d99-a51177c918a2'},\n",
       "  {'_id': 'TUD_70ad3815-d280-4e2b-8810-43ecdcf2086e'},\n",
       "  {'_id': 'TUD_96cb75a3-7cf9-41a4-bce9-bc6a6ff5eb7e'},\n",
       "  {'_id': 'TUD_a6f87376-f084-4f33-bcfc-ced18c6c0fc4'},\n",
       "  {'_id': 'TUD_02156130-7e6c-445a-922d-98ed37bf87e3'},\n",
       "  {'_id': 'TUD_740626ef-fe0c-42b6-a177-c50270aaeca5'},\n",
       "  {'_id': 'TUD_e848e945-aa35-49c5-b62c-53baac9c06c9'},\n",
       "  {'_id': 'TUD_c4826844-77f6-4ee4-8b62-799fdbf12ee5'},\n",
       "  {'_id': 'TUD_de9a0da0-3d86-4a83-b25f-f93fd5bcc058'},\n",
       "  {'_id': 'TUD_f310b31e-aaec-43fe-9c9c-90e46076c0b9'},\n",
       "  {'_id': 'TUD_1af94380-1414-4497-bfc6-a67b213de050'},\n",
       "  {'_id': 'TUD_976110c5-9254-4994-9630-a7a03687f469'},\n",
       "  {'_id': 'TUD_4e36121c-da15-46ab-953e-d6a87d1d4271'},\n",
       "  {'_id': 'TUD_e695c2ce-e7f8-4f55-b073-e4768cf39bbe'},\n",
       "  {'_id': 'TUD_1a9dbc80-aeda-4564-a5d3-878eabc062b5'},\n",
       "  {'_id': 'TUD_b9c99dac-329c-4e57-b27c-d447cdf916bc'},\n",
       "  {'_id': 'TUD_acf11f8a-631b-4069-94fe-89bf746fdee1'},\n",
       "  {'_id': 'TUD_59d84718-b299-48f7-bdef-62c57377dfcf'},\n",
       "  {'_id': 'TUD_85cef785-c17c-41d7-9a58-6a183c468523'},\n",
       "  {'_id': 'TUD_ef93503c-ce97-41d4-baf3-c7e7c08ed355'},\n",
       "  {'_id': 'TUD_464eb73e-e250-4174-b792-36cc1ec15331'},\n",
       "  {'_id': 'TUD_ee554cc0-a7e6-49ee-939e-312bcef3d674'},\n",
       "  {'_id': 'TUD_d29a69f8-4410-4e74-a533-e9bf6e82f015'},\n",
       "  {'_id': 'TUD_f23ea108-9c84-4d4b-acdf-ecdd87188572'},\n",
       "  {'_id': 'TUD_cc957424-7611-4dd5-b84c-91ee9b1283d0'},\n",
       "  {'_id': 'TUD_4e24a08a-6b73-4c30-bcf1-ac6af8c0e1f9'},\n",
       "  {'_id': 'TUD_ba2c8674-878c-4f0e-814c-e93b96e7f788'},\n",
       "  {'_id': 'TUD_6e0e2b2b-9c26-4dc4-bca4-c177f936e90b'},\n",
       "  {'_id': 'TUD_66591440-00bd-4e67-a9b8-6738334909a3'},\n",
       "  {'_id': 'TUD_0fa98a15-e579-4542-8c9f-cfecf3b64272'},\n",
       "  {'_id': 'TUD_48b95ec8-a5aa-477d-8640-64a48d3d4fdf'},\n",
       "  {'_id': 'TUD_76f1c917-b89d-42e0-b5b3-14c3cdb11f7a'},\n",
       "  {'_id': 'TUD_bbe25dba-c11b-4be2-bda2-8b38fef2b6ca'},\n",
       "  {'_id': 'TUD_3eacd020-fc29-472b-9088-cec44de14a0a'},\n",
       "  {'_id': 'TUD_91601ffa-d704-45ab-a528-a3b83224fed7'},\n",
       "  {'_id': 'TUD_7a8aa949-fdaf-4ea9-bb0a-3a77b49557f9'},\n",
       "  {'_id': 'TUD_393b73c9-2943-49e6-9a72-30d0e590cb77'},\n",
       "  {'_id': 'TUD_69d2d334-8653-406b-b63a-fc195021a4d8'},\n",
       "  {'_id': 'TUD_77807364-bcad-47e1-92dd-5d3bcef92a7e'},\n",
       "  {'_id': 'TUD_e23175cb-d1a8-4cb6-af7b-9909f6412cf5'},\n",
       "  {'_id': 'TUD_80c01881-4210-4979-8820-57ed80b8995c'},\n",
       "  {'_id': 'TUD_7f48e4c8-22df-4b99-9028-80fed9add7ef'},\n",
       "  {'_id': 'TUD_0fcd09e9-0a0f-4d23-a155-fb03aa8dcbcc'},\n",
       "  {'_id': 'TUD_17f35cb5-61dc-47a2-a97d-d30261f62b6d'},\n",
       "  {'_id': 'TUD_681a1aee-f2df-4477-bf66-523c6204d5c1'},\n",
       "  {'_id': 'TUD_8f0cd1af-dcab-4124-aa42-50f9240c236c'},\n",
       "  {'_id': 'TUD_eef80801-b933-4cd2-ae0a-ca797aa74774'},\n",
       "  {'_id': 'TUD_66fefd5e-697a-468b-b58e-1ad3b4c25bf0'},\n",
       "  {'_id': 'TUD_5cc01f8f-52be-4811-b8ae-fec6e9f74f6b'},\n",
       "  {'_id': 'TUD_6a9e06e1-d7ba-411b-b8f9-7f373fcb9c50'},\n",
       "  {'_id': 'TUD_ed9b743c-a4ad-44be-9622-b51de5f39795'},\n",
       "  {'_id': 'TUD_bf16e165-9c76-4fb2-b1b0-719b73e60735'},\n",
       "  {'_id': 'TUD_b619e7d6-3df5-40ee-8108-ea7411ffeda7'},\n",
       "  {'_id': 'TUD_7000a1fb-695f-4bd9-980d-6481dadba321'},\n",
       "  {'_id': 'TUD_0cebb406-ffa4-4da7-9c66-0df77a48b1b3'},\n",
       "  {'_id': 'TUD_4a6bd52a-62fe-4522-8dfa-ea6f1a2195f5'},\n",
       "  {'_id': 'TUD_9800e588-cb61-4283-a41c-9d3d77ffc764'},\n",
       "  {'_id': 'TUD_adabe8f0-e85f-467b-8046-c44403e005e9'},\n",
       "  {'_id': 'TUD_d82de64b-b446-4df6-b335-36a3e058c8f8'},\n",
       "  {'_id': 'TUD_db8b269c-6f09-4a3d-9378-4e573900c7dd'},\n",
       "  {'_id': 'TUD_0c22a102-85cc-4d3c-9068-098a5eb5341d'},\n",
       "  {'_id': 'TUD_1ae97783-b8a5-41d8-a1d7-50c8817915e6'},\n",
       "  {'_id': 'TUD_5e1452a4-18da-49c1-b219-bb6cd5742c46'},\n",
       "  {'_id': 'TUD_9392f54c-14ff-4d6b-bf24-9692efde84a2'},\n",
       "  {'_id': 'TUD_1640ad86-fc07-4e54-9f05-1629296738c7'},\n",
       "  {'_id': 'TUD_e3021462-e39b-48b2-893b-59fa8ba75fb7'},\n",
       "  {'_id': 'TUD_dcb25701-2f87-4264-b50e-95f6ee5de5d7'},\n",
       "  {'_id': 'TUD_ff6b1363-019d-4891-b138-c90105e49cdf'},\n",
       "  {'_id': 'TUD_4f4d4450-ac86-4adf-82e4-7ccba84faa0c'},\n",
       "  {'_id': 'TUD_bb27eb90-fa46-4db2-a28b-7c719f0d6b7b'},\n",
       "  {'_id': 'TUD_04fc8fe3-d338-42b4-8bd3-6c76ea04b844'},\n",
       "  {'_id': 'TUD_18898596-1c6a-4c00-8d7e-40d9d80ce0ec'},\n",
       "  {'_id': 'TUD_ade5368d-2dc0-4bd9-8c36-48171483c4ab'},\n",
       "  {'_id': 'TUD_085fd330-b3d0-496f-a6cc-e8f025f73dc0'},\n",
       "  {'_id': 'TUD_02afa9dc-d585-4e17-a102-dc06d770a232'},\n",
       "  {'_id': 'TUD_16cec059-45b2-4946-8291-b92b72db8913'},\n",
       "  {'_id': 'TUD_015747c3-6c47-42e5-9ece-2ebdaaa2ba29'},\n",
       "  {'_id': 'TUD_dfb98715-941d-4a62-8743-b0acb0d701b7'},\n",
       "  {'_id': 'TUD_68fe8d4d-6a3d-41d9-884a-608314fc75d5'},\n",
       "  {'_id': 'TUD_3a01d973-d125-430f-82e2-fb83cc9239fb'},\n",
       "  {'_id': 'TUD_16ee713c-0651-425f-accb-b8398f09e751'},\n",
       "  {'_id': 'TUD_861b2cc7-8ca3-4bed-8d51-f94a359824fc'},\n",
       "  {'_id': 'TUD_893d1673-279e-4aa0-ab1c-efee64d4d03e'},\n",
       "  {'_id': 'TUD_5ee7e852-fce5-45af-8737-cf950cebd8a5'},\n",
       "  {'_id': 'TUD_2e8cf03e-2353-4770-be9b-686f3cf12229'},\n",
       "  {'_id': 'TUD_9e21bbb7-d034-4f10-8a28-3674187a0e59'},\n",
       "  {'_id': 'TUD_8174a80b-b048-47df-b4a6-9e48d83bae96'},\n",
       "  {'_id': 'TUD_e5c541f3-d5d6-4bc7-ab4b-b7b10c5c775c'},\n",
       "  {'_id': 'TUD_9b542aca-20fd-4bcf-a8f4-c0231e625404'},\n",
       "  {'_id': 'TUD_340a1ab8-c9b6-443f-9f7c-122a3b61f4e6'},\n",
       "  {'_id': 'TUD_9aa9ebe3-d2fa-4cf3-b22e-45326d700053'},\n",
       "  {'_id': 'TUD_202f2531-34f4-4b44-bbcf-a7c61aad2a2e'},\n",
       "  {'_id': 'TUD_22ef6407-682a-46ce-9393-ddf71286b448'},\n",
       "  {'_id': 'TUD_a19a6464-c9d2-47ca-a5aa-9bdcc8f51b6f'},\n",
       "  {'_id': 'TUD_7f3eeb52-77bd-4fdb-84a9-ea9ca0a35b94'},\n",
       "  {'_id': 'TUD_904db2dd-b63a-4d17-8742-1ebbf1627e01'},\n",
       "  {'_id': 'TUD_ec115fda-092c-469f-b8b2-450a72b3e489'},\n",
       "  {'_id': 'TUD_6da2ee07-99df-450d-93bd-2367725f4f70'},\n",
       "  {'_id': 'TUD_b859167f-6c9e-4ff0-acb8-044d55d6e69a'},\n",
       "  {'_id': 'TUD_9e2b32fb-278f-4f6f-9a32-91f6f49bc656'},\n",
       "  {'_id': 'TUD_dc8ce7c6-d13a-4ee3-9332-5294eb58d055'},\n",
       "  {'_id': 'TUD_76e3a56a-a8b2-478c-ae04-dba5c6a13a2e'},\n",
       "  {'_id': 'TUD_9e062646-1f23-48cf-bfc0-90eb58d7ab4a'},\n",
       "  {'_id': 'TUD_51828825-d26d-47e6-b760-6f14f2ba2f7b'},\n",
       "  {'_id': 'TUD_e6c72c1e-4c88-4f6a-b966-e5072bb928d2'},\n",
       "  {'_id': 'TUD_d4d2230a-a034-43fa-bd29-0eb6b954410b'},\n",
       "  {'_id': 'TUD_5b3a8ae9-a605-4933-adcb-b8f736425423'},\n",
       "  {'_id': 'TUD_d4655d6f-ba1b-4306-a0f1-1b495189ca9a'},\n",
       "  {'_id': 'TUD_3fac5a1c-c8ea-4fee-a856-9ed2afd9e96f'},\n",
       "  {'_id': 'TUD_5e399c00-8724-4ebf-8de3-d54efe407354'},\n",
       "  {'_id': 'TUD_878a94cc-5839-42f4-9ccf-ff48742526ef'},\n",
       "  {'_id': 'TUD_f503211b-3444-49aa-828a-ddcf0a7ea389'},\n",
       "  {'_id': 'TUD_2db8fe18-436c-4c76-9947-014e33ddffcf'},\n",
       "  {'_id': 'TUD_e7241d8a-418d-4aa7-806a-cce2055f1488'},\n",
       "  {'_id': 'TUD_cee5e97b-d023-4e27-8cb6-75522528e62d'},\n",
       "  {'_id': 'TUD_7f7b27b2-a2f9-4366-9bda-8e722d12b188'},\n",
       "  {'_id': 'TUD_68f91b3f-bc5d-4b1c-a927-d14d262bb4b4'},\n",
       "  {'_id': 'TUD_b13cd88b-099e-489c-a7e4-ae5ec0313cea'},\n",
       "  {'_id': 'TUD_5251d378-675c-4540-9bbd-a9e5bade7cc6'},\n",
       "  {'_id': 'TUD_00e26d49-a0c6-43ec-9820-bc6927948f80'},\n",
       "  {'_id': 'TUD_d4603eaa-055c-433c-99d2-3ce7b2704aa8'},\n",
       "  {'_id': 'TUD_96bde5c2-3db4-42c3-aa72-829b7eb54739'},\n",
       "  {'_id': 'TUD_8c63437d-ada3-4d67-8a51-b17fc4608efb'},\n",
       "  {'_id': 'TUD_a332673e-0fab-499e-ae08-cf6ff5ce8197'},\n",
       "  {'_id': 'TUD_de5b12d4-d625-4d4d-8f22-af155fd25959'},\n",
       "  {'_id': 'TUD_83bc5ff4-8e33-4962-9014-68cfbad8226f'},\n",
       "  {'_id': 'TUD_0022768b-2d96-46bf-84a3-b92067f762b9'},\n",
       "  {'_id': 'TUD_d059fea6-2861-49b4-ae36-5d31db109231'},\n",
       "  {'_id': 'TUD_2aa6252e-1175-42ec-ab5a-80731af65520'},\n",
       "  {'_id': 'TUD_73f81186-198d-4de9-a19a-52423b694d43'},\n",
       "  {'_id': 'TUD_3a53a0ff-4279-4492-8eee-5a47da9aad40'},\n",
       "  {'_id': 'TUD_a8516f8b-3aaa-4978-a079-85b9c2a80086'},\n",
       "  {'_id': 'TUD_ec871698-8a96-4836-ad74-da71c9231d29'},\n",
       "  {'_id': 'TUD_67f21f09-c6a0-44b9-b66a-479ba25bee66'},\n",
       "  {'_id': 'TUD_03860c22-0d7a-4812-b173-f285c1e87f85'},\n",
       "  {'_id': 'TUD_f3abc79e-6b62-4faf-a977-3845bb3ebc5c'},\n",
       "  {'_id': 'TUD_d8a6b83e-61ad-41ed-94c3-f090ac256ba3'},\n",
       "  {'_id': 'TUD_caa7e54d-1f8a-40d0-9bb1-78b202581d8f'},\n",
       "  {'_id': 'TUD_6d524027-0abc-46ec-a710-dc958062a38d'},\n",
       "  {'_id': 'TUD_33cbf224-4a0d-410e-85ad-cf8d05409f52'},\n",
       "  {'_id': 'TUD_6234f613-497d-4a2b-95bf-4bc52b1e3ed1'},\n",
       "  {'_id': 'TUD_12cb7563-45db-4824-bcf7-8f5af11ff52f'},\n",
       "  {'_id': 'TUD_3b7e2cc6-4810-44a9-a19a-3cc8a260d495'},\n",
       "  {'_id': 'TUD_700ffbee-3511-465d-ad6d-4eeee8710dbd'},\n",
       "  {'_id': 'TUD_c4edb267-f2c2-4085-8477-3a0d065f6278'},\n",
       "  {'_id': 'TUD_2b68174d-683c-4342-8e4a-4f705ab5871b'},\n",
       "  {'_id': 'TUD_727ec48a-1598-4cd5-91dc-0c86814abc71'},\n",
       "  {'_id': 'TUD_c3a4b57e-8679-4393-a78e-7a8658f62090'},\n",
       "  {'_id': 'TUD_4f879ecf-95d5-4482-8931-8c40abde0e79'},\n",
       "  {'_id': 'TUD_0475ae7f-cbf2-4722-9716-7481068de192'},\n",
       "  {'_id': 'TUD_59b71cd4-a7a0-430e-bba8-246745865607'},\n",
       "  {'_id': 'TUD_bf960362-6fa2-4879-b036-a2d662f3297b'},\n",
       "  {'_id': 'TUD_e17c7526-bb84-47f9-ae0a-36cacc343e1b'},\n",
       "  {'_id': 'TUD_5e3dfbfe-19a8-4ac4-b9b6-117aab0737bf'},\n",
       "  {'_id': 'TUD_3c99ceca-d210-47d7-8096-7a5d1138462f'},\n",
       "  {'_id': 'TUD_43dae97c-431c-4ed3-8064-35686940a32a'},\n",
       "  {'_id': 'TUD_5182f8c5-c9c6-4e6f-a0bd-7fbbe43cd977'},\n",
       "  {'_id': 'TUD_a1490c8b-35a3-4f41-a433-0e0d0899c833'},\n",
       "  {'_id': 'TUD_d901168b-b72e-4777-ab29-2533132b86c9'},\n",
       "  {'_id': 'TUD_fbe7382b-01c0-4e17-9982-755bac76064c'},\n",
       "  {'_id': 'TUD_4e8dd607-0447-4e33-90b0-0fa60f9db44b'},\n",
       "  {'_id': 'TUD_b9ad8db5-a436-4254-a80b-c8ebbc147a48'},\n",
       "  {'_id': 'TUD_23a4eb7f-1df4-4e9e-8a95-dee92f620086'},\n",
       "  {'_id': 'TUD_71a090f4-5c57-4a1a-a90e-aeed525f9fa2'},\n",
       "  {'_id': 'TUD_585a8d14-1ff6-4e45-ace4-fbc3d7b23002'},\n",
       "  {'_id': 'TUD_f66aa5a4-9f51-4846-9d9c-d064e942979e'},\n",
       "  {'_id': 'TUD_75480279-c222-4f13-addc-4afd52e38c24'},\n",
       "  {'_id': 'TUD_1454bdfe-ff41-4fca-9751-a4fef67233d5'},\n",
       "  {'_id': 'TUD_889a8830-9e61-4d21-bdef-624f7aec18e3'},\n",
       "  {'_id': 'TUD_976d6f08-9fe6-4eae-a636-fd0b66d5f645'},\n",
       "  {'_id': 'TUD_24352ff9-c11a-46c9-87d4-4d9d8968ed81'},\n",
       "  {'_id': 'TUD_54536c30-032d-4dcf-b909-2fc2dba88b4c'},\n",
       "  {'_id': 'TUD_6fe3398b-a45e-431c-876f-7e778885c454'},\n",
       "  {'_id': 'TUD_744a8044-94ea-48b7-b6db-e17f0d3a417d'},\n",
       "  {'_id': 'TUD_6b44e9c9-bb91-4e3c-94b8-d4ba8cd9408e'},\n",
       "  {'_id': 'TUD_3c23309a-5004-41e2-a2a9-fb1ab1a840ae'},\n",
       "  {'_id': 'TUD_fa826ec7-f53f-409d-b4bb-d037e674be3e'},\n",
       "  {'_id': 'TUD_c5eb65d2-cce5-44db-9e16-e790bd9e1945'},\n",
       "  {'_id': 'TUD_3c130adc-9e4a-4d54-93a0-74330b17599d'},\n",
       "  {'_id': 'TUD_fb2ad023-dff7-467d-b843-72a0f90a823a'},\n",
       "  {'_id': 'TUD_2dff4595-5fc9-4eb4-969b-b464ac59918e'},\n",
       "  {'_id': 'TUD_5e1071d4-9462-40f4-8673-ae24cb9ee475'},\n",
       "  {'_id': 'TUD_c6eeb03d-2681-46b6-a12a-58433eddd3a2'},\n",
       "  {'_id': 'TUD_b89f0d6f-2c73-4c6c-9c8f-ae759e4cad3f'},\n",
       "  {'_id': 'TUD_5f2a32a7-2851-4b7e-8c2d-d5fdfb1f8032'},\n",
       "  {'_id': 'TUD_e606b0a9-f273-425f-86ac-11840676e03d'},\n",
       "  {'_id': 'TUD_809eaec7-883c-47b0-9d57-8e605eaaaed1'},\n",
       "  {'_id': 'TUD_827c7123-231b-440e-95bd-a9c315efe27e'},\n",
       "  {'_id': 'TUD_850eb79f-b8de-47b5-832f-7c2c138787be'},\n",
       "  {'_id': 'TUD_3c7f869c-44b9-41f7-8deb-36ba284606cd'},\n",
       "  {'_id': 'TUD_ef8b0473-d699-4844-9521-855f3ff8fb71'},\n",
       "  {'_id': 'TUD_d4fac3ee-7e83-4c1e-93da-2875b2c20121'},\n",
       "  {'_id': 'TUD_b0daae03-f2ac-4780-8149-fc3509b814e5'},\n",
       "  {'_id': 'TUD_c9174083-4584-4074-be3b-2378bf8444e5'},\n",
       "  {'_id': 'TUD_88add684-df53-4f64-968c-2b44eba315d1'},\n",
       "  {'_id': 'TUD_4216cabc-ecd1-4987-bf1f-4c8cf3e96447'},\n",
       "  {'_id': 'TUD_fd6789f2-16b5-4dc9-8a12-536a0a04d3b2'},\n",
       "  {'_id': 'TUD_f3c0ade4-80bf-4f86-a07a-4f86a29f653d'},\n",
       "  {'_id': 'TUD_b933c6e5-7541-44f9-8262-ce949df9fbfd'},\n",
       "  {'_id': 'TUD_f669d142-ce4f-4faf-b028-c6c5775568b2'},\n",
       "  {'_id': 'TUD_ceb1e69d-dc14-44c5-8941-7ddc1d0365ba'},\n",
       "  {'_id': 'TUD_d9b49464-25f2-4354-9069-4e2534ed4152'},\n",
       "  {'_id': 'TUD_9040d3df-ae22-4537-82b2-4d084dd5b01c'},\n",
       "  {'_id': 'TUD_584cc65f-72ab-4fb8-bf11-3659867f3eec'},\n",
       "  {'_id': 'TUD_7fe80a02-fcf6-4e88-ad99-4dd3cf7ada2d'},\n",
       "  {'_id': 'TUD_d62f16d2-2139-4c94-bbb8-915f4c11537f'},\n",
       "  {'_id': 'TUD_aeb2c298-5a43-4d4a-9076-91e2f56d3e7f'},\n",
       "  {'_id': 'TUD_3200e567-a0cc-4021-bf9c-44d2c5586657'},\n",
       "  {'_id': 'TUD_6afeebc9-b574-453d-ac21-5682f57686bc'},\n",
       "  {'_id': 'TUD_d6e74f6e-ed5c-4641-ae0e-c50e0676e63a'},\n",
       "  {'_id': 'TUD_4e193978-6886-461a-8225-4d338c354b98'},\n",
       "  {'_id': 'TUD_58684b58-f0a6-4044-a70e-268d842ad7ec'},\n",
       "  {'_id': 'TUD_6dbf0e52-bf8a-4911-bf4b-1e7f361e037e'},\n",
       "  {'_id': 'TUD_bd83bae1-e1b6-4e02-bc75-72827e4ec798'},\n",
       "  {'_id': 'TUD_c7106f0b-5b8f-44fe-bca7-421653ec76c5'},\n",
       "  {'_id': 'TUD_21123648-f4ba-43e2-bbe1-04d1890f2d5e'},\n",
       "  {'_id': 'TUD_566b7b54-b40b-45fa-85c2-debaaa8098d4'},\n",
       "  {'_id': 'TUD_4439ccef-1079-43ec-96b5-d143eda4e23d'},\n",
       "  {'_id': 'TUD_c107f01a-3fb7-4dad-99b9-240cbdae2b90'},\n",
       "  {'_id': 'TUD_95a72bc6-2246-46ec-aa3d-9c222b11ceb0'},\n",
       "  {'_id': 'TUD_b242650e-9420-4cc2-b851-9bac0af62b0d'},\n",
       "  {'_id': 'TUD_eab4f47a-17c6-40c7-b55a-da09a5813cf8'},\n",
       "  {'_id': 'TUD_78eed80f-2d9e-4c9a-b806-40b8adb9ccda'},\n",
       "  {'_id': 'TUD_a5157424-63cf-4770-9e07-20c35971a84c'},\n",
       "  {'_id': 'TUD_7e5ccc35-774e-41d6-b12a-a502b1960256'},\n",
       "  {'_id': 'TUD_7bcddd81-b8de-4bcc-a069-7080bef1bdf6'},\n",
       "  {'_id': 'TUD_8fb5b6f6-93a6-4aed-a446-013125541ff9'},\n",
       "  {'_id': 'TUD_136cd4d6-396a-4520-98bf-d9dc9782f807'},\n",
       "  {'_id': 'TUD_bd7ab4e8-602d-4b24-a9ce-63313e47e6b0'},\n",
       "  {'_id': 'TUD_be7f6eee-e1e7-40d4-b2f8-5922139153b2'},\n",
       "  {'_id': 'TUD_20170a8c-7e1e-434f-b5d6-9ba933e2ab6e'},\n",
       "  {'_id': 'TUD_a3bea0c0-fc6d-47a1-9b94-bf39df4aff8f'},\n",
       "  {'_id': 'TUD_4f9e5af1-bbda-4685-9045-2014171fb928'},\n",
       "  {'_id': 'TUD_fdaf7d6a-1d45-41a5-8157-15586fa3480f'},\n",
       "  {'_id': 'TUD_93fba141-d466-468d-96d5-6991e5b658cf'},\n",
       "  {'_id': 'TUD_ae14fcf5-707e-4666-8b34-f3a91fd25f78'},\n",
       "  {'_id': 'TUD_db5f3d18-ce49-496a-8328-ec9203136949'},\n",
       "  {'_id': 'TUD_43902b9c-ed6b-4743-b00b-d8f803b111e0'},\n",
       "  {'_id': 'TUD_3a4b2767-6172-4eeb-a6b4-1ddae7b531c5'},\n",
       "  {'_id': 'TUD_01bfcad0-9a56-454b-a30e-dc2fd3394f12'},\n",
       "  {'_id': 'TUD_fd98efb8-8225-4eea-bf6c-5f56a2d022f5'},\n",
       "  {'_id': 'TUD_8b5b5120-c99b-4183-bec1-918e44860bfa'},\n",
       "  {'_id': 'TUD_d2bfe438-36f4-4fb6-8b51-0480f17449c2'},\n",
       "  {'_id': 'TUD_62da8ff0-fbfc-483a-8177-0f7b50b16c89'},\n",
       "  {'_id': 'TUD_5c84b35a-39d0-4c19-a905-a7bf60239138'},\n",
       "  {'_id': 'TUD_73541a63-93e9-44ff-a913-2fd16239bddc'},\n",
       "  {'_id': 'TUD_769ccdf5-5f90-4f66-a1ba-fa3a09c38ccf'},\n",
       "  {'_id': 'TUD_6e23606d-c5a6-4635-8f8d-ea3b2b630763'},\n",
       "  {'_id': 'TUD_61f7f022-60a4-4935-af13-6b5438d89c04'},\n",
       "  {'_id': 'TUD_0925832a-1afd-4ee3-9a3c-1c35fcad2ad5'},\n",
       "  {'_id': 'TUD_313c929c-9543-4037-9fa7-06645e654cf3'},\n",
       "  {'_id': 'TUD_f6903dcf-87e4-45a1-8a17-60699a478ab0'},\n",
       "  {'_id': 'TUD_3387006b-dd3c-430e-979e-f8a87e57ca11'},\n",
       "  {'_id': 'TUD_b8d4403e-574a-4e71-ab0d-93fd24717a66'},\n",
       "  {'_id': 'TUD_3e6fe824-5d96-4f1b-890d-8f9218cf41ea'},\n",
       "  {'_id': 'TUD_c372f6e8-92f2-4793-9b14-e4b445b1a6c8'},\n",
       "  {'_id': 'TUD_dc6b61d6-40fc-42d1-8205-bdfc5e1072f2'},\n",
       "  {'_id': 'TUD_467fe2ad-db79-4d4a-9a4b-83e7c49d9106'},\n",
       "  {'_id': 'TUD_1cdeaa76-3b5a-4231-9ff5-8ba7b283ba35'},\n",
       "  {'_id': 'TUD_3d088772-b776-49b8-ac7e-fc196783180e'},\n",
       "  {'_id': 'TUD_328930cc-d48e-44ae-9cfb-03aa3d0dc429'},\n",
       "  {'_id': 'TUD_0bd2fbdf-bdde-4c6f-8a96-c42077bb2d49'},\n",
       "  {'_id': 'TUD_a03b025b-1fd5-4cd1-9423-a5c1bd85fefe'},\n",
       "  {'_id': 'TUD_f10f3598-c47b-4f61-8827-34febd6afb45'},\n",
       "  {'_id': 'TUD_c45e3307-2e1c-442d-8451-8511927707de'},\n",
       "  {'_id': 'TUD_b6ae4f9e-89c5-4d55-87d8-aeba045b8cb3'},\n",
       "  {'_id': 'TUD_61b8d02d-2413-4565-bcba-d7822b0a9476'},\n",
       "  {'_id': 'TUD_4b956b17-a5da-4b87-99a9-aeefb2c938fe'},\n",
       "  {'_id': 'TUD_6ff865cc-1f70-46ad-abd6-08a9b6dccd8b'},\n",
       "  {'_id': 'TUD_7a248cd8-7749-4bd5-a938-ed56a7bd9992'},\n",
       "  {'_id': 'TUD_9e92acb7-8191-4714-90cc-6c3f2ad78df3'},\n",
       "  {'_id': 'TUD_36954cd6-385b-45ce-8d49-26463cfa5766'},\n",
       "  {'_id': 'TUD_c71c85bc-d742-449b-88e7-33e172392ec2'},\n",
       "  {'_id': 'TUD_de3e099b-e6d0-4a8b-aaab-aefbc1ffd707'},\n",
       "  {'_id': 'TUD_06d80709-ad0b-4f9b-a1cc-65634cc42d00'},\n",
       "  {'_id': 'TUD_16c43d66-80e3-47ce-83dc-3b8a5877368e'},\n",
       "  {'_id': 'TUD_2a145431-37c4-413c-9a02-c514a69695e6'},\n",
       "  {'_id': 'TUD_1b30ea32-f6a1-45f4-80e2-b4b6f29f6666'},\n",
       "  {'_id': 'TUD_650ec0d0-4613-4dae-96b1-1f685dff0e60'},\n",
       "  {'_id': 'TUD_c0a208a8-d012-450e-9c07-f23a39910855'},\n",
       "  {'_id': 'TUD_d063b265-3c59-48ac-bb4e-53dab8866cc3'},\n",
       "  {'_id': 'TUD_d564cdae-0a85-42aa-9251-addeb1173620'},\n",
       "  {'_id': 'TUD_f4ac1938-d7d8-4fa4-84ef-31e0404186f7'},\n",
       "  {'_id': 'TUD_6d58c3c1-51c1-4282-a9f8-888d50040755'},\n",
       "  {'_id': 'TUD_6bd8fee2-806b-4a95-ac66-056348f8c36c'},\n",
       "  {'_id': 'TUD_9b192f84-f3f7-483e-9b67-99b1480e29c7'},\n",
       "  {'_id': 'TUD_64ce9217-e5e5-495f-ae0a-858cc1ef5886'},\n",
       "  {'_id': 'TUD_bf3325ce-f246-4977-91bc-785f877347b8'},\n",
       "  {'_id': 'TUD_103551c5-8e23-4dd0-8b25-6e54f77b2dba'},\n",
       "  {'_id': 'TUD_6741902f-fc75-4bc2-91d5-5d204218c6d6'},\n",
       "  {'_id': 'TUD_9b2db13b-5948-4928-b3a4-c6df1ae1e9f1'},\n",
       "  {'_id': 'TUD_6be235a6-5a78-4481-971d-0d4b39073361'},\n",
       "  {'_id': 'TUD_d0b5c0b9-356f-4852-ba98-f3703831daf6'},\n",
       "  {'_id': 'TUD_ea39f04d-eec3-4957-a009-ab9a0a00a935'},\n",
       "  {'_id': 'TUD_7624a97f-cf52-4925-a6eb-6dbe112de305'},\n",
       "  {'_id': 'TUD_25d80828-efc2-4255-8719-1fd297380109'},\n",
       "  {'_id': 'TUD_a4371a0a-6176-4da6-819c-2be96222a5f1'},\n",
       "  {'_id': 'TUD_968f224e-3a30-4c6b-8dc6-9a4e1e43b2ec'},\n",
       "  {'_id': 'TUD_2e19140c-9a84-451b-a641-46c4c40ad9ac'},\n",
       "  {'_id': 'TUD_1c2f845a-1afe-4488-9446-97e9b4252b60'},\n",
       "  {'_id': 'TUD_58ad76d8-4552-461e-aa61-54299d021bd1'},\n",
       "  {'_id': 'TUD_2f16fbcc-1a61-4144-bcd2-72effdfc2b9d'},\n",
       "  {'_id': 'TUD_a93e7843-552e-432c-8133-f17b5ef6de89'},\n",
       "  {'_id': 'TUD_2b95ac2a-09d7-4f14-ab08-864d33e61fea'},\n",
       "  {'_id': 'TUD_01f0fe35-f0a8-4fec-85d1-b6c3fe2f099a'},\n",
       "  {'_id': 'TUD_3e7839c5-d2f4-4f75-8495-ad2d818524a4'},\n",
       "  {'_id': 'TUD_ceeb7e26-db2c-4108-ae3b-38628e8806f1'},\n",
       "  {'_id': 'TUD_8907468c-b102-4a35-aa84-d49bb2110541'},\n",
       "  {'_id': 'TUD_7d41c07a-3927-4144-91fa-b7c4308ef245'},\n",
       "  {'_id': 'TUD_90323d56-d208-401e-8d3c-44bec4fca4f4'},\n",
       "  {'_id': 'TUD_09903120-2d84-45af-aa61-4ec3c1d1c5a3'},\n",
       "  {'_id': 'TUD_24fda8d4-05e0-4957-9c44-869aa8a32439'},\n",
       "  {'_id': 'TUD_ba6c8459-2fdb-4ff0-b89b-0100fe96ed08'},\n",
       "  {'_id': 'TUD_7d02ec4a-0d99-453a-8950-f54287d91e2a'},\n",
       "  {'_id': 'TUD_bc6a6d49-8ea4-4aca-ace6-0cd6934070da'},\n",
       "  {'_id': 'TUD_d9d802af-9ebf-4524-9dbc-e7657d67921e'},\n",
       "  {'_id': 'TUD_e83b184c-c972-402a-a0c6-418222cf11ad'},\n",
       "  {'_id': 'TUD_38e5d9ac-8c04-48d5-801f-b0c9308f67fa'},\n",
       "  {'_id': 'TUD_5141bbee-4b12-4979-b0ee-9b800e33662b'},\n",
       "  {'_id': 'TUD_841fa20a-b1a9-4eb6-afd1-070ea938ae4b'},\n",
       "  {'_id': 'TUD_bdd85ee5-a7eb-49f7-84c2-fcf0dbaa907c'},\n",
       "  {'_id': 'TUD_906008f1-0805-432f-9eb2-73492ae26c71'},\n",
       "  {'_id': 'TUD_9c3829aa-294b-4bed-9b12-70f8f13b01dd'},\n",
       "  {'_id': 'TUD_9e8e9c3a-edc3-49a0-ae6b-980bb2eea5e5'},\n",
       "  {'_id': 'TUD_edf396c5-5c3a-4b5c-9fc4-b8bb5ff6eeee'},\n",
       "  {'_id': 'TUD_23c1ac2a-0c6c-4ade-9409-1df69fc5b38f'},\n",
       "  {'_id': 'TUD_22ade306-1d31-4e4a-b228-0a3895a5c0d5'},\n",
       "  {'_id': 'TUD_269dbdaa-e4a0-425b-847b-ffe9e1d9157b'},\n",
       "  {'_id': 'TUD_ed394af6-b781-4637-ada6-0dfd8b997841'},\n",
       "  {'_id': 'TUD_cf199182-30c7-4b1e-92bb-836fbbc98d23'},\n",
       "  {'_id': 'TUD_0b930688-5c94-443e-bbdc-fc7ddfe3cfd9'},\n",
       "  {'_id': 'TUD_83bfe59a-5cf3-4127-9f77-9d48abd97da1'},\n",
       "  {'_id': 'TUD_5ccea299-4d16-458a-b7b3-f34d2fbe19e2'},\n",
       "  {'_id': 'TUD_c7ceb365-3f9f-4c31-968f-f5b829a17e1f'},\n",
       "  {'_id': 'TUD_a6809d09-31bb-4854-b513-693d3ae0bcea'},\n",
       "  {'_id': 'TUD_81b3059f-42b9-47b6-b60e-c610fee0a07d'},\n",
       "  {'_id': 'TUD_39069582-d34b-4586-bb46-92ea49b07b57'},\n",
       "  {'_id': 'TUD_2f72bf7f-e04a-4eb1-8bac-b050175b0afe'},\n",
       "  {'_id': 'TUD_317d30a7-cdfa-4130-a82f-0cf9fdded486'},\n",
       "  {'_id': 'TUD_3d834130-8dd7-420a-9af9-6e77761cdad6'},\n",
       "  {'_id': 'TUD_40a61f6a-8f67-4089-a73d-ceb0290cdd69'},\n",
       "  {'_id': 'TUD_10370845-77ec-4169-be70-4da58be43742'},\n",
       "  {'_id': 'TUD_ff7acb60-a3e9-4f72-9c8d-bc65398d8d6a'},\n",
       "  {'_id': 'TUD_19037c4d-d787-4cab-a182-6fcc33badf70'},\n",
       "  {'_id': 'TUD_e38b3a2e-aba2-4193-abe1-b2cb71e6a376'},\n",
       "  {'_id': 'TUD_e7dbaaaa-2f7a-432e-a0e3-51b4f67b3c58'},\n",
       "  {'_id': 'TUD_51063af3-b8b8-4f3c-9745-dd422a3a2d35'},\n",
       "  {'_id': 'TUD_d9b5456d-722a-401d-9f1a-c530c46d6491'},\n",
       "  {'_id': 'TUD_e327885c-35de-4ced-a0a9-b0ca8710a0a6'},\n",
       "  {'_id': 'TUD_33e151a3-f059-4a79-bd85-81415e81528b'},\n",
       "  {'_id': 'TUD_f943dcfd-b11c-4b2c-ab55-db8d04ee0b28'},\n",
       "  {'_id': 'TUD_c0387ec6-6e2e-4406-87b0-11076438f355'},\n",
       "  {'_id': 'TUD_abf4adc9-b198-4cb1-a338-64567fc62521'},\n",
       "  {'_id': 'TUD_fd4944a9-b82c-4df3-a8ef-c5a9f13f57c4'},\n",
       "  {'_id': 'TUD_b40d40dd-f155-4b60-a245-04bdd0760861'},\n",
       "  {'_id': 'TUD_9bb67144-311c-4a9b-92e6-0d8b0b899d27'},\n",
       "  {'_id': 'TUD_dfa0ef97-a72a-47f5-88ff-c2d10872faec'},\n",
       "  {'_id': 'TUD_d23770ce-51ad-43d3-960b-3fa2ad7623f1'},\n",
       "  {'_id': 'TUD_e664e575-4455-4bd8-907f-f28c667d0e51'},\n",
       "  {'_id': 'TUD_512f9d26-6a8d-48d6-872b-a62f9cac33af'},\n",
       "  {'_id': 'TUD_82e85895-ba53-4910-b6df-bfed9a735d3f'},\n",
       "  {'_id': 'TUD_c4722b45-252d-4385-9ece-378348d1df35'},\n",
       "  {'_id': 'TUD_b0d08f56-c203-42c7-9c7f-e332c1ea5412'},\n",
       "  {'_id': 'TUD_8a144747-576f-431a-b091-79cebf20ba8b'},\n",
       "  {'_id': 'TUD_a170e23f-9fee-45fd-b99b-3a85e4d551cf'},\n",
       "  {'_id': 'TUD_276fcdbf-9c71-4175-879c-176102028e08'},\n",
       "  {'_id': 'TUD_bcba7e5b-e898-4e59-b636-234ad3fdc432'},\n",
       "  {'_id': 'TUD_66c177e8-9da2-407a-83b4-fa80789a676e'},\n",
       "  {'_id': 'TUD_9070de1a-6b2a-432a-9eea-c60ca9633391'},\n",
       "  {'_id': 'TUD_080a2485-1177-4275-b717-a79c1bc9274f'},\n",
       "  {'_id': 'TUD_7179edff-b498-4c3b-900d-62d61f8665df'},\n",
       "  {'_id': 'TUD_0f890657-5cf1-4d0d-88f6-b183ea49425d'},\n",
       "  {'_id': 'TUD_91e1b370-4b5b-4998-88bb-1d9d8a3d8de4'},\n",
       "  {'_id': 'TUD_3134a851-2b40-4af1-896a-41a6ce7a5816'},\n",
       "  {'_id': 'TUD_a54a370f-3a45-461e-83a0-d4a008114d89'},\n",
       "  {'_id': 'TUD_8566e258-c89a-4c97-a35f-46189e7fcb21'},\n",
       "  {'_id': 'TUD_f5c7d8ca-febb-4531-bafe-de416c24ac7d'},\n",
       "  {'_id': 'TUD_bfa8a148-0d3a-49bc-a0db-aba16211a90d'},\n",
       "  {'_id': 'TUD_b814e309-dc00-4c33-999f-516a54827b8c'},\n",
       "  {'_id': 'TUD_755b1378-70c9-446c-a2c0-7eb871959ec0'},\n",
       "  {'_id': 'TUD_69cd3749-71cf-45d0-bbf8-e825cb6c5704'},\n",
       "  {'_id': 'TUD_944a11fd-eea8-445e-9a46-98855f7766d3'},\n",
       "  {'_id': 'TUD_883796f3-49fc-46de-92f8-cd2bbd592d77'},\n",
       "  {'_id': 'TUD_73a4025d-c519-4e3a-b1cd-c1c8aa0fdfeb'},\n",
       "  {'_id': 'TUD_3d0521b7-7fd4-40ab-8bc4-7109267b5ba4'},\n",
       "  {'_id': 'TUD_113b6229-c33a-4100-93be-22e1c8912672'},\n",
       "  {'_id': 'TUD_b884bf6f-f72a-4194-b43a-4f7258ca2ae2'},\n",
       "  {'_id': 'TUD_5b485a4a-e502-42d9-8bd2-21c02226ed91'},\n",
       "  {'_id': 'TUD_9e4a7ce2-4259-4816-930d-522008aff9b4'},\n",
       "  {'_id': 'TUD_3ebe06a4-77ce-4a07-9ce9-cbddbea972c6'},\n",
       "  {'_id': 'TUD_c100340b-b31e-4ffe-91e2-ea6e59e5ba0c'},\n",
       "  {'_id': 'TUD_4ab25966-8291-4ad2-bed2-2962262af902'},\n",
       "  {'_id': 'TUD_4456b202-2b1a-4c0e-acce-63d88500a110'},\n",
       "  {'_id': 'TUD_12455395-db49-49f7-9afb-433b06fbc37e'},\n",
       "  {'_id': 'TUD_03c7e637-7f9f-4577-9165-ec300452c6a1'},\n",
       "  {'_id': 'TUD_55762b74-697e-4685-b7cf-02cf7614fa3a'},\n",
       "  {'_id': 'TUD_47e228e5-645e-497c-a5e2-ec3b4df5e299'},\n",
       "  {'_id': 'TUD_58e639f7-ea6a-4e89-baf8-0bfa64c9a7f4'},\n",
       "  {'_id': 'TUD_5b2299cd-c895-41a6-81a6-e81e0d686971'},\n",
       "  {'_id': 'TUD_a0bc538a-c447-44c8-bf56-4a9090211b98'},\n",
       "  {'_id': 'TUD_3feed710-ffc3-4600-87f3-ec734484c88f'},\n",
       "  {'_id': 'TUD_7530a76d-e25b-486c-9f4b-fb74cf19c8c9'},\n",
       "  {'_id': 'TUD_3f42dbd7-a72b-4b55-b818-1cda9915cdc4'},\n",
       "  {'_id': 'TUD_4d22ccdd-f4e7-458e-b455-7405eaba6245'},\n",
       "  {'_id': 'TUD_82984407-b611-405c-82f4-da4208dcf93a'},\n",
       "  {'_id': 'TUD_0b513351-7ce5-438c-b266-56fd9838b543'},\n",
       "  {'_id': 'TUD_6f4aaf07-47b6-485c-b2b9-4a02df71a7a5'},\n",
       "  {'_id': 'TUD_461ec191-470b-448f-ae7d-e567773c4751'},\n",
       "  {'_id': 'TUD_f9b4ead2-0bd2-49f0-b022-a2c5fd9bb163'},\n",
       "  {'_id': 'TUD_d5ec57f7-86fa-4889-92cc-c86fc0bd2b25'},\n",
       "  {'_id': 'TUD_f6aa2f88-e657-4fef-b684-188a212c71ad'},\n",
       "  {'_id': 'TUD_9170e10a-d07f-47ba-8605-95342128e194'},\n",
       "  {'_id': 'TUD_04703e41-543e-456f-8068-4410d9860dd6'},\n",
       "  {'_id': 'TUD_e558e01e-443c-4cc7-9fed-9ccf0313e971'},\n",
       "  {'_id': 'TUD_870b2cd3-9524-4b57-bf28-a71e00c90e5f'},\n",
       "  {'_id': 'TUD_1391866c-18f9-48ba-8c69-87ebd58e8516'},\n",
       "  {'_id': 'TUD_cb9d7857-ed48-45bc-837b-b0ac0ca9a133'},\n",
       "  {'_id': 'TUD_2b257abd-a17a-43c0-938b-2b25fcead09a'},\n",
       "  {'_id': 'TUD_37b78c00-b987-415e-b07a-8171827e743f'},\n",
       "  {'_id': 'TUD_867c84ae-6cf5-45aa-ba5f-ab6ec4bbf5ed'},\n",
       "  {'_id': 'TUD_4681f604-ef07-4ac8-86bf-42257e6c96ed'},\n",
       "  {'_id': 'TUD_28ae3ef0-13ce-4133-a205-9d678074d3a0'},\n",
       "  {'_id': 'TUD_82a619ee-d099-43a7-9d81-d41cc8cb2869'},\n",
       "  {'_id': 'TUD_0b691916-b2d6-47a1-a637-d038319d7d7c'},\n",
       "  {'_id': 'TUD_3acabfb3-c896-4964-b0e1-6c9b50e79995'},\n",
       "  {'_id': 'TUD_46562c4a-8e99-422f-b2e3-6949358299f6'},\n",
       "  {'_id': 'TUD_cff5da7b-d449-4bf6-b2f9-6924b5c7bd6c'},\n",
       "  {'_id': 'TUD_407b1564-04d7-4f24-badf-61bb960ec9d2'},\n",
       "  {'_id': 'TUD_68a5cbff-447d-4568-8053-25b794ff72bd'},\n",
       "  {'_id': 'TUD_d19a1d28-e3eb-420a-a872-5a40baadcc09'},\n",
       "  {'_id': 'TUD_81815b28-ec74-4905-9ec7-ec47261740a1'},\n",
       "  {'_id': 'TUD_3074d624-1468-41e9-b737-a1d3e11db0d8'},\n",
       "  {'_id': 'TUD_2660f3fd-878b-47fe-8849-dc4faf696264'},\n",
       "  {'_id': 'TUD_dc3af055-f75f-4a1f-a4f5-062a273ba954'},\n",
       "  {'_id': 'TUD_25aea2bb-cd25-499f-9fe6-8de69462e49d'},\n",
       "  {'_id': 'TUD_28672b0c-9230-45dc-9b6f-402cb42972f4'},\n",
       "  {'_id': 'TUD_383ccfc6-c496-4f77-8eb5-b83e82d64471'},\n",
       "  {'_id': 'TUD_b683f5cb-d6ed-4ed2-b2e5-6612ea4016bd'},\n",
       "  {'_id': 'TUD_6e02be89-3d5a-4207-a449-ca14eff30231'},\n",
       "  {'_id': 'TUD_349398f7-a24a-4ade-b1b6-ccd1858269f3'},\n",
       "  {'_id': 'TUD_73fe7835-43ac-4d65-bbf1-9202c7d72c45'},\n",
       "  {'_id': 'TUD_56647f6c-78d8-4f63-a01c-50661f015e56'},\n",
       "  {'_id': 'TUD_ade2482a-15fc-42ba-bb1e-a5b6d9dfa842'},\n",
       "  {'_id': 'TUD_6b65c5f5-398c-4a41-8806-31c638b1891c'},\n",
       "  {'_id': 'TUD_ca9364a7-4d3f-43e2-92d9-df63c04f7d12'},\n",
       "  {'_id': 'TUD_c3b17264-a7ed-4f6d-aca7-88c34f2f6958'},\n",
       "  {'_id': 'TUD_0bdbe6b1-df28-4d52-948d-1c25a901de75'},\n",
       "  {'_id': 'TUD_d0715121-d6d3-4816-95b1-4254af5a75c1'},\n",
       "  {'_id': 'TUD_2c8b4b8d-e9ea-4d27-9055-23f6087da7e4'},\n",
       "  {'_id': 'TUD_c7f2664a-24cf-4d35-81b8-c6368e51428b'},\n",
       "  {'_id': 'TUD_441c5611-5072-454a-909d-fc20c94c5ef5'},\n",
       "  {'_id': 'TUD_d82980e2-b9e1-497c-a1f0-be1d379f081b'},\n",
       "  {'_id': 'TUD_9833d7d1-3b38-46a6-92d1-eaa8711ef0c0'},\n",
       "  {'_id': 'TUD_fc0cf997-4900-435c-b213-00e5828490de'},\n",
       "  {'_id': 'TUD_9ab06527-d848-4155-b7bd-63e07eb79002'},\n",
       "  {'_id': 'TUD_b3f1eeba-127c-406b-87ec-2aa4550dadc2'},\n",
       "  {'_id': 'TUD_21fb9d33-fec2-4324-b6f1-b0b7535060e4'},\n",
       "  {'_id': 'TUD_5588bf01-6e37-4689-967a-30dd2a5202f3'},\n",
       "  {'_id': 'TUD_2cf66a04-cb31-4f3a-9526-17e93fc7bd39'},\n",
       "  {'_id': 'TUD_4d1844c8-89df-4787-b777-c742b4a27217'},\n",
       "  {'_id': 'TUD_e007961e-feb5-41d8-8612-2280c6a70e43'},\n",
       "  {'_id': 'TUD_09179df1-7418-4b33-8a75-e0c557c6079c'},\n",
       "  {'_id': 'TUD_1baa5d42-e81d-42f3-89a0-63b73178c77c'},\n",
       "  {'_id': 'TUD_0573cbb7-5b61-46e9-97f7-af01fcdad1b7'},\n",
       "  {'_id': 'TUD_b5aef612-424f-41a8-8041-c6206b4c3b64'},\n",
       "  {'_id': 'TUD_d2ebe038-fc1e-47b4-a708-e043e9d3ca74'},\n",
       "  {'_id': 'TUD_320ae614-d8fd-4799-9684-bd4f8feb011d'},\n",
       "  {'_id': 'TUD_d87fbb80-54a9-463e-8379-acfe9ce769b9'},\n",
       "  {'_id': 'TUD_6e9c5f5e-da96-499b-b63b-bb53b1698190'},\n",
       "  {'_id': 'TUD_7415eaf7-5f63-4254-9899-1b7c0a6a8997'},\n",
       "  {'_id': 'TUD_16a1d98d-2a2e-44bc-9051-35c1942f9e9b'},\n",
       "  {'_id': 'TUD_29e86b3f-fee4-41e9-9b56-cda14ef928fb'},\n",
       "  {'_id': 'TUD_9bf41aed-ebd0-4038-8961-9fdbb699b1a7'},\n",
       "  {'_id': 'TUD_637fcd21-999f-4bfb-85e9-f82d87d91caa'},\n",
       "  {'_id': 'TUD_7c73ded2-1669-4f48-9865-5eddc3a20ab0'},\n",
       "  {'_id': 'TUD_de51c043-d424-4820-95c3-ee44b93799c1'},\n",
       "  {'_id': 'TUD_6e8f2999-71cf-420b-b5fc-235de3569285'},\n",
       "  {'_id': 'TUD_4c67ea0b-837c-4aed-994d-7f89c42f40e9'},\n",
       "  {'_id': 'TUD_7bf74f2c-0cd3-4eb8-b9eb-2f931cae3694'},\n",
       "  {'_id': 'TUD_65a7a027-1338-4e9e-bf55-1966c75d2521'},\n",
       "  {'_id': 'TUD_4cc2de9e-40f9-4ae9-8868-f940e2f3529f'},\n",
       "  {'_id': 'TUD_3a15293b-16f6-4e9d-b6a2-f02cd52f1a9e'},\n",
       "  {'_id': 'TUD_11f50230-8a9a-41f3-96fe-2cecaae39fbe'}]]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extracted_publications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Training NER (module 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from m1_preprocessing import seed_data_extraction, term_sentence_expansion, training_data_generation, ner_training\n",
    "from m1_postprocessing import extract_new_entities, filtering\n",
    "import config as cfg\n",
    "import gensim\n",
    "import elasticsearch\n",
    "import time\n",
    "import re\n",
    "doc2vec_model = gensim.models.Doc2Vec.load(cfg.ROOTPATH + '/embedding_models/doc2vec.model')\n",
    "es = elasticsearch.Elasticsearch([{'host': 'localhost', 'port': 9200}])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "seeds = ['hierarchical agglomerative', 'selection algorithm', 'stochastic gradient descent', 'pearson correlation', \n",
    "         'semantic relevance', 'gpbased', 'pattern matching', 'clir', 'random forest', 'random indexing', \n",
    "         'basic load control method', 'linear regression', 'recursive function', 'latent dirichlet allocation', \n",
    "         'convolutional dnn', 'likelihood function', 'folding-in', 'restricted boltzmann machine', 'lstm', \n",
    "         'radial basis function network', 'bmecat', 'lib', 'fast fourier', 'adaptive filter', 'spectral clustering', \n",
    "         'dmp method', 'reinforcement learning', 'graph-based propagation', 'semantictyper', 'hierarchical clustering',\n",
    "         'variational em', 'qald', 'fourier analysis', 'simple random algorithm', 'random search', 'lsh method', \n",
    "         'regular expression', 'rapid7', 'word embedding', 'autoencoder', 'bayesian nonparametric', \n",
    "         'variational bayesian inference', 'tsa algorithm', 'predictive modeling', 'query optimization', 'softmax', \n",
    "         'ridge regularization', 'tdcm', 'support vector machine', 'mcmc']\n",
    "\n",
    "# seeds = ['buzzfeed', 'pslnl', 'dailymed', 'robust04', 'scovo', 'ask.com', 'cacm', 'stanford large network dataset', 'mediaeval', 'lexvo', 'spambase', 'shop.com', 'orkut', 'jnlpba', 'cyworld', 'citebase', 'blog06', 'worldcat', 'booking.com', 'semeval', 'imagenet', 'nasdaq', 'brightkite', 'movierating', 'webkb', 'ionosphere', 'moviepilot', 'duc2001', 'datahub', 'cifar', 'tdt', 'refseq', 'stack overflow', 'wikiwars', 'blogpulse', 'ws-353', 'gerbil', 'wikia', 'reddit', 'ldoce', 'kitti dataset', 'specweb', 'fedweb', 'wt2g', 'as3ap', 'friendfeed', 'new york times', 'chemid', 'imageclef', 'newegg']\n",
    "\n",
    "\n",
    "context_words = ['method', 'algorithm', 'approach', 'evaluate']\n",
    "sentence_expansion = True\n",
    "training_cycles = 4 \n",
    "model_name = 'method_tud'\n",
    "filtering_pmi = True\n",
    "filtering_st = False\n",
    "filtering_ws = True\n",
    "filtering_kbl = True\n",
    "filtering_majority = True\n",
    "\n",
    "cycle = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Started initial training data extraction\n",
      "Extracting sentences for 50 seed terms\n",
      ".................................................."
     ]
    }
   ],
   "source": [
    "sentences, entities = seed_data_extraction.sentence_extraction(model_name, cycle, seeds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".hierarchical agglomerative 17\n",
      ".selection algorithm 524\n",
      ".stochastic gradient descent 29\n",
      ".pearson correlation 93\n",
      ".semantic relevance 38\n",
      ".gpbased 0\n",
      ".pattern matching 301\n",
      ".clir 0\n",
      ".random forest 98\n",
      ".random indexing 2\n",
      ".basic load control method 0\n",
      ".linear regression 331\n",
      ".recursive function 63\n",
      ".latent dirichlet allocation 8\n",
      ".convolutional dnn 2\n",
      ".likelihood function 204\n",
      ".folding-in 104\n",
      ".restricted boltzmann machine 0\n",
      ".lstm 5\n",
      ".radial basis function network 3\n",
      ".bmecat 0\n",
      ".lib 54\n",
      ".fast fourier 127\n",
      ".adaptive filter 78\n",
      ".spectral clustering 22\n",
      ".dmp method 0\n",
      ".reinforcement learning 14\n",
      ".graph-based propagation 4\n",
      ".semantictyper 0\n",
      ".hierarchical clustering 81\n",
      ".variational em 0\n",
      ".qald 0\n",
      ".fourier analysis 124\n",
      ".simple random algorithm 18\n",
      ".random search 160\n",
      ".lsh method 0\n",
      ".regular expression 176\n",
      ".rapid7 2\n",
      ".word embedding 28\n",
      ".autoencoder 0\n",
      ".bayesian nonparametric 6\n",
      ".variational bayesian inference 0\n",
      ".tsa algorithm 3\n",
      ".predictive modeling 14\n",
      ".query optimization 135\n",
      ".softmax 3\n",
      ".ridge regularization 20\n",
      ".tdcm 0\n",
      ".support vector machine 85\n",
      ".mcmc 30\n"
     ]
    }
   ],
   "source": [
    "for entity in seeds:\n",
    "        entity_name = re.sub(r'\\([^)]*\\)', '', entity)\n",
    "        print('.', end='')\n",
    "        query = {\"query\":\n",
    "                    {\"match\":\n",
    "                        {\"content.chapter.sentpositive\":\n",
    "                            {\"query\": entity_name,\n",
    "                             \"operator\": \"and\"\n",
    "                             }\n",
    "                         }\n",
    "                     }\n",
    "                 }\n",
    "\n",
    "        res = es.search(index=\"twosent_tud\", doc_type=\"twosentnorules\",\n",
    "                        body=query, size=1000)\n",
    "\n",
    "        # clean up the sentences and if they don't contain the names of the test set then add them as\n",
    "        # the training data\n",
    "#         for doc in res['hits']['hits']:\n",
    "        print(entity, len(res['hits']['hits']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Processing Entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import gensim\n",
    "import sys\n",
    "import os\n",
    "from nltk.tag.stanford import StanfordNERTagger\n",
    "from elasticsearch import Elasticsearch\n",
    "from pymongo import MongoClient\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.corpus import stopwords\n",
    "# from gensim.models.wrappers import FastText\n",
    "\n",
    "from m1_postprocessing import process_extracted_entities\n",
    "import config as cfg\n",
    "\n",
    "# embedding_model = FastText.load_fasttext_format('/data/modelFT')\n",
    "# embedding_model = gensim.models.Word2Vec.load('embedding_models/modelword2vecbigram.model')\n",
    "embedding_model = gensim.models.Word2Vec.load('full_corpus_models/modelword2vecbigram.model')\n",
    "\n",
    "es = Elasticsearch([{'host': 'localhost', 'port': 9200}])\n",
    "es.cluster.health(wait_for_status='yellow')\n",
    "\n",
    "client = MongoClient('localhost:' + str(cfg.mongoDB_Port))\n",
    "db_mongo = client.TU_Delft_Library.entities\n",
    "path_to_jar = cfg.STANFORD_NER_PATH\n",
    "entity_names = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def store_entity_in_mongo(db, _id, title, journal, year, word, in_wordnet, filtered_words, pmi_data, pmi_method,\n",
    "                          ds_similarity, mt_similarity, ds_sim_50, ds_sim_60, ds_sim_70, ds_sim_80, ds_sim_90,\n",
    "                          mt_sim_50, mt_sim_60, mt_sim_70, mt_sim_80, mt_sim_90):\n",
    "    my_ner = {\n",
    "        \"paper_id\": _id,\n",
    "        \"title\": title,\n",
    "        \"journal\": journal,\n",
    "        \"year\": year,\n",
    "        \"word\": word,\n",
    "        \"in_wordnet\": in_wordnet,\n",
    "        \"filtered_word\": filtered_word,\n",
    "        \"pmi_data\": pmi_data,\n",
    "        \"pmi_method\": pmi_method,\n",
    "        \"ds_similarity\": ds_similarity,\n",
    "        \"mt_similarity\": mt_similarity,\n",
    "        \"ds_sim_50\": ds_sim_50,\n",
    "        \"ds_sim_60\": ds_sim_60,\n",
    "        \"ds_sim_70\": ds_sim_70,\n",
    "        \"ds_sim_80\": ds_sim_80,\n",
    "        \"ds_sim_90\": ds_sim_90,\n",
    "        \"mt_sim_50\": mt_sim_50,\n",
    "        \"mt_sim_60\": mt_sim_60,\n",
    "        \"mt_sim_70\": mt_sim_70,\n",
    "        \"mt_sim_80\": mt_sim_80,\n",
    "        \"mt_sim_90\": mt_sim_90\n",
    "\n",
    "    }\n",
    "\n",
    "#     db.entities.update_one({'$and': [{'paper_id': my_ner['paper_id']}, {'word': word}]}, {'$set': my_ner}, upsert=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def get_entities(words, current_model):\n",
    "    results = []\n",
    "    facet_tag = current_model.upper()\n",
    "    for i, (a, b) in enumerate(words):\n",
    "        if b == facet_tag:\n",
    "            temp = a\n",
    "            if i > 1:\n",
    "                j = i - 1\n",
    "                if words[j][1] == facet_tag:\n",
    "                    continue\n",
    "            j = i + 1\n",
    "            try:\n",
    "                if words[j][1] == facet_tag:\n",
    "                    temp = b\n",
    "                    temp = words[j][0] + ' ' + b\n",
    "                    z = j + 1\n",
    "                    if words[j][1] == facet_tag:\n",
    "                        temp = a + ' ' + words[j][0] + ' ' + words[z][0]\n",
    "            except IndexError:\n",
    "                continue\n",
    "            results.append(temp)\n",
    "\n",
    "\n",
    "    wordnet_filtered = []\n",
    "    filtered_words = [word for word in set(results) if word not in stopwords.words('english')]\n",
    "\n",
    "    filtered_words = [word for word in set(results) if word not in stopwords.words('english')]\n",
    "    print(len(filtered_words), 'entities to process in this document')\n",
    "    \n",
    "    for word in set(filtered_words):\n",
    "        in_wordnet = 1\n",
    "        if not wordnet.synsets(word):\n",
    "            wordnet_filtered.append(word)\n",
    "            in_wordnet = 0\n",
    "\n",
    "        filtered_word, pmi_data, pmi_method, ds_similarity, mt_similarity, ds_sim_50, \\\n",
    "        ds_sim_60, ds_sim_70, ds_sim_80, ds_sim_90, mt_sim_50, mt_sim_60, mt_sim_70, \\\n",
    "        mt_sim_80, mt_sim_90 = process_extracted_entities.filter_it(word, embedding_model)\n",
    "        \n",
    "        print(filtered_word, pmi_data, pmi_method, ds_similarity, mt_similarity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "955 publications to process\n"
     ]
    }
   ],
   "source": [
    "publications = ['tudelft']\n",
    "model_name = 'method_tud_full'\n",
    "\n",
    "for publication in publications:\n",
    "    query = {\"query\":\n",
    "                {\"match\":\n",
    "                    {\"journal\":\n",
    "                        {\"query\": publication,\n",
    "                         \"operator\": \"and\"\n",
    "                         }\n",
    "                     }\n",
    "                 }\n",
    "             }\n",
    "\n",
    "    res = es.search(index=\"ir_tud\", doc_type=\"publications\",\n",
    "                    body=query, size=10000)\n",
    "\n",
    "    print(len(res['hits']['hits']), 'publications to process')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "955 publications to process\n",
      "Modeling and Inversion of Scattered Surface waves\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/nltk/tag/stanford.py:183: DeprecationWarning: \n",
      "The StanfordTokenizer will be deprecated in version 3.2.5.\n",
      "Please use \u001b[91mnltk.tag.corenlp.CoreNLPPOSTagger\u001b[0m or \u001b[91mnltk.tag.corenlp.CoreNLPNERTagger\u001b[0m instead.\n",
      "  super(StanfordNERTagger, self).__init__(*args, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21 entities to process in this document\n",
      "Fourier transformation given 0 0 0 0\n",
      "Problem Results 2.7750146346224454 2.975827828907664 0 0\n",
      "Fourier Transforms In 0 4.521796198012956 0 0\n",
      "sequence expansion 0 3.6737992914580064 0 0\n",
      "Fourier transform 1.661819561165165 3.383869658561152 0.019224574602318377 -0.005251271983596223\n",
      "number model 2.7162790577988436 2.4984881377850257 0 0\n",
      "conjugate gradient method 0 6.995727386345369 0 0\n",
      "analytical expression 0 4.078189546537342 0 0\n",
      "number 2.7174994785616624 2.4816776074273763 0.00345131345018591 -0.007272460138335303\n",
      "Fourier transform 1.661819561165165 3.383869658561152 0.019224574602318377 -0.005251271983596223\n",
      "number frequencies 0 1.5254074515653355 0 0\n",
      "part 2.0034735801073804 2.0719213203249787 0.015343580136696854 0.006523893347861064\n",
      "transform 2.5337531862654417 3.1997273592553404 -0.015198943795714124 0.029077524026399577\n",
      "effect 2.3119554295121936 2.2518145284046653 -0.014363177827582797 -0.004376279637342002\n",
      "number cells 2.382383548391881 2.472165430288355 0 0\n",
      "effects scattering 0 0 0 0\n",
      "whereas 1.9799460858926707 2.4885570462303632 -0.0015646925924531873 0.03225921959693965\n",
      "Fourier transform 1.661819561165165 3.383869658561152 0.019224574602318377 -0.005251271983596223\n",
      "Experimental Data In 4.064643250467952 3.6398519595346683 0 0\n",
      "Appendix C 3.325800020025514 2.8161198315057168 -0.0067145164687971645 -0.016743604998713996\n",
      "Conjugate Gradient Method 0 6.995727386345369 0 0\n",
      "955 more documents to go\n",
      "\n",
      "Hardware Acceleration of Bioinformatics Sequence Alignment Applications\n",
      "119 entities to process in this document\n",
      "sequence alignment method 0 6.995727386345369 0 0\n",
      "number available 2.868253214376944 2.1336066608568722 0 0\n",
      "biological sequence alignment 0 2.908264545095029 0 0\n",
      "Future Research Directions 1.4140924081192192 0.7669086958494877 0 0\n",
      "Sequence Alignment Methods 0 3.08883679073685 0 0\n",
      "Sequence Alignment Software 0 0 0 0\n",
      "quence alignment • 0 3.6737992914580064 0 0\n",
      "Local alignment Local 4.038049040456239 2.8059028274653515 0 0\n",
      "sequence alignment 0 2.0665676268958175 0.037093863404269234 0.0170699064573763\n",
      "overview molecular 0 0 0 0\n",
      "sequence alignment 0 2.0665676268958175 0.037093863404269234 0.0170699064573763\n",
      "GPUs 2.1653776736024923 0.5181939613327602 -0.020161342536205464 0.006879104128612698\n",
      "local alignment algorithms 0 4.295287668204277 0 0\n",
      "NVIDIA 0 0.7440082937394085 -0.022932910454096667 0.0041326171772631815\n",
      "pairwise sequence alignment 0 3.5362957677080717 0 0\n",
      "Hidden Morkov Models 0 0 0 0\n",
      "VHDL 0.3254984848502311 1.1808151131096827 0.0008947375803520471 -0.03771041502043908\n",
      "number launched 0 2.908264545095029 0 0\n",
      "pairwise local sequence 0 0 0 0\n",
      "future research direc- 0 0 0 0\n",
      "overview 2.622625597770453 2.21702787207048 -0.010700000212686597 0.004191451532088544\n",
      "sequence alignment applications 0 2.4309427675618425 0 0\n",
      "sequence alignment depends 0 0 0 0\n",
      "performance 3.2010203292113837 2.292535872491193 -0.002634496012545798 0.003204215063815152\n",
      "overview bioinformatics 0 0 0 0\n",
      "datasets various 6.188345235149618 1.7862740207164187 0 0\n",
      "Mean Square Error 0 3.825802384903056 0 0\n",
      "number PEs 0 3.0261010353888875 0 0\n",
      "FASTA 3.9799460858926707 2.7477998729017834 0.0034011731129268133 0.02090868853600473\n",
      "Local alignment G 0 0 0 0\n",
      "dynamic programming matrix 0 0 0 0\n",
      "sequence 1.521619345763053 2.552507538605959 0.00823192596936356 0.006785486977558884\n",
      "number database 4.248051481273887 1.1679083617280488 0 0\n",
      "Multiple sequence alignment 0 1.4565685752373374 0 0\n",
      "number sequence 1.2477340216970991 2.6005503094273674 0 0\n",
      "number N 1.9537450833001868 2.9366117612801497 0 0\n",
      "sequence alignment 0 2.0665676268958175 0.037093863404269234 0.0170699064573763\n",
      "sequence alignment meth- 0 0 0 0\n",
      "Sequence alignment 0 2.0665676268958175 0.037093863404269234 0.0170699064573763\n",
      "number 2.7174994785616624 2.4816776074273763 0.00345131345018591 -0.0091020007483803\n",
      "number threads 2.183479479977803 0.9513332669869152 0 0\n",
      "overview bioin- 0 0 0 0\n",
      "sequence alignment methods 0 3.08883679073685 0 0\n",
      "sequence alignment may 0 0 0 0\n",
      "number ways 2.555448257364761 2.060267638540079 0 0\n",
      "number positions 2.3949835851715147 3.332762373622939 0 0\n",
      "part S-W 0 0 0 0\n",
      "Sequence alignment aims 0 0 0 0\n",
      "Sequence alignment using 0 4.230192639982391 0 0\n",
      "overview cells 0 3.2952876682042764 0 0\n",
      "sequence alignment application. 0 0 0 0\n",
      "number steps 2.389930357445229 3.0057810510092917 0 0\n",
      "reconfigurable -1.546913460264917 0.5428684216315575 -0.0157283077922942 -0.021219734032477375\n",
      "dynamic programming algorithms. 0 3.6737992914580064 0 0\n",
      "Future research directions 1.4140924081192192 0.7669086958494877 0 0\n",
      "performance Swiss-Prot 0 0 0 0\n",
      "pairwise 4.083678872987362 3.1994559634167805 0.010092820958949712 -0.0163273090325268\n",
      "number memory 2.2695280337427595 2.011386612218927 0 0\n",
      "Local Alignment Search 0 5.4107648856242125 0 0\n",
      "sequence alignment 0 2.0665676268958175 0.037093863404269234 0.0170699064573763\n",
      "HMMER 0 0.5038742900156938 0.0021091524773374074 0.008332882975600272\n",
      "FPGA -1.2927450812200234 1.0799707639479499 -0.010260356473376931 -0.013272142489529062\n",
      "number anti-diagonals 0 0 0 0\n",
      "ClustalW ClustalW Comparison 0 0 0 0\n",
      "overview 2.622625597770453 2.21702787207048 -0.010700000212686597 0.008645604935861707\n",
      "alignment 1.9095567580012727 2.238125499484864 0.016444729504755517 -0.011621814947496261\n",
      "sequence alignment methods 0 3.08883679073685 0 0\n",
      "number PEs 0 3.0261010353888875 0 0\n",
      "query sequence symbol 0 0 0 0\n",
      "local alignment 4.038049040456239 2.8059028274653515 0.01743512736223148 0.01558070529933158\n",
      "FPGAs -0.9193313256059724 0.43348496212429594 0.003289706860586617 -0.0029051257754840066\n",
      "CUDA 0 1.8881122866984028 7.377261044053313e-05 -0.026242495696647373\n",
      "number chromosomes 0 0 0 0\n",
      "future research directions 1.4140924081192192 0.7669086958494877 0 0\n",
      "memory 1.4424299203545723 1.6683043771932744 0.0018341079088440148 -0.01653582498698649\n",
      "sequence alignment Hardware 0 3.5694626316432707 0 0\n",
      "local alignment 4.038049040456239 2.8059028274653515 0.01743512736223148 0.01558070529933158\n",
      "effect execution 0 1.7862740207164187 0 0\n",
      "number thread 3.108932526612749 2.8767863136218614 0 0\n",
      "sequence alignment applications. 0 2.4309427675618425 0 0\n",
      "SIMD -0.6639101038820538 1.0108342787355766 -0.009854619100223167 0.018095202164654066\n",
      "effect memory 0 1.1477304797904184 0 0\n",
      "performance 3.2010203292113837 2.292535872491193 -0.002634496012545798 0.015473163194908258\n",
      "whereas HMMER 0 0 0 0\n",
      "number PEs. 0 3.0261010353888875 0 0\n",
      "Hidden Markov Models 2.5554482573647608 3.3233020443738734 0 0\n",
      "sequence alignment 0 2.0665676268958175 0.037093863404269234 0.0170699064573763\n",
      "sequence alignment are: 0 1.8019556429486883 0 0\n",
      "number cycles 0 1.9204392590411312 0 0\n",
      "Intel 3.4552840954393287 1.3386150998683772 0.014985793655356735 0.0193745799773641\n",
      "whereas 1.9799460858926707 2.4885570462303632 -0.0015646925924531873 0.03225921959693965\n",
      "part species 0 3.2952876682042764 0 0\n",
      "number query 4.353404481420115 2.1212582684292274 0 0\n",
      "optimal alignment though. 0 0 0 0\n",
      "number matrix 2.787004431725387 3.8767863136218614 0 0\n",
      "sequence alignment applications 0 2.4309427675618425 0 0\n",
      "NVIDIA GPUs 0 0 0 0\n",
      "sequence alignment 0 2.0665676268958175 0.037093863404269234 0.0170699064573763\n",
      "substitution 0 1.2452995323726 -0.0012734473963206944 -0.004461111782801313\n",
      "part 2.0034735801073804 2.0719213203249787 0.015343580136696854 0.02434871557894319\n",
      "GPUs biological 0 0 0 0\n",
      "number recommendations 0 2.3403755577328145 0 0\n",
      "CUDASW++ 0 2.9957273863453686 0 0\n",
      "query sequence 0 2.655877383460744 0 0\n",
      "GPUs 2.1653776736024923 0.5181939613327602 -0.020161342536205464 0.021219992609523355\n",
      "genetic expression glosses 0 0 0 0\n",
      "number segments 0 2.592005200294359 0 0\n",
      "number GPU 4.273677288949381 2.8895279825134432 0 0\n",
      "maximum alignment score 0 0 0 0\n",
      "number 2.7174994785616624 2.4816776074273763 0.00345131345018591 -0.0091020007483803\n",
      "local alignment is: 3.835556176557496 2.6034099635666084 0 0\n",
      "Mutiple 0 0 -0.01778645645338525 0.037998511994723996\n",
      "dynamic programming 0.826994163054072 3.880250168925433 -0.009199939317276954 0.009305080120468294\n",
      "query sequence 0 2.655877383460744 0 0\n",
      "sequence alignment outperforming 0 0 0 0\n",
      "biological 3.9029430162873706 1.7862740207164187 0.00025443317053210497 -0.006017936703524939\n",
      "number top-scoring 0 0 0 0\n",
      "sequence 1.521619345763053 2.552507538605959 0.00823192596936356 0.012651008643499013\n",
      "dynamic programming algorithms 0 3.6737992914580064 0 0\n",
      "954 more documents to go\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(len(res['hits']['hits']), 'publications to process')\n",
    "prog = len(res['hits']['hits'])\n",
    "path_to_model = 'crf_trained_files/' + model_name + '_TSE_model_0.ser.gz'\n",
    "ner_tagger = StanfordNERTagger(path_to_model, path_to_jar)\n",
    "\n",
    "for doc in res['hits']['hits'][:2]:\n",
    "    text = doc[\"_source\"][\"content\"]\n",
    "    print(doc[\"_source\"][\"title\"])\n",
    "    labelled_words = ner_tagger.tag(text.split())\n",
    "    get_entities(labelled_words, model_name)\n",
    "    print(prog, 'more documents to go')\n",
    "    prog = prog - 1\n",
    "    print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/nltk/tag/stanford.py:183: DeprecationWarning: \n",
      "The StanfordTokenizer will be deprecated in version 3.2.5.\n",
      "Please use \u001b[91mnltk.tag.corenlp.CoreNLPPOSTagger\u001b[0m or \u001b[91mnltk.tag.corenlp.CoreNLPNERTagger\u001b[0m instead.\n",
      "  super(StanfordNERTagger, self).__init__(*args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "model_name = 'dataset_100'\n",
    "path_to_model = 'crf_trained_files/' + model_name + '_TSE_model_0.ser.gz'\n",
    "ner_tagger = StanfordNERTagger(path_to_model, path_to_jar)\n",
    "labelled_words = ner_tagger.tag(text.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from m1_postprocessing.filtering import normalized_entity_distance\n",
    "# import gensim \n",
    "\n",
    "# embedding_model = gensim.models.Word2Vec.load('models/modelword2vecbigram.model')\n",
    "\n",
    "ds_names = []\n",
    "mt_names = []\n",
    "# prnames = []\n",
    "\n",
    "dataset_path = 'data/dataset_names2.txt'\n",
    "with open(dataset_path, \"r\") as file:\n",
    "    for row in file.readlines():\n",
    "        ds_names.append(row.strip())\n",
    "\n",
    "method_path = 'data/method_names2.txt'\n",
    "with open(method_path, \"r\") as file:\n",
    "    for row in file.readlines():\n",
    "        mt_names.append(row.strip())\n",
    "        \n",
    "def is_int_or_float(s):\n",
    "    \"\"\"\n",
    "    return 1 for int, 2 for float, -1 for not a number\n",
    "    \"\"\"\n",
    "    try:\n",
    "        float(s)\n",
    "        return 1 if s.count('.') == 0 else 2\n",
    "    except ValueError:\n",
    "        return -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['markov', 'chains']"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ner_word = 'markov chains'\n",
    "ner_word = ner_word.split()\n",
    "ner_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "filter_by_wordnet = []\n",
    "filtered_words = [word for word in ner_word if word not in stopwords.words('english')]\n",
    "\n",
    "for word in filtered_words:\n",
    "    isint = is_int_or_float(word)\n",
    "    if isint != -1:\n",
    "        filtered_words.remove(word)\n",
    "\n",
    "for word in set(filtered_words):\n",
    "    in_wordnet = 1\n",
    "    inds = 0\n",
    "\n",
    "    if not wordnet.synsets(word):\n",
    "        in_wordnet = 0\n",
    "        filter_by_wordnet.append(word)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['markov', 'chains']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "filtered_words = ' '.join(filtered_words)\n",
    "filtered_words = filtered_words.replace('(', '')\n",
    "filtered_words = filtered_words.replace(')', '')\n",
    "filtered_words = filtered_words.replace('[', '')\n",
    "filtered_words = filtered_words.replace(']', '')\n",
    "filtered_words = filtered_words.replace('{', '')\n",
    "filtered_words = filtered_words.replace('}', '')\n",
    "filtered_word = filtered_words.replace(',', '')\n",
    "lower_filtered_word = filtered_word.lower()\n",
    "filter_by_wordnet = ' '.join(filter_by_wordnet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'markov chains'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lower_filtered_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.908264545095029, 4.140410758085917)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pmi_data = normalized_entity_distance(filtered_word, 'dataset')\n",
    "pmi_method = normalized_entity_distance(filtered_word, 'method')\n",
    "pmi_method, pmi_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'markov_chains'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds_similarity = []\n",
    "mt_similarity = []\n",
    "lower_filtered_word = lower_filtered_word.replace(' ', '_')\n",
    "lower_filtered_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "simulated_annealing -0.07921258353650174 --------------------------------------------------\n",
      "linear_regression 0.1696744497926577 --------------------------------------------------\n",
      "genetic_programming -0.0703554376625785 --------------------------------------------------\n",
      "qald 0.014135182261109129 --------------------------------------------------\n",
      "downhill_simplex -0.0657312573852436 --------------------------------------------------\n",
      "regular_expression -0.03982873864022178 --------------------------------------------------\n",
      "selection_algorithm 0.19964432417909211 --------------------------------------------------\n",
      "imrank -0.11654380197968744 --------------------------------------------------\n",
      "adarank 0.028082375279707885 --------------------------------------------------\n",
      "similarity_search 0.04953733805337633 --------------------------------------------------\n",
      "game_theory -0.0413717585760423 --------------------------------------------------\n",
      "mcmc -0.1346701443416963 --------------------------------------------------\n",
      "fourier_analysis -0.00979193576186222 --------------------------------------------------\n",
      "linkage_analysis 0.10348088579238943 --------------------------------------------------\n",
      "pearson_correlation -0.035060773985509094 --------------------------------------------------\n",
      "dijkstra -0.0894115666356401 --------------------------------------------------\n",
      "cyclades 0.07117641269640328 --------------------------------------------------\n",
      "fuzzy_clustering -0.19731268016114167 --------------------------------------------------\n",
      "bayesian_nonparametric -0.20509912506937783 --------------------------------------------------\n",
      "fast_fourier -0.12219122764248315 --------------------------------------------------\n",
      "dmp_method 0.001336153266869799 --------------------------------------------------\n",
      "query_expansion -0.08872601322803182 --------------------------------------------------\n",
      "spectral_clustering 0.016198490623966992 --------------------------------------------------\n",
      "transfer_function 0.06855390798381635 --------------------------------------------------\n",
      "recursive_function 0.01647558212724418 --------------------------------------------------\n",
      "rapid7 -0.07367569534822502 --------------------------------------------------\n",
      "random_forest -0.011770682055785374 --------------------------------------------------\n",
      "quicksort -0.053875686394518896 --------------------------------------------------\n",
      "imputation -0.0023843621889304816 --------------------------------------------------\n",
      "hill_climbing 0.051099888256524235 --------------------------------------------------\n",
      "likelihood_function -0.07324299439990543 --------------------------------------------------\n",
      "dynamic_programming -0.031083150697933708 --------------------------------------------------\n",
      "skipgram -0.006959886697026212 --------------------------------------------------\n",
      "predictive_modeling 0.09944809852350339 --------------------------------------------------\n",
      "deep_learning -0.06452013451613826 --------------------------------------------------\n",
      "bcdrw -0.20191631182979478 --------------------------------------------------\n",
      "shannon_entropy -0.06464438924060649 --------------------------------------------------\n",
      "tagassist 0.05887162912493832 --------------------------------------------------\n",
      "model_fitting -0.023249947184859385 --------------------------------------------------\n",
      "lstm 0.03943523879385936 --------------------------------------------------\n",
      "autoencoder 0.06436846904824316 --------------------------------------------------\n",
      "dbscan 0.14211198808674738 --------------------------------------------------\n",
      "plsa 0.06790531150159516 --------------------------------------------------\n",
      "clir -0.04779872748335329 --------------------------------------------------\n",
      "klsh -0.1359864018892617 --------------------------------------------------\n",
      "pattern_matching -0.17429639056199595 --------------------------------------------------\n",
      "gpbased -0.03748863124661735 --------------------------------------------------\n",
      "merge_sort -0.15868832476753786 --------------------------------------------------\n",
      "query_optimization -0.05489809047056623 --------------------------------------------------\n",
      "hierarchical_agglomerative -0.01977300788906216 --------------------------------------------------\n",
      "tdcm 0.15117626522975627 --------------------------------------------------\n",
      "tsa_algorithm -0.04878368547098474 --------------------------------------------------\n",
      "greedy_algorithm -0.05916425980370022 --------------------------------------------------\n",
      "53 47\n",
      "[-0.07921258353650174, 0.1696744497926577, -0.0703554376625785, 0.014135182261109129, -0.0657312573852436, -0.03982873864022178, 0.19964432417909211, -0.11654380197968744, 0.028082375279707885, 0.04953733805337633, -0.0413717585760423, -0.1346701443416963, -0.00979193576186222, 0.10348088579238943, -0.035060773985509094, -0.0894115666356401, 0.07117641269640328, -0.19731268016114167, -0.20509912506937783, -0.12219122764248315, 0.001336153266869799, -0.08872601322803182, 0.016198490623966992, 0.06855390798381635, 0.01647558212724418, -0.07367569534822502, -0.011770682055785374, -0.053875686394518896, -0.0023843621889304816, 0.051099888256524235, -0.07324299439990543, -0.031083150697933708, -0.006959886697026212, 0.09944809852350339, -0.06452013451613826, -0.20191631182979478, -0.06464438924060649, 0.05887162912493832, -0.023249947184859385, 0.03943523879385936, 0.06436846904824316, 0.14211198808674738, 0.06790531150159516, -0.04779872748335329, -0.1359864018892617, -0.17429639056199595, -0.03748863124661735, -0.15868832476753786, -0.05489809047056623, -0.01977300788906216, 0.15117626522975627, -0.04878368547098474, -0.05916425980370022]\n"
     ]
    }
   ],
   "source": [
    "mt_similarity = []\n",
    "x = 0\n",
    "y = 0\n",
    "for mt in mt_names:\n",
    "    try:\n",
    "        mt = mt.replace(' ', '_')\n",
    "        similarity = embedding_model.wv.similarity(mt, lower_filtered_word)\n",
    "        print(mt, similarity, '-'*50)\n",
    "        mt_similarity.append(similarity)\n",
    "        y = y + 1\n",
    "    except KeyError:\n",
    "#         print(mt, 'not in vectorspace')\n",
    "        x = x + 1\n",
    "print(y, x)\n",
    "# try:\n",
    "#     mt_similarity = float(sum(mt_similarity)) / len(mt_similarity)\n",
    "# except ZeroDivisionError:\n",
    "#     pass\n",
    "\n",
    "print(mt_similarity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "query = {}\n",
    "res = es.search(index=\"twosent_tud\", doc_type=\"twosentnorules\", body=query)\n",
    "    total_a = res['hits']['total']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "593162"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_a"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Editing Entities in Mongo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000\n",
      "20000\n",
      "30000\n",
      "40000\n",
      "50000\n",
      "60000\n",
      "70000\n",
      "80000\n",
      "90000\n",
      "100000\n",
      "110000\n",
      "120000\n",
      "130000\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "from pymongo import MongoClient\n",
    "from config import mongoDB_Port\n",
    "import string\n",
    "\n",
    "tr = str.maketrans(\"\", \"\", string.punctuation)\n",
    "client = MongoClient('localhost:' + str(mongoDB_Port))\n",
    "db = client.TU_Delft_Library\n",
    "pub = db.publications\n",
    "es = Elasticsearch(\n",
    "    [{'host': 'localhost', 'port': 9200}], timeout=30, max_retries=10, retry_on_timeout=True\n",
    ")\n",
    "es.cluster.health(wait_for_status='yellow', request_timeout=1)\n",
    "allentities = db.entities.find()\n",
    "words = []\n",
    "x = 0\n",
    "for rr in allentities:\n",
    "    word = rr['word'].lower()\n",
    "    no_punkt = word.translate(tr)\n",
    "    clean = ''.join([i for i in no_punkt if not i.isdigit()])\n",
    "    lower = rr['word'].lower()\n",
    "    \n",
    "    db.entities.update_one({'_id' : rr['_id']}, \n",
    "                           {\"$set\" : {'clean': clean, 'no_punkt': no_punkt,\n",
    "                                      'word_lower' : lower}}, upsert = False)\n",
    "    x = x + 1\n",
    "    if x % 10000 == 0:\n",
    "        print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "tud_db = client.TU_Delft_Library\n",
    "pub_db = client.pub\n",
    "tud_entities = tud_db.entities.find()\n",
    "pub_entities = pub_db.entities.find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "ann = {}\n",
    "for rr in pub_entities:\n",
    "    if rr['Annotator'] != 'undefined':\n",
    "        ann[rr['word_lower']] = rr['Annotator']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "tud_entities = tud_db.entities.find()\n",
    "for rr in tud_entities:\n",
    "    try:\n",
    "        new = ann[rr['word_lower']]\n",
    "        db.entities.update_one({'_id' : rr['_id']}, {\"$set\" : {'Annotator' : new}}, upsert = False)\n",
    "    except KeyError:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "tud_entities = tud_db.entities.find()\n",
    "for rr in tud_entities:\n",
    "    db.entities.update_one({'_id' : rr['_id']}, {\"$set\" : {'experiment' : '1cycle_tud_word2vec'}}, upsert = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}