5. Linking to WikiData.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d865bd1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import random \n",
    "from tqdm import tqdm\n",
    "import threading\n",
    "import requests\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1518642b",
   "metadata": {},
   "source": [
    "### Read the CSV file"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad62bb14",
   "metadata": {},
   "source": [
    "### Load the file SUBRELOBJ.csv from https://zenodo.org/records/10022727"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3be66a79",
   "metadata": {},
   "outputs": [],
   "source": [
    "SUBRELOBJ = pd.read_csv('RDF Files/SUBRELOBJ.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a73efd20",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Number of rows: {}\".format(SUBRELOBJ.shape))\n",
    "print(\"Columns : {}\".format(SUBRELOBJ.columns.tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03f86f67",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_entities = list(set(SUBRELOBJ['Subject'].tolist() + SUBRELOBJ['Object'].tolist()))\n",
    "print(\"Number of unique entities : {}\".format(len(unique_entities)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "caf071c2",
   "metadata": {},
   "source": [
    "## Query wikidata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6300eb7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_entity_info(index):\n",
    "    \n",
    "    entity_name = unique_entities[index]\n",
    "    # Construct the Wikidata API URL for the search query\n",
    "    url = f\"https://www.wikidata.org/w/api.php?action=wbsearchentities&search={entity_name}&language=en&format=json\"\n",
    "\n",
    "    # Send a GET request to the Wikidata API\n",
    "    response = requests.get(url)\n",
    "\n",
    "    # Check if the search was successful\n",
    "    if response.status_code == 200:\n",
    "        # Extract the first search result from the JSON response\n",
    "        search_results = response.json().get(\"search\")\n",
    "        if search_results:\n",
    "           \n",
    "            entity_dict[entity_name]=search_results\n",
    "        else:\n",
    "            return None\n",
    "    else:\n",
    "        return None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76325679",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def arg_func(i):\n",
    "    return i\n",
    "\n",
    "def download_threader(func, max_num, threads, arg_func, N=None):\n",
    "    \n",
    "    for i in tqdm(range(0,max_num,threads)):\n",
    "            \n",
    "            limit=min(threads, max_num-i)\n",
    "        \n",
    "            parserThreads=[]\n",
    "            \n",
    "            for r in range(0,limit):\n",
    "    \n",
    "                parserThread=threading.Thread(target=func,args=(arg_func(i+r),))\n",
    "                parserThreads.append(parserThread)    \n",
    "                parserThread.start()\n",
    "                    \n",
    "            for parserThread in parserThreads:\n",
    "                parserThread.join()\n",
    "                \n",
    "            \n",
    "            if (i+1)%500==0:\n",
    "                 print(\"Number of entities found : {}\".format(len(entity_dict)))\n",
    "                 time.sleep(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9564685",
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_dict = {}\n",
    "download_threader(get_entity_info, len(unique_entities), 9, arg_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9233599c",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(entity_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cccd27a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "vineeth_10",
   "language": "python",
   "name": "vineeth_10"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}