hatebase_api_howto.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialize Variables and authenticate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize variables\n",
    "auth_url = \"https://api.hatebase.org/4-2/authenticate\"\n",
    "vocab_url = \"https://api.hatebase.org/4-2/get_vocabulary\"\n",
    "lang = \"eng\"\n",
    "resp_format = \"json\"\n",
    "token = \"\"\n",
    "pages = 0\n",
    "total_entries = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize authentication payload\n",
    "auth_payload = \"api_key=mmKPRoEEPSCHTI34598dhtP\"\n",
    "headers = {\n",
    "    'Content-Type': \"application/x-www-form-urlencoded\",\n",
    "    'cache-control': \"no-cache\"\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# authenticate\n",
    "auth_resp = requests.request(\"POST\", auth_url, data=auth_payload, headers=headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the session token\n",
    "token = auth_resp.json()[\"result\"][\"token\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# quick check if it worked\n",
    "print(auth_resp.text, token)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initialize Vocabulary payload and build df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize vocabulary payload\n",
    "# first without any given page-number\n",
    "vocab_payload = \"token=\" + token + \"&format=\" + resp_format + \"&language=\" + lang\n",
    "voc_resp = requests.request(\"POST\", vocab_url, data=vocab_payload, headers=headers)\n",
    "voc_json = voc_resp.json()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you have a look at the received voc_json (i.e. print it), it should look somewhat similar to this:\n",
    "```\n",
    "{\n",
    "    \"datetime\": \"2019-03-11 08:04:08\",\n",
    "    \"token\": \"kiooZAEGnEdxjXhPtxNqVVNLrNfMPoXA\",\n",
    "    \"format\": \"json\",\n",
    "    \"language\": \"eng\",\n",
    "    \"number_of_results\": 1384,\n",
    "    \"number_of_pages\": 14,\n",
    "    \"page\": 1,\n",
    "    \"result\": [\n",
    "        {\n",
    "            \"vocabulary_id\": \"Id0fThatVoc4b\",\n",
    "            \"term\": \"some-term\",\n",
    "```\n",
    "            \n",
    "etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# how many pages in total? and how many results= entries in the vocab?\n",
    "pages = voc_json[\"number_of_pages\"]\n",
    "results = voc_json[\"number_of_results\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14, 1384)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check pages & results\n",
    "pages, results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create vocabulary df from first resultset\n",
    "df_voc = pd.DataFrame(voc_json[\"result\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# now get results of all remaining pages and append to df_voc\n",
    "for page in range(2,pages+1):\n",
    "    vocab_payload = \"token=\" + token + \"&format=json&language=\" + lang + \"&page=\" + str(page)\n",
    "    voc_resp = requests.request(\"POST\", vocab_url, data=vocab_payload, headers=headers)\n",
    "    voc_json = voc_resp.json()\n",
    "    df_voc = df_voc.append(voc_json[\"result\"])\n",
    "\n",
    "# reset df_voc index\n",
    "df_voc.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1384, 23)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_voc.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## save hate vocabulary to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_voc.to_csv(\"hatebase_vocab.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}