notebook/robo_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Derive Analytics on data using Python Jupyter Notebook, Watson Conversation & NAO Robot \n",
    "### <font color='blue'> Expose an integration point using websockets for orchestration with Node-RED using Watson Conversation & NAO Robot.</font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Setup\n",
    "To prepare your environment, you need to install some packages.\n",
    "\n",
    "### 1.1 Install the necessary packages\n",
    "\n",
    "You need the latest versions of these packages:<br>\n",
    "- websocket-client: is a python client for the Websockets.<br>\n",
    "\n",
    "** Install the websocket client: **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install websocket-client"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2 Import packages and libraries\n",
    "\n",
    "Import the packages and libraries that are needed:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from io import StringIO\n",
    "import requests\n",
    "import json\n",
    "import pandas as pd\n",
    "import websocket\n",
    "import json\n",
    "from datetime import datetime\n",
    "import _thread\n",
    "import time\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Read the Data & convert it into Dataframe\n",
    "Click on Insert to code and then select Insert pandas DataFrame in the below empty cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import pandas as pd\n",
    "#df = pd.read_csv('C:/Users/IBM_ADMIN/Desktop/ICM 5.3.0/data.csv')\n",
    "import sys\n",
    "import types\n",
    "import pandas as pd\n",
    "from botocore.client import Config\n",
    "import ibm_boto3\n",
    "\n",
    "def __iter__(self): return 0\n",
    "\n",
    "# @hidden_cell\n",
    "# The following code accesses a file in your IBM Cloud Object Storage. It includes your credentials.\n",
    "# You might want to remove those credentials before you share your notebook.\n",
    "client_554c07c959184a5fb85f7723b7045646 = ibm_boto3.client(service_name='s3',\n",
    "    ibm_api_key_id='WtG7s66gkXY7EubDvehAdfx-KH3r0WurPV92SECjEQUU',\n",
    "    ibm_auth_endpoint=\"https://iam.ng.bluemix.net/oidc/token\",\n",
    "    config=Config(signature_version='oauth'),\n",
    "    endpoint_url='https://s3-api.us-geo.objectstorage.service.networklayer.com')\n",
    "\n",
    "body = client_554c07c959184a5fb85f7723b7045646.get_object(Bucket='patternsnew-donotdelete-pr-sjba0eve5tgfo9',Key='data.csv')['Body']\n",
    "# add missing __iter__ method, so pandas accepts body as file-like object\n",
    "if not hasattr(body, \"__iter__\"): body.__iter__ = types.MethodType( __iter__, body )\n",
    "\n",
    "df_data_1 = pd.read_csv(body)\n",
    "df_data_1.head()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Assign a new name for the dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Rename the dataframe to df'''\n",
    "df = df_data_1\n",
    "#df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Rounding off the numeric columns in the dataset'''\n",
    "tmp = df.select_dtypes(include=[np.number])\n",
    "df.loc[:, tmp.columns] = np.round(tmp)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Create Generic re-usable functions for Statistics\n",
    "\n",
    "#####      These functions can be applied to any dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def get_max_val(tuple1,tuple2,tuple3,tuple4):\n",
    "    '''This Function takes four input parameters to find max value'''\n",
    "    mr = df[(df[tuple1[0]] == tuple1[1]) & (df[tuple2[0]] == tuple2[1]) & (df[tuple3[0]] == tuple3[1])] [tuple4[0]].max()\n",
    "    return mr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_min_val(tuple1,tuple2,tuple3,tuple4):\n",
    "    '''This Function takes four input parameters to find min value'''\n",
    "    mr = df[(df[tuple1[0]] == tuple1[1]) & (df[tuple2[0]] == tuple2[1]) & (df[tuple3[0]] == tuple3[1])] [tuple4[0]].min()\n",
    "    return mr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_total_val(tuple1,tuple2,tuple3):\n",
    "    '''This Function takes three input parameters to find total value'''\n",
    "    mr = df[(df[tuple1[0]] == tuple1[1]) & (df[tuple2[0]] == tuple2[1])] [tuple3[0]].sum()\n",
    "    return mr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_avg_val(tuple1,tuple2,tuple3):\n",
    "    '''This Function takes three input parameters to find average value'''\n",
    "    mr = df[(df[tuple1[0]] == tuple1[1]) & (df[tuple2[0]] == tuple2[1])] [tuple3[0]].mean()\n",
    "    return mr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_three(t1,t2):\n",
    "    '''This Function takes two input parameters to find top three'''\n",
    "    tt = df.groupby(t1)[[t2]].sum().apply(lambda x: x.sort_values(ascending=False).head(3))\n",
    "    return tt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_new(t1,t2,t3):\n",
    "    '''This Function takes three input parameters to find top n value - top of n value is dynamic'''\n",
    "    tt = df.groupby(t1)[[t2]].sum().apply(lambda x: x.sort_values(ascending=False).head(t3))\n",
    "    return tt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_three_1(t1,t2,t3,t4):\n",
    "    '''This Function takes four input parameters to find top n value - top of n value is dynamic'''\n",
    "    df1 = df[(df[t1[0]] == t1[1])] \n",
    "    tt = df1.groupby(t2)[[t3]].sum().apply(lambda x: x.sort_values(ascending=False).head(t4))\n",
    "    return tt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_three_2(t1,t2,t3,t4,t5):\n",
    "    '''This Function takes five input parameters to find top n value - top of n value is dynamic'''\n",
    "    df1 = df[(df[t1[0]] == t1[1]) & (df[t2[0]] == t2[1])] \n",
    "    tt = df1.groupby(t3)[[t4]].sum().apply(lambda x: x.sort_values(ascending=False).head(t5))\n",
    "    return tt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Bottom_three(t1,t2):\n",
    "    '''This Function takes two input parameters to find bottom three'''\n",
    "    bt = df.groupby(t1)[[t2]].sum().apply(lambda x: x.sort_values(ascending=False).tail(3))\n",
    "    return bt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Bottom_new(t1,t2,t3):\n",
    "    '''This Function takes three input parameters to find bottom n value - bottom of n value is dynamic'''\n",
    "    bt = df.groupby(t1)[[t2]].sum().apply(lambda x: x.sort_values(ascending=False).tail(t3))\n",
    "    return bt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bottom_three_1(t1,t2,t3,t4):\n",
    "    '''This Function takes four input parameters to find bottom n value - bottom of n value is dynamic'''\n",
    "    df1 = df[(df[t1[0]] == t1[1])] \n",
    "    tt = df1.groupby(t2)[[t3]].sum().apply(lambda x: x.sort_values(ascending=False).tail(t4))\n",
    "    return tt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bottom_three_2(t1,t2,t3,t4,t5):\n",
    "    '''This Function takes five input parameters to find bottom n value - bottom of n value is dynamic'''\n",
    "    df1 = df[(df[t1[0]] == t1[1]) & (df[t2[0]] == t2[1])] \n",
    "    tt = df1.groupby(t3)[[t4]].sum().apply(lambda x: x.sort_values(ascending=False).tail(t5))\n",
    "    return tt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Expose Integration Point with a Websocket Client "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''This part of the code imports the question from websocket, executes the function as per \n",
    "   corresponding intent and sends the output back to the websocket.''' \n",
    "\n",
    "\n",
    "def on_message(ws, message):\n",
    "    msg = json.loads(message)\n",
    "    for item in msg[\"botresponse\"][\"messageout\"][\"context\"][\"entities_array\"]:\n",
    "        if item['entity'] == 'Year':\n",
    "            date = datetime.strptime(item['value'],'%Y')\n",
    "            item['value'] = date.year\n",
    "    print(msg)  \n",
    "        \n",
    "    entitylist = []\n",
    "    for index,entity in enumerate(msg[\"botresponse\"][\"messageout\"][\"context\"][\"entities_array\"]):\n",
    "        print(entity,index)\n",
    "        entitylist.append((entity[\"entity\"],entity[\"value\"]))\n",
    "\n",
    "    contextlist = [list(e) for e in msg[\"botresponse\"][\"messageout\"][\"context\"].items()]\n",
    "\n",
    "    if msg[\"botresponse\"][\"messageout\"][\"context\"][\"intent\"] == 'summation':\n",
    "        response = get_total_val(entitylist[1],entitylist[0],entitylist[2])\n",
    "        response = round(response,2)\n",
    "        print(response)\n",
    "    elif msg[\"botresponse\"][\"messageout\"][\"context\"][\"intent\"] == 'average':\n",
    "        response = get_avg_val(entitylist[1],entitylist[0],entitylist[2])\n",
    "        response = round(response,2)\n",
    "        print(response)\n",
    "    elif msg[\"botresponse\"][\"messageout\"][\"context\"][\"intent\"] == 'max':\n",
    "        response = get_max_val(entitylist[2],entitylist[1],entitylist[0],entitylist[3])\n",
    "        response = round(response,2)\n",
    "        print(response)\n",
    "    elif msg[\"botresponse\"][\"messageout\"][\"context\"][\"intent\"] == 'min':\n",
    "        response = get_min_val(entitylist[2],entitylist[1],entitylist[0],entitylist[3])\n",
    "        response = round(response,2)\n",
    "        print(response)\n",
    "    elif msg[\"botresponse\"][\"messageout\"][\"context\"][\"intent\"] == 'top':\n",
    "        response = top_new(entitylist[0][0],entitylist[2][0],entitylist[1][0])        \n",
    "        response = response.to_json()\n",
    "        print(response)\n",
    "    elif msg[\"botresponse\"][\"messageout\"][\"context\"][\"intent\"] == 'bottom':\n",
    "        response = Bottom_new(entitylist[0][0],entitylist[2][0],entitylist[1][0])\n",
    "        response = response.to_json()\n",
    "        print(response)\n",
    "            \n",
    "    Question = str(msg[\"botresponse\"][\"messageout\"][\"input\"][\"text\"])\n",
    "    responsejson = {}\n",
    "    responsejson[\"Question\"] = Question\n",
    "    responsejson[\"Answer\"] = str(response)\n",
    "    print(responsejson)\n",
    "    print(type(responsejson))\n",
    "    ws.send(json.dumps(responsejson))\n",
    "        \n",
    "def on_error(ws, error):\n",
    "    print(error)\n",
    "\n",
    "def on_close(ws):\n",
    "    ws.send(\"DSX Listen End\")\n",
    "\n",
    "def on_open(ws):\n",
    "    def run(*args):\n",
    "        for i in range(10000):\n",
    "            hbeat = '{\"cmd\":\"DSX HeartBeat\"}'\n",
    "            ws.send(hbeat)\n",
    "            time.sleep(100)\n",
    "\n",
    "    thread.start_new_thread(run, ())\n",
    "    \n",
    "        \n",
    "def start_websocket_listener():\n",
    "    websocket.enableTrace(True)\n",
    "    ws = websocket.WebSocketApp(\"ws://orchestrate-app.au-syd.mybluemix.net/ws/orchestrate\",\n",
    "                              on_message = on_message,\n",
    "                              on_error = on_error,\n",
    "                              on_close = on_close)\n",
    "    ws.on_open = on_open\n",
    "    ws.run_forever()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Start Websocket Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_websocket_listener()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3.5",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}