final_glue.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Final Exercise\n",
    "\n",
    "Use the `chatbot`, `template_matching` and `speech_processing` notebooks to create a voice activated chatbot that answers yes/know questions.\n",
    "\n",
    "Solution:\n",
    "\n",
    "- Use the `Bot` class and the `yes_no_processor` to get a ready made chatbot\n",
    "- Create a new `speech_source` for your `Bot` instance\n",
    "- Use the `AudioManager` from `speech_processing` to record audio\n",
    "- Extract MFCCs for the audio clips corresponding to yes and no\n",
    "- Use the `Trellis` idea from `template_matching` to recognize yes/no "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing notebook from chatbot.ipynb\n",
      "importing notebook from template_matching.ipynb\n",
      "importing notebook from speech_processing.ipynb\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "import importer\n",
    "from chatbot import StatementProcessor, get_yes_no_processor, get_keyboard_source, Bot\n",
    "from template_matching import Trellis\n",
    "from speech_processing import AudioManager\n",
    "\n",
    "from python_speech_features import mfcc\n",
    "from python_speech_features.base import delta\n",
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "import pickle\n",
    "import os\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    # Install python_speech_features that contains a routine to extract mfcc\n",
    "    !pip install -U python_speech_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "class TemplateManager:\n",
    "\n",
    "    @staticmethod\n",
    "    def build_templates(words=[\"test\", \"hello\", \"welcome\", \"goodbye\"],\n",
    "                        no_templates=1,\n",
    "                        output_file=\"templates.out\"):\n",
    "\n",
    "        audioManager = AudioManager()\n",
    "\n",
    "        templates = defaultdict(list)\n",
    "\n",
    "        for word in words:\n",
    "            for ii in range(no_templates):\n",
    "                ok = 'n'\n",
    "                while (ok.lower()=='n'):\n",
    "                    print(\"%d/%d Say %s\" %(ii, no_templates, word))\n",
    "                    samples = audioManager.record(2, filter_silence=False)\n",
    "                    audioManager.play(samples)\n",
    "                    features = feature_extractor(samples)\n",
    "                    templates[word].append(features)\n",
    "                    #ok = raw_input(\"OK?\") # python2\n",
    "                    ok = input(\"OK?\") # python3\n",
    "        pickle.dump(templates, open(output_file,\"wb\"))\n",
    "    \n",
    "    @staticmethod\n",
    "    def get_templates(filename):\n",
    "        if os.path.exists(filename):\n",
    "            return pickle.load(open(output_file,\"rb\"))\n",
    "        else:\n",
    "            print(\"Template file not found.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def feature_extractor(samples):\n",
    "    samples = np.concatenate(samples)\n",
    "    samples = samples/np.abs(samples).max()\n",
    "    samples = samples - samples.mean()\n",
    "    mfcc_features = mfcc(samples, samplerate=8000, winlen=0.032, winstep=0.016, numcep=13, appendEnergy=True, preemph=0)\n",
    "    #features = np.vstack((mfcc_features, delta(mfcc_features, 1)))\n",
    "    features = mfcc_features\n",
    "    return features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def scoring_func(x, y):\n",
    "    #print(x.shape, y.shape)\n",
    "    #print(x, y)\n",
    "    return np.abs(x - y).sum()\n",
    "\n",
    "\n",
    "def get_speech_source(filename):\n",
    "    # Load speech templates\n",
    "    # Return a function that can detect speech\n",
    "    audioManager = AudioManager()\n",
    "    trellis = Trellis(match_weight=1.0, delete_weight=1.0, add_weight=1.0, scoring_func=scoring_func)\n",
    "    \n",
    "    templates_dict = TemplateManager.get_templates(filename)\n",
    "    statement_processor = StatementProcessor()\n",
    "    \n",
    "    def speech_source():\n",
    "        best_scoring_word = \"\"\n",
    "        #inp = raw_input(\"Start recording?\") # python2\n",
    "        inp = input(\"Start recording?\") # python3\n",
    "        if len(inp)>0 and inp[0] == \"/\":\n",
    "            return inp\n",
    "        samples = audioManager.record(2, wait_for_kb=False)\n",
    "        features = feature_extractor(samples)\n",
    "        \n",
    "        min_score = 1e9\n",
    "        min_word = \"\"\n",
    "        for word, word_templates in templates_dict.items():\n",
    "            avg_score = 0.0\n",
    "            for word_template in word_templates:\n",
    "                score, bp = trellis.match(word_template, features)\n",
    "                avg_score += score\n",
    "            avg_score = avg_score / float(len(word_templates))\n",
    "            #print(word, avg_score)\n",
    "            if avg_score < min_score:\n",
    "                min_score = avg_score\n",
    "                min_word = word\n",
    "        print(\"YOU>> \", min_word)\n",
    "        return min_word\n",
    "        # Record some audio\n",
    "        # Match the audio with every template using Trellis\n",
    "        # Return the best scoring result \n",
    "        return best_scoring_word\n",
    "    return speech_source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "words = [\"yes\", \"no\"]\n",
    "no_templates = 1\n",
    "output_file = \"templates.out\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0/1 Say yes\n",
      "Press Enter to start recording...\n",
      "* recording\n",
      "* done recording\n",
      "OK?\n",
      "0/1 Say no\n",
      "Press Enter to start recording...\n",
      "* recording\n",
      "* done recording\n",
      "OK?\n"
     ]
    }
   ],
   "source": [
    "TemplateManager.build_templates(words, no_templates, output_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start recording?\n",
      "* recording\n",
      "* done recording\n",
      "before 125\n",
      "after 125\n",
      "YOU>>  yes\n",
      "[ 0 ] Poincare >>  Is it raining?\n",
      "[ 0 ] Poincare >>  Give the right answer.\n",
      "Start recording?\n"
     ]
    }
   ],
   "source": [
    "chatbot = Bot(statement_processor=StatementProcessor(statement_logic=get_yes_no_processor()),\n",
    "             input_source=get_speech_source(output_file))\n",
    "chatbot.start_bot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}