5_Generative_Models_I_Class_Generators.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Worksheet 5 Generative models I \n",
    "# author: Michael Galarnyk test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "\n",
    "class multiNB:\n",
    "    \n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def get_data(self, data_filename):\n",
    "        with open(data_filename) as f:\n",
    "            data_dic = {}\n",
    "            increment = 0\n",
    "            for line in f:\n",
    "                doc_word_count_gen = (int(i) for i in line.split()) # generator comprehension\n",
    "                doc_id = next(doc_word_count_gen)\n",
    "                word_id = next(doc_word_count_gen)\n",
    "                count = next(doc_word_count_gen)\n",
    "                data_dic[increment] =  [doc_id, word_id, count] \n",
    "                increment +=1\n",
    "        return data_dic\n",
    "    \n",
    "    def read_labels(self,label_filename):\n",
    "        with open(label_filename) as f:\n",
    "            label_list = [0]\n",
    "            classify = {i:0 for i in xrange(1,21)}\n",
    "            for line in f:\n",
    "                label_list.append(int(line))\n",
    "                classify[int(line)] = classify.get(int(line),0) + 1\n",
    "            pi = {i:classify[i] / float(len(label_list)) for i in xrange(1,21)} #dictionary for Pi\n",
    "            label_array = np.array(label_list)\n",
    "        return label_array, pi\n",
    "\n",
    "# stop words list from sklearn \n",
    "# https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/feature_extraction/stop_words.py\n",
    "    def setup_multinomial_model(self, label, data):\n",
    "        'data form is: doc_id  word_id  count'\n",
    "        m = np.zeros((21, 61189)) \n",
    "        len_data = len(data)\n",
    "        for i in xrange(len_data):\n",
    "            data_iter = iter(data[i])\n",
    "            doc_id = next(data_iter)\n",
    "            word_id = next(data_iter)\n",
    "            count = next(data_iter)\n",
    "            classify = label[doc_id]\n",
    "            m[classify][word_id] += count\n",
    "        # Remove stop words\n",
    "        stop_word = {12:\"of\", 23:\"and\",139:\"an\",978:\"am\",297:\"at\",51:\"but\",52:\"with\",33:\"to\",48:\"on\",27:\"are\",29:\"the\",72:\"can\",1367:\"else\",81:\"for\",301:\"he\",389:\"she\",99:\"so\"}\n",
    "        for k in stop_word:\n",
    "            m[:, k] = 0.0\n",
    "        m += 1\n",
    "        m[:,0] = 0.0\n",
    "        s = np.sum(m, axis = 1)\n",
    "        s_trans = np.transpose([s])\n",
    "        m = m / s_trans\n",
    "        m[:,0] = 1.0\n",
    "        m = np.log2(m)\n",
    "        self.m = m;\n",
    "        return m\n",
    "    \n",
    "    def naive_bayes(self, pi, test_data, test_label):\n",
    "        len_test_data = len(test_data)\n",
    "        number_doc_plus_1 = len(test_label)\n",
    "        test_m = np.zeros((number_doc_plus_1, 61189))\n",
    "        for i in range(len_test_data):\n",
    "            data_iter = iter(test_data[i])\n",
    "            doc_id = next(data_iter)\n",
    "            word_id = next(data_iter)\n",
    "            count = next(data_iter)\n",
    "            test_m[doc_id][word_id] += count\n",
    "        test_m = np.log2(1+test_m)\n",
    "        error = 0\n",
    "        for i in xrange(1, number_doc_plus_1):\n",
    "            cur_doc = test_m[i]\n",
    "            cur_s = np.sum(cur_doc * self.m, axis = 1)\n",
    "            final = cur_s + ([0] + pi.values())\n",
    "            final = final[1:]\n",
    "            label = np.argmax(final) + 1\n",
    "            if label != test_label[i]:\n",
    "                error += 1\n",
    "        return error * 100.0 / (number_doc_plus_1 - 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error 20.5329780147\n"
     ]
    }
   ],
   "source": [
    "NB = multiNB()\n",
    "data_dic = NB.get_data('20news-bydate/matlab/train.data')\n",
    "label_array, pi = NB.read_labels('20news-bydate/matlab/train.label')\n",
    "m = NB.setup_multinomial_model(label_array, data_dic)\n",
    "test_data = NB.get_data('20news-bydate/matlab/test.data')\n",
    "test_label, _ = NB.read_labels('20news-bydate/matlab/test.label')\n",
    "\n",
    "err = NB.naive_bayes(pi, test_data, test_label)\n",
    "print \"Error\", err"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}