prob_calc.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import pprint\n",
    "import numpy as np\n",
    "# Can also use NLTK.FreqDist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9114687\n"
     ]
    }
   ],
   "source": [
    "f = open(f\"./train_processed.txt\", \"r\")\n",
    "word_tag_pairs = f.readlines()\n",
    "f.close()\n",
    "print(len(word_tag_pairs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "by_PRP\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(word_tag_pairs[4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_tag_pairs = [word_tag_pair[:-1] for word_tag_pair in word_tag_pairs]\n",
    "all_words = [pair.split('_')[0] for pair in word_tag_pairs]\n",
    "word_freq = Counter(all_words).most_common() # represent how many times each word appeared in dataset\n",
    "train_words = [word[0] for word in word_freq] # all the words in the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "by_PRP\n",
      "by\n",
      "(\"'s\", 69743)\n",
      "'s\n",
      "No. of unique words in train dataset = 192526\n"
     ]
    }
   ],
   "source": [
    "print(word_tag_pairs[4])\n",
    "print(all_words[4])\n",
    "print(word_freq[10])\n",
    "print(train_words[10])\n",
    "print(f\"No. of unique words in train dataset = {len(train_words)}\")\n",
    "with open(\"./train_words.txt\", \"w+\") as f:\n",
    "    for word in train_words:\n",
    "        f.write(f\"{word}\\n\")\n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('The_AT0', 62218)\n",
      "No. of unique word tag pairs = 255396\n"
     ]
    }
   ],
   "source": [
    "unique_word_tag_pairs = Counter(word_tag_pairs).most_common()\n",
    "print(unique_word_tag_pairs[10])\n",
    "print(f\"No. of unique word tag pairs = {len(unique_word_tag_pairs)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "freq_of_tags_for_each_word = {} # Represents the freq of each tag used with the particular word for all words \n",
    "for word_tag_pair in unique_word_tag_pairs:\n",
    "    #print(word)\n",
    "    key = word_tag_pair[0].split('_')[0]\n",
    "    if key not in freq_of_tags_for_each_word:\n",
    "        freq_of_tags_for_each_word[key] = []\n",
    "    freq_of_tags_for_each_word[key] += [word_tag_pair]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word to be checked: of\n",
      "[('the_AT0', 525421)]\n"
     ]
    }
   ],
   "source": [
    "print(f\"Word to be checked: {train_words[1]}\")\n",
    "print(freq_of_tags_for_each_word[train_words[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#np_freq_of_tags_for_each_word = np.array(freq_of_tags_for_each_word)\n",
    "np.save(\"model.npy\", freq_of_tags_for_each_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "loaded_freq_of_tags_for_each_word = np.load(\"model.npy\").item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word to be checked: in\n",
      "[('in_PRP', 157584), ('in_PRP-AVP', 9277), ('in_AVP', 3261), ('in_AVP-PRP', 1156), ('in_UNC', 62), ('in_VBZ', 4), ('in_AV0', 2)]\n"
     ]
    }
   ],
   "source": [
    "# Prediction\n",
    "print(f\"Word to be checked: in\")\n",
    "print(loaded_freq_of_tags_for_each_word.get(\"in\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}