nlp-reference-notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# some of the common NLP tasks reference notebook\n",
    "__(using NLTK)__\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text transformation tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"sample_text.txt\") as file:\n",
    "    raw_text = file.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text:\n",
      " hello world! Its Aman Singh from India. This is a reference notebook for some of the common Natural Language Processing (NLP) tasks.\n",
      "\n",
      "tokenized words:\n",
      "['hello', 'world', '!', 'Its', 'Aman', 'Singh', 'from', 'India', '.', 'This', 'is', 'a', 'reference', 'notebook', 'for', 'some', 'of', 'the', 'common', 'Natural', 'Language', 'Processing', '(', 'NLP', ')', 'tasks', '.']\n",
      "\n",
      "tokenized sentences:\n",
      "['hello world!', 'Its Aman Singh from India.', 'This is a reference notebook for some of the common Natural Language Processing (NLP) tasks.']\n",
      "\n",
      "French Text:\n",
      " Ceci est 1 première phrase. Puis j'en écris une seconde. pour finir en voilà une troisième sans mettre de majuscule\n",
      "\n",
      "tokenized french sentences:\n",
      "['Ceci est 1 première phrase.', \"Puis j'en écris une seconde.\", 'pour finir en voilà une troisième sans mettre de majuscule']\n",
      "\n",
      "tokenized french words:\n",
      "['Ceci', 'est', '1', 'première', 'phrase', '.', 'Puis', \"j'en\", 'écris', 'une', 'seconde', '.', 'pour', 'finir', 'en', 'voilà', 'une', 'troisième', 'sans', 'mettre', 'de', 'majuscule']\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import word_tokenize, sent_tokenize\n",
    "\n",
    "some_text = \"hello world! Its Aman Singh from India. This is a reference notebook for some of the common Natural Language Processing (NLP) tasks.\"\n",
    "# word tokenization\n",
    "tokenized_words = word_tokenize(some_text)\n",
    "print(\"Text:\\n\", some_text)\n",
    "print(\"\\ntokenized words:\")\n",
    "print(tokenized_words)\n",
    "\n",
    "# sentence tokenization\n",
    "tokenized_sents = sent_tokenize(some_text)\n",
    "print(\"\\ntokenized sentences:\")\n",
    "print(tokenized_sents)\n",
    "\n",
    "# sentence/word tokenization with other laguages\n",
    "french_text = \"Ceci est 1 première phrase. Puis j'en écris une seconde. pour finir en voilà une troisième sans mettre de majuscule\"\n",
    "tokenized_sents = sent_tokenize(french_text, language = 'french')\n",
    "print(\"\\nFrench Text:\\n\", french_text)\n",
    "print(\"\\ntokenized french sentences:\")\n",
    "print(tokenized_sents)\n",
    "\n",
    "tokenized_words = word_tokenize(french_text, language = 'french')\n",
    "print(\"\\ntokenized french words:\")\n",
    "print(tokenized_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SYNONYMS:\n",
      " ['large', 'big', 'big', 'bad', 'big', 'big', 'big', 'large', 'prominent', 'big', 'heavy', 'boastful', 'braggart', 'bragging', 'braggy', 'big', 'cock-a-hoop', 'crowing', 'self-aggrandizing', 'self-aggrandising', 'big', 'swelled', 'vainglorious', 'adult', 'big', 'full-grown', 'fully_grown', 'grown', 'grownup', 'big', 'big', 'large', 'magnanimous', 'big', 'bighearted', 'bounteous', 'bountiful', 'freehanded', 'handsome', 'giving', 'liberal', 'openhanded', 'big', 'enceinte', 'expectant', 'gravid', 'great', 'large', 'heavy', 'with_child', 'big', 'boastfully', 'vauntingly', 'big', 'large', 'big', 'big']\n",
      "\n",
      "ANTONYMS:\n",
      " ['small', 'little', 'small']\n"
     ]
    }
   ],
   "source": [
    "# nltk's wordnet package includes groups of synonyms, antonyms and also a brief definition for each\n",
    "# here only examplifying synonyms and antonyms\n",
    "from nltk.corpus import wordnet\n",
    "\n",
    "synonyms, antonyms = [], []\n",
    "for syn in wordnet.synsets(\"big\"):\n",
    "    for lemma in syn.lemmas():\n",
    "        synonyms.append(lemma.name())\n",
    "        if lemma.antonyms():\n",
    "            antonyms.append(lemma.antonyms()[0].name())\n",
    "print(\"SYNONYMS:\\n\",synonyms)\n",
    "print(\"\\nANTONYMS:\\n\",antonyms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "stemming:\n",
      "increas\n",
      "\n",
      "lemmatization:\n",
      "increase\n"
     ]
    }
   ],
   "source": [
    "# stemming vs lemmatization\n",
    "from nltk.stem import PorterStemmer, WordNetLemmatizer\n",
    "\n",
    "# stemming\n",
    "stemmer = PorterStemmer()\n",
    "print(\"stemming:\")\n",
    "print(stemmer.stem(\"increases\"))\n",
    "\n",
    "# lemmatization\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "print(\"\\nlemmatization:\")\n",
    "print(lemmatizer.lemmatize(\"increases\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Text:\n",
      " In the fields of computational linguistics and probability, an n-gram is a contiguous sequence of n items from a given sample of text or speech.\n",
      "\n",
      "bi-grams:\n",
      " [('In', 'the'), ('the', 'fields'), ('fields', 'of'), ('of', 'computational'), ('computational', 'linguistics'), ('linguistics', 'and'), ('and', 'probability'), ('probability', ','), (',', 'an'), ('an', 'n-gram'), ('n-gram', 'is'), ('is', 'a'), ('a', 'contiguous'), ('contiguous', 'sequence'), ('sequence', 'of'), ('of', 'n'), ('n', 'items'), ('items', 'from'), ('from', 'a'), ('a', 'given'), ('given', 'sample'), ('sample', 'of'), ('of', 'text'), ('text', 'or'), ('or', 'speech'), ('speech', '.')]\n",
      "\n",
      "tri-grams:\n",
      " [('In', 'the', 'fields'), ('the', 'fields', 'of'), ('fields', 'of', 'computational'), ('of', 'computational', 'linguistics'), ('computational', 'linguistics', 'and'), ('linguistics', 'and', 'probability'), ('and', 'probability', ','), ('probability', ',', 'an'), (',', 'an', 'n-gram'), ('an', 'n-gram', 'is'), ('n-gram', 'is', 'a'), ('is', 'a', 'contiguous'), ('a', 'contiguous', 'sequence'), ('contiguous', 'sequence', 'of'), ('sequence', 'of', 'n'), ('of', 'n', 'items'), ('n', 'items', 'from'), ('items', 'from', 'a'), ('from', 'a', 'given'), ('a', 'given', 'sample'), ('given', 'sample', 'of'), ('sample', 'of', 'text'), ('of', 'text', 'or'), ('text', 'or', 'speech'), ('or', 'speech', '.')]\n"
     ]
    }
   ],
   "source": [
    "# generating n-grams from tokens\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.util import ngrams\n",
    "\n",
    "text = \"In the fields of computational linguistics and probability, an n-gram is a contiguous sequence of n items from a given sample of text or speech.\"\n",
    "word_tokens = word_tokenize(text)\n",
    "bigrams = ngrams(word_tokens, 2)\n",
    "trigrams = ngrams(word_tokens, 3)\n",
    "print(\"Text:\\n\", text)\n",
    "print(\"\\nbi-grams:\\n\", list(bigrams))\n",
    "print(\"\\ntri-grams:\\n\", list(trigrams))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Information extraction tasks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "POS Tags:\n",
      " [('I', 'PRP'), ('like', 'VBP'), ('to', 'TO'), ('go', 'VB'), ('to', 'TO'), ('the', 'DT'), ('park', 'NN'), ('with', 'IN'), ('my', 'PRP$'), ('dog', 'NN')]\n"
     ]
    }
   ],
   "source": [
    "# Part-Of-Speech (POS) tagging\n",
    "from nltk.tag import pos_tag\n",
    "\n",
    "text = \"I like to go to the park with my dog\"\n",
    "\n",
    "tokens = word_tokenize(text)\n",
    "tags = pos_tag(tokens)\n",
    "print(\"POS Tags:\\n\", tags)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### POS tags References:\n",
    "---------------------------------\n",
    "\n",
    "    CC  | Coordinating conjunction |\n",
    "    CD  | Cardinal number |\n",
    "    DT  | Determiner |\n",
    "    EX  | Existential there |\n",
    "    FW  | Foreign word |\n",
    "    IN  | Preposition or subordinating conjunction |\n",
    "    JJ  | Adjective |\n",
    "    JJR | Adjective, comparative |\n",
    "    JJS | Adjective, superlative |\n",
    "    LS  | List item marker |\n",
    "    MD  | Modal |\n",
    "    NN  | Noun, singular or mass |\n",
    "    NNS | Noun, plural |\n",
    "    NNP | Proper noun, singular |\n",
    "    NNPS| Proper noun, plural |\n",
    "    PDT | Predeterminer |\n",
    "    POS | Possessive ending |\n",
    "    PRP | Personal pronoun |\n",
    "    PRP$| Possessive pronoun |\n",
    "    RB  | Adverb |\n",
    "    RBR | Adverb, comparative |\n",
    "    RBS | Adverb, superlative |\n",
    "    RP  | Particle |\n",
    "    SYM | Symbol |\n",
    "    TO  | to |\n",
    "    UH  | Interjection |\n",
    "    VB  | Verb, base form |\n",
    "    VBD | Verb, past tense |\n",
    "    VBG | Verb, gerund or present participle |\n",
    "    VBN | Verb, past participle |\n",
    "    VBP | Verb, non-3rd person singular present |\n",
    "    VBZ | Verb, 3rd person singular present |\n",
    "    WDT | Wh-determiner |\n",
    "    WP  | Wh-pronoun |\n",
    "    WP$ | Possessive wh-pronoun |\n",
    "    WRB | Wh-adverb |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Named-Entity-Recognition:\n",
      " (S\n",
      "  (PERSON Mark/NNP)\n",
      "  (PERSON Elliot/NNP Zuckerberg/NNP)\n",
      "  (/(\n",
      "  born/VBN\n",
      "  May/NNP\n",
      "  14/CD\n",
      "  ,/,\n",
      "  1984/CD\n",
      "  )/)\n",
      "  is/VBZ\n",
      "  a/DT\n",
      "  co-founder/NN\n",
      "  of/IN\n",
      "  (GPE Facebook/NNP)\n",
      "  ./.)\n"
     ]
    }
   ],
   "source": [
    "# Named-Entity-Recognition (NER)\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.tag import pos_tag\n",
    "from nltk.chunk import ne_chunk\n",
    "\n",
    "text = \"Mark Elliot Zuckerberg (born May 14, 1984) is a co-founder of Facebook.\"\n",
    "\n",
    "# tokenize sentence into words\n",
    "tokens = word_tokenize(text)\n",
    "\n",
    "# POS tagging of the tokens\n",
    "tags = pos_tag(tokens)\n",
    " \n",
    "# using the ner function\n",
    "ner = ne_chunk(tags)\n",
    "\n",
    "print(\"Named-Entity-Recognition:\\n\", ner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#todo: make an abstractive summarizer like smmry.com\n",
    "#PAPER: https://arxiv.org/pdf/1602.06023.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}