spaCy/guide/Linguistic_Features.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "spaCy - Linguistic Features.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyOUZ0Hr7dBqQUXis9cUBOHN",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sourcecode369/deep-natural-language-processing/blob/master/spaCy/guide/Linguistic_Features.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "83AP46E7HwRf",
        "colab_type": "text"
      },
      "source": [
        "### Part - of - speech tagging"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H9Bf0JyeEwVC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "doc = nlp(\"Hello my name is sourcecode369 and i live in delhi, India\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FwvQzIyBN3Eg",
        "colab_type": "code",
        "outputId": "dadaccdc-4ffb-488e-87be-655684e6ba05",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "for token in doc:\n",
        "  print(token.text, token.lemma_, token.pos_, token.dep_, token.shape_, token.is_alpha, token.is_stop)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Hello hello INTJ intj Xxxxx True False\n",
            "my -PRON- DET poss xx True True\n",
            "name name NOUN nsubj xxxx True True\n",
            "is be AUX ROOT xx True True\n",
            "sourcecode369 sourcecode369 PROPN attr xxxxddd False False\n",
            "and and CCONJ cc xxx True True\n",
            "i i PRON nsubj x True True\n",
            "live live VERB ROOT xxxx True False\n",
            "in in ADP prep xx True True\n",
            "delhi delhi PROPN pobj xxxx True False\n",
            ", , PUNCT punct , False False\n",
            "India India PROPN appos Xxxxx True False\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C_F4kgroOLnk",
        "colab_type": "text"
      },
      "source": [
        "### Dependency Parsing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u2JP1IwZPmnz",
        "colab_type": "text"
      },
      "source": [
        "#### Noun Chunks"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sVF3sKRPQO6z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# ext: The original noun chunk text.\n",
        "# Root text: The original text of the word connecting the noun chunk to the rest of the parse.\n",
        "# Root dep: Dependency relation connecting the root to its head.\n",
        "# Root head text: The text of the root token’s head."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MrPSDoYdPoom",
        "colab_type": "code",
        "outputId": "6265960b-601c-4770-ffe9-ce955003839a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "for chunk in doc.noun_chunks:\n",
        "  print(chunk.text, chunk.root.text, chunk.root.dep_, chunk.root.head.text)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "my name name nsubj is\n",
            "sourcecode369 sourcecode369 attr is\n",
            "i i nsubj live\n",
            "delhi delhi pobj in\n",
            "India India appos delhi\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SVPhlcMsQCZU",
        "colab_type": "text"
      },
      "source": [
        "#### Navigataing the parse tree"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7FJEoE43Q_Xr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Text: The original token text.\n",
        "# Dep: The syntactic relation connecting child to head.\n",
        "# Head text: The original text of the token head.\n",
        "# Head POS: The part-of-speech tag of the token head.\n",
        "# Children: The immediate syntactic dependents of the token."
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lh1YHzVTQXGU",
        "colab_type": "code",
        "outputId": "40d9785d-7d17-4def-a43e-c25c533d2f51",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "for token in doc:\n",
        "  print(token.text, token.dep_, token.head.text, token.head.pos_, [child for child in token.children])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Hello intj is AUX []\n",
            "my poss name NOUN []\n",
            "name nsubj is AUX [my]\n",
            "is ROOT is AUX [Hello, name, sourcecode369]\n",
            "sourcecode369 attr is AUX []\n",
            "and cc live VERB []\n",
            "i nsubj live VERB []\n",
            "live ROOT live VERB [and, i, in]\n",
            "in prep live VERB [delhi]\n",
            "delhi pobj in ADP [,, India]\n",
            ", punct delhi PROPN []\n",
            "India appos delhi PROPN []\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3-WrT7RBQ1qM",
        "colab_type": "code",
        "outputId": "761bc141-da19-4de2-c90e-d5390ee00af1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from spacy.symbols import nsubj, VERB\n",
        "\n",
        "verbs = set()\n",
        "for possible_subject in doc:\n",
        "  if possible_subject.dep == nsubj and possible_subject.head.pos==VERB:\n",
        "    verbs.add(possible_subject.head)\n",
        "print(verbs)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{live}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eOJ-IQewSmID",
        "colab_type": "text"
      },
      "source": [
        "#### Iterating around the local tree"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4z-w8uwoTnMT",
        "colab_type": "code",
        "outputId": "d847d023-f910-4fd5-8e9d-32a9f40c0688",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print([token.text for token in doc[2].lefts])\n",
        "print([token.text for token in doc[2].rights])\n",
        "print(doc[2].n_lefts)\n",
        "print(doc[2].n_rights)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['my']\n",
            "[]\n",
            "1\n",
            "0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UTYItgodUEMc",
        "colab_type": "code",
        "outputId": "aae38938-d12c-4bfd-ae70-dfca30730806",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "### You can get a whole phrase by its syntactic head using the Token.subtree attribute. \n",
        "### This returns an ordered sequence of tokens. \n",
        "### You can walk up the tree with the Token.ancestors attribute, \n",
        "### and check dominance with Token.is_ancestor\n",
        "\n",
        "\n",
        "doc = nlp(\"Credit and mortgage account holders must submit their requests\")\n",
        "root = [token for token in doc if token.head == token][0]\n",
        "subject = list(root.lefts)[0]\n",
        "for descendant in subject.subtree:\n",
        "  assert subject is descendant or subject.is_ancestor(descendant)\n",
        "  print(descendant.text, descendant.dep_, descendant.n_lefts,\n",
        "            descendant.n_rights,\n",
        "            [ancestor.text for ancestor in descendant.ancestors])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Credit nmod 0 2 ['account', 'holders', 'submit']\n",
            "and cc 0 0 ['Credit', 'account', 'holders', 'submit']\n",
            "mortgage conj 0 0 ['Credit', 'account', 'holders', 'submit']\n",
            "account compound 1 0 ['holders', 'submit']\n",
            "holders nsubj 1 0 ['submit']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j89ncMByVy9G",
        "colab_type": "code",
        "outputId": "8081dc12-8013-498a-be04-99e482eb2bd9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# Finally, the .left_edge and .right_edge attributes can be especially useful, \n",
        "# because they give you the first and last token of the subtree. \n",
        "# This is the easiest way to create a Span object for a syntactic phrase. \n",
        "# Note that .right_edge gives a token within the subtree — \n",
        "# so if you use it as the end-point of a range, don’t forget to +1!\n",
        "\n",
        "span = doc[doc[4].left_edge.i:doc[4].right_edge.i+1]\n",
        "with doc.retokenize() as retokenizer:\n",
        "  retokenizer.merge(span)\n",
        "\n",
        "for token in doc:\n",
        "  print(token.text, token.pos_, token.dep_, token.head.text)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Credit and mortgage account holders NOUN nsubj submit\n",
            "must VERB aux submit\n",
            "submit VERB ROOT submit\n",
            "their DET poss requests\n",
            "requests NOUN dobj submit\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OJgQPGKzVzn1",
        "colab_type": "text"
      },
      "source": [
        "#### Vizualizing dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3gqJvAewceVq",
        "colab_type": "code",
        "outputId": "3ade5ae0-e9be-46db-c968-a4a4547305cd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 158
        }
      },
      "source": [
        "displacy.render(doc, style='dep',jupyter=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-11-46a0e103ea77>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mdisplacy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdoc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstyle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'dep'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mjupyter\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'displacy' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yHDXGN5FciTj",
        "colab_type": "text"
      },
      "source": [
        "#### disabling the parser"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5hgpeuYWdvrS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from spacy.lang.en import English"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "prNEr5fAd2K7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "nlp = spacy.load('en_core_web_sm', disable=['parser'])\n",
        "doc = nlp('I dont want parser.', disable=['parser'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "StlD4JgPeGZ_",
        "colab_type": "text"
      },
      "source": [
        "### Named entity Recognition"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mYd848GpeVAa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "nlp = spacy.load('en_core_web_sm')\n",
        "doc = nlp(\"Google is buying DeepMind startup for $500 billion.\")\n",
        "\n",
        "for ent in doc.ents:\n",
        "  print(ent.text, ent.start_char, ent.end_char, ent.label_)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H901kE2jfRVP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "doc = nlp(\"San Francisco considers banning sidewalk delivery robots\")\n",
        "ents = [(e.text, e.start_char, e.end_char, e.label_) for e in doc.ents]\n",
        "print(ents)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dV10Hsymf6yT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# I – Token is inside an entity.\n",
        "# O – Token is outside an entity.\n",
        "# B – Token is the beginning of an entity.\n",
        "\n",
        "ent_san = [doc[0].text, doc[0].ent_iob_, doc[0].ent_type_]\n",
        "ent_francisco = [doc[1].text, doc[1].ent_iob_, doc[1].ent_type_]\n",
        "print(ent_san)  # ['San', 'B', 'GPE']\n",
        "print(ent_francisco)  # ['Francisco', 'I', 'GPE']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tqPFj92PgJu7",
        "colab_type": "text"
      },
      "source": [
        "#### Setting entity annotations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rkulpPvjhUiD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "doc = nlp(\"fb is hiring a new vice president of global policy\")\n",
        "[(ent.text, ent.start_char, ent.end_char, ent.label_) for ent in doc.ents]\n",
        "from spacy.tokens import Span\n",
        "fb_ent = Span(doc, 0, 1, \"ORG\")\n",
        "doc.ents = list(doc.ents) + [fb_ent]\n",
        "[(ent.text, ent.start_char, ent.end_char, ent.label_) for ent in doc.ents]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Fh6y117jJ2c",
        "colab_type": "text"
      },
      "source": [
        "#### Setting entity annotations from array"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ktoEJ6tojSA7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy \n",
        "import spacy\n",
        "from spacy.attrs import ENT_IOB, ENT_TYPE"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M_KKsjjbDuzx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "nlp = spacy.load('en_core_web_sm')\n",
        "doc = nlp.make_doc(\"London is a big city in United Kindom.\")\n",
        "print(\"Before: \",doc.ents)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w8gAFeosD9gQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "header = [ENT_IOB, ENT_TYPE]\n",
        "attr_array = numpy.zeros((len(doc), len(header)))\n",
        "attr_array[0,0]  = 3\n",
        "attr_array[0,1] = doc.vocab.strings[\"GPE\"]\n",
        "doc.from_array(header, attr_array)\n",
        "print(\"After:\", doc.ents)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2mO1uX3cEo84",
        "colab_type": "text"
      },
      "source": [
        "#### Vizualizing Entities"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "No4LMJhhVCam",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "from spacy import displacy\n",
        "\n",
        "text = \"When Sebastian Thrun started working on self-driving cars at Google in 2007, few people outside of the company took him seriously.\"\n",
        "\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "doc = nlp(text)\n",
        "displacy.render(doc, style='ent',jupyter=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eOX6z20JVUfH",
        "colab_type": "text"
      },
      "source": [
        "### Entity Linking"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_vj_g8hOV0td",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "doc = nlp(\"Ada Lovelace was born in London\")\n",
        "\n",
        "\n",
        "ents = [(e.text, e.label_, e.kb_id_) for e in doc.ents]\n",
        "print(ents)  \n",
        "\n",
        "ent_ada_0 = [doc[0].text, doc[0].ent_type_, doc[0].ent_kb_id_]\n",
        "ent_ada_1 = [doc[1].text, doc[1].ent_type_, doc[1].ent_kb_id_]\n",
        "ent_london_5 = [doc[5].text, doc[5].ent_type_, doc[5].ent_kb_id_]\n",
        "print(ent_ada_0)  \n",
        "print(ent_ada_1)  \n",
        "print(ent_london_5)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DEix7ajRWPJu",
        "colab_type": "text"
      },
      "source": [
        "### Tokenization\n",
        "\n",
        "![tokenizer](https://spacy.io/language_data-ef63e6a58b7ec47c073fb59857a76e5f.svg)\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qeso-KtmWvN2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "doc = nlp(\"Apple is looking at buying U.K. startup for $1 billion\")\n",
        "for token in doc:\n",
        "    print(token.text)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YG6CgmORWzoX",
        "colab_type": "text"
      },
      "source": [
        "#### Add Special Case Tokenization Rules "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mIl-QKh_YR6s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "from spacy.symbols import ORTH\n",
        "\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "doc = nlp(\"gimme that\")  \n",
        "print([w.text for w in doc])  \n",
        "\n",
        "special_case = [{ORTH: \"gim\"}, {ORTH: \"me\"}]\n",
        "nlp.tokenizer.add_special_case(\"gimme\", special_case)\n",
        "\n",
        "print([w.text for w in nlp(\"gimme that\")])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uoxOd5f9YaGl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from spacy.lang.en import English\n",
        "\n",
        "nlp = English()\n",
        "text = '''\"Let's go!\"'''\n",
        "doc = nlp(text)\n",
        "tok_exp = nlp.tokenizer.explain(text)\n",
        "assert [t.text for t in doc if not t.is_space] == [t[1] for t in tok_exp]\n",
        "for t in tok_exp:\n",
        "    print(t[1], \"\\t\", t[0])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0Rz_MMmFkAho",
        "colab_type": "text"
      },
      "source": [
        "#### Customizing spaCy's Tokenizer class"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kWqvAcKN_gfT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import re\n",
        "import spacy\n",
        "from spacy.tokenizer import Tokenizer\n",
        "\n",
        "special_cases = {\":)\": [{\"ORTH\": \":)\"}]}\n",
        "prefix_re = re.compile(r'''^[[(\"']''')\n",
        "suffix_re = re.compile(r'''[])\"']$''')\n",
        "infix_re = re.compile(r'''[-~]''')\n",
        "simple_url_re = re.compile(r'''^https?://''')\n",
        "\n",
        "def custom_tokenizer(nlp):\n",
        "    return Tokenizer(nlp.vocab, rules=special_cases,\n",
        "                                prefix_search=prefix_re.search,\n",
        "                                suffix_search=suffix_re.search,\n",
        "                                infix_finditer=infix_re.finditer,\n",
        "                                token_match=simple_url_re.match)\n",
        "\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "nlp.tokenizer = custom_tokenizer(nlp)\n",
        "doc = nlp(\"hello-world. :)\")\n",
        "print([t.text for t in doc]) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ra4P5iIC_g4E",
        "colab_type": "text"
      },
      "source": [
        "#### Modifying existing rule sets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TsbI5Zo6CC3I",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "from spacy.lang.char_classes import ALPHA, ALPHA_LOWER, ALPHA_UPPER\n",
        "from spacy.lang.char_classes import CONCAT_QUOTES, LIST_ELLIPSES, LIST_ICONS\n",
        "from spacy.util import compile_infix_regex\n",
        "\n",
        "# default tokenizer\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "doc = nlp(\"mother-in-law\")\n",
        "print([t.text for t in doc]) # ['mother', '-', 'in', '-', 'law']\n",
        "\n",
        "# modify tokenizer infix patterns\n",
        "infixes = (\n",
        "    LIST_ELLIPSES\n",
        "    + LIST_ICONS\n",
        "    + [\n",
        "        r\"(?<=[0-9])[+\\-\\*^](?=[0-9-])\",\n",
        "        r\"(?<=[{al}{q}])\\.(?=[{au}{q}])\".format(\n",
        "            al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES\n",
        "        ),\n",
        "        r\"(?<=[{a}]),(?=[{a}])\".format(a=ALPHA),\n",
        "        # EDIT: commented out regex that splits on hyphens between letters:\n",
        "        #r\"(?<=[{a}])(?:{h})(?=[{a}])\".format(a=ALPHA, h=HYPHENS),\n",
        "        r\"(?<=[{a}0-9])[:<>=/](?=[{a}])\".format(a=ALPHA),\n",
        "    ]\n",
        ")\n",
        "\n",
        "infix_re = compile_infix_regex(infixes)\n",
        "nlp.tokenizer.infix_finditer = infix_re.finditer\n",
        "doc = nlp(\"mother-in-law\")\n",
        "print([t.text for t in doc]) # ['mother-in-law']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j5sJU4IxCGX6",
        "colab_type": "text"
      },
      "source": [
        "#### Hooking an arbitrary tokenizer into the pipeline"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CxVSH9b7CQr6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "from spacy.tokens import Doc\n",
        "\n",
        "class WhitespaceTokenizer(object):\n",
        "    def __init__(self, vocab):\n",
        "        self.vocab = vocab\n",
        "\n",
        "    def __call__(self, text):\n",
        "        words = text.split(' ')\n",
        "        spaces = [True] * len(words)\n",
        "        return Doc(self.vocab, words=words, spaces=spaces)\n",
        "\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "nlp.tokenizer = WhitespaceTokenizer(nlp.vocab)\n",
        "doc = nlp(\"What's happened to me? he thought. It wasn't a dream.\")\n",
        "print([t.text for t in doc])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yu13mcgVCWfb",
        "colab_type": "text"
      },
      "source": [
        "#### Bringing own annotations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KH_v4F4tCeYa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "from spacy.tokens import Doc\n",
        "from spacy.lang.en import English\n",
        "\n",
        "nlp = English()\n",
        "doc = Doc(nlp.vocab, words=[\"Hello\", \",\", \"world\", \"!\"],\n",
        "          spaces=[False, True, False, False])\n",
        "print([(t.text, t.text_with_ws, t.whitespace_) for t in doc])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qILNrI-kC3az",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "from spacy.tokens import Doc\n",
        "from spacy.lang.en import English\n",
        "\n",
        "nlp = English()\n",
        "bad_spaces = Doc(nlp.vocab, words=[\"Hello\", \",\", \"world\", \"!\"])\n",
        "good_spaces = Doc(nlp.vocab, words=[\"Hello\", \",\", \"world\", \"!\"],\n",
        "                  spaces=[False, True, False, False])\n",
        "\n",
        "print(bad_spaces.text)   # 'Hello , world !'\n",
        "print(good_spaces.text)  # 'Hello, world!'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f0FPvtb_DaaQ",
        "colab_type": "text"
      },
      "source": [
        "#### Aligning Tokenization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cnzbnmSgC6iq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from spacy.gold import align\n",
        "\n",
        "other_tokens = [\"i\", \"listened\", \"to\", \"obama\", \"'\", \"s\", \"podcasts\", \".\"]\n",
        "spacy_tokens = [\"i\", \"listened\", \"to\", \"obama\", \"'s\", \"podcasts\", \".\"]\n",
        "cost, a2b, b2a, a2b_multi, b2a_multi = align(other_tokens, spacy_tokens)\n",
        "print(\"Misaligned tokens:\", cost)  # 2\n",
        "print(\"One-to-one mappings a -> b\", a2b)  # array([0, 1, 2, 3, -1, -1, 5, 6])\n",
        "print(\"One-to-one mappings b -> a\", b2a)  # array([0, 1, 2, 3, 5, 6, 7])\n",
        "print(\"Many-to-one mappings a -> b\", a2b_multi)  # {4: 4, 5: 4}\n",
        "print(\"Many-to-one mappings b-> a\", b2a_multi)  # {}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7_VWLEFoDT9C",
        "colab_type": "text"
      },
      "source": [
        "### Merging and Splitting"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uXJCYjbCEB_b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "doc = nlp(\"I live in New York\")\n",
        "print(\"Before:\", [token.text for token in doc])\n",
        "\n",
        "with doc.retokenize() as retokenizer:\n",
        "    retokenizer.merge(doc[3:5], attrs={\"LEMMA\": \"new york\"})\n",
        "print(\"After:\", [token.text for token in doc])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ReykpjQp8LRj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "from spacy import displacy\n",
        "\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "doc = nlp(\"I live in NewYork\")\n",
        "print(\"Before:\", [token.text for token in doc])\n",
        "displacy.render(doc, jupyter=True)  # displacy.serve if you're not in a Jupyter environment\n",
        "\n",
        "with doc.retokenize() as retokenizer:\n",
        "    heads = [(doc[3], 1), doc[2]]\n",
        "    attrs = {\"POS\": [\"PROPN\", \"PROPN\"], \"DEP\": [\"pobj\", \"compound\"]}\n",
        "    retokenizer.split(doc[3], [\"New\", \"York\"], heads=heads, attrs=attrs)\n",
        "print(\"After:\", [token.text for token in doc])\n",
        "displacy.render(doc, jupyter=True) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hpT0qs_G9keo",
        "colab_type": "text"
      },
      "source": [
        "#### Overwriting custom extension attribute"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bhPHIBuj81uK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "from spacy.tokens import Token\n",
        "\n",
        "# Register a custom token attribute, token._.is_musician\n",
        "Token.set_extension(\"is_musician\", default=False)\n",
        "\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "doc = nlp(\"I like David Bowie\")\n",
        "print(\"Before:\", [(token.text, token._.is_musician) for token in doc])\n",
        "\n",
        "with doc.retokenize() as retokenizer:\n",
        "    retokenizer.merge(doc[2:4], attrs={\"_\": {\"is_musician\": True}})\n",
        "print(\"After:\", [(token.text, token._.is_musician) for token in doc])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XTQjkkFb9yKJ",
        "colab_type": "text"
      },
      "source": [
        "### Sentence Segmentation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hlrV0kkx92Fh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}