Word2vec.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Word2vec.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "e301fbbcddbc4c1d9295620e5bff45b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b57aa662139d4efcbc2f7e7a8857a483",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_affd720a78754ba5874b6b7b36414344",
              "IPY_MODEL_1f616f1b81cc447b8c39d48317f14852"
            ]
          }
        },
        "b57aa662139d4efcbc2f7e7a8857a483": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "affd720a78754ba5874b6b7b36414344": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_845058b7baba4706a163973585f348cc",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_601be8a1b65a42f584377bd7dec1ccb2"
          }
        },
        "1f616f1b81cc447b8c39d48317f14852": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_233d898d4eb44a8d88295a058ab34ec2",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1701/? [01:19&lt;00:00, 21.49it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a1aae7c823f042ceab8e9030ccfd4e4f"
          }
        },
        "845058b7baba4706a163973585f348cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "601be8a1b65a42f584377bd7dec1ccb2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "233d898d4eb44a8d88295a058ab34ec2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a1aae7c823f042ceab8e9030ccfd4e4f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fefb7410cb9d42f6b2e7175f047fae02": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_60203681914241159d9f211802bf4ca0",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_28c2b357616f4b528b46297b252a2e76",
              "IPY_MODEL_833251bdd02046ba8593f25f5d214a5f"
            ]
          }
        },
        "60203681914241159d9f211802bf4ca0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "28c2b357616f4b528b46297b252a2e76": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_74bac74892064d91a6aeb4bc65525919",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 10001,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10001,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_561496653b3248d28fef46009bd2bcd1"
          }
        },
        "833251bdd02046ba8593f25f5d214a5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ee6951d02d04410abc3c8fa3273fe254",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10001/10001 [15:27&lt;00:00, 10.78it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_117c67d957074b1ebc66c70f88e7cf24"
          }
        },
        "74bac74892064d91a6aeb4bc65525919": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "561496653b3248d28fef46009bd2bcd1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ee6951d02d04410abc3c8fa3273fe254": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "117c67d957074b1ebc66c70f88e7cf24": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0k3TWCdmLepH"
      },
      "source": [
        "This is a Word2Vec embedding model with skip-gram and negative sampling method using pure tensorflow 2."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7B3P803wJqos"
      },
      "source": [
        "# Making Data Ready"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0KXm_ruQHxds",
        "outputId": "421ae833-044a-43e9-ee97-3d6a47878944",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!pip install tqdm # This is for showing a smart progress meter in any loop"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (4.41.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2EIGNtWfmsMc",
        "outputId": "df86aed0-4235-4cdc-8a87-fbf10765b810",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from tqdm.auto import tqdm # This is for showing a smart progress meter in any loop\n",
        "import gensim.downloader as api\n",
        "corpus = api.load(\"text8\") # This is tokenized corpus of text from wikipedia"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[==================================================] 100.0% 31.6/31.6MB downloaded\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N11j9MjKmtr3",
        "outputId": "e63ae7cb-742f-4408-e24a-fab22f98960d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "# just to show what is inside the corpus\n",
        "for i in corpus:\n",
        "  print(i[:20])\n",
        "  break"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['anarchism', 'originated', 'as', 'a', 'term', 'of', 'abuse', 'first', 'used', 'against', 'early', 'working', 'class', 'radicals', 'including', 'the', 'diggers', 'of', 'the', 'english']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r16RnsXrK3p8"
      },
      "source": [
        "Now we want to build our skip-grams which means we should pair every word with its neighbours. For defining neighbour we choose a `window_size`.\n",
        "\n",
        "For example suppose the sentense \"*I want a new car*\". its skip-grams with `window_size = 1` would be:\n",
        "\n",
        "*   I , want\n",
        "*   want , I\n",
        "*   want , a  \n",
        "*   a , want\n",
        "*   a , new\n",
        "*   new , a\n",
        "*   new , car\n",
        "*   car , new\n",
        "\n",
        "And with `window_size = 2` words near `new` would be:\n",
        "\n",
        "*   new , car\n",
        "*   new , want\n",
        "*   new , a  \n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2lXK6xCqncVW"
      },
      "source": [
        "window_size = 2"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cuAxhxnK5nc_",
        "outputId": "4f1ceaf7-80da-4d43-fc99-bf53a6cb1305",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104,
          "referenced_widgets": [
            "e301fbbcddbc4c1d9295620e5bff45b0",
            "b57aa662139d4efcbc2f7e7a8857a483",
            "affd720a78754ba5874b6b7b36414344",
            "1f616f1b81cc447b8c39d48317f14852",
            "845058b7baba4706a163973585f348cc",
            "601be8a1b65a42f584377bd7dec1ccb2",
            "233d898d4eb44a8d88295a058ab34ec2",
            "a1aae7c823f042ceab8e9030ccfd4e4f"
          ]
        }
      },
      "source": [
        "# Now we should build our skip-grams and find neoghbour words, e.g. {('modern', 'era'), ('means', 'to'), ('economics', 'there'), ('while', 'is'),...}\n",
        "\n",
        "from collections import Counter\n",
        "neighbour_counter = Counter()\n",
        "\n",
        "for text in tqdm(corpus):\n",
        "  dump_txt = ['_TEMP_'] * window_size + text + ['_TEMP_'] * window_size # e.g. sentence \"I love code\" with window_size = 2 would be \"_TEMP_ _TEMP_ I love code _TEMP_ _TEMP_\"\n",
        "  for i in range(len(dump_txt) - (window_size + 1) * 2):\n",
        "    middle_window = dump_txt[window_size + i] \n",
        "    for j in range(window_size): # we choose words near the center word as its nighbours (context words)\n",
        "      location = window_size + i - 1 \n",
        "      if not middle_window == dump_txt[location - j] and not dump_txt[location - j] == '_TEMP_': # This is just for making sure neighbours aren't the same, like (the, the) and we ignore it when it becomes naighbour to \"_TEMP_\"\n",
        "        neighbour_counter[(middle_window, dump_txt[location - j])] += 1\n",
        "      if not middle_window == dump_txt[location + j] and not dump_txt[location + j] == '_TEMP_': # This is just for making sure neighbours aren't the same, like (the, the) and we ignore it when it becomes naighbour to \"_TEMP_\"\n",
        "        neighbour_counter[(middle_window, dump_txt[location + j])] += 1\n",
        "\n",
        "print(len(neighbour_counter))\n",
        "print(neighbour_counter.most_common(3))"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e301fbbcddbc4c1d9295620e5bff45b0",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "8551710\n",
            "[(('nine', 'one'), 358337), (('the', 'of'), 333414), (('the', 'in'), 197295)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yk8k5kOf8WtP",
        "outputId": "bc812ee1-231a-4293-da57-abac8b94828e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "neighbours = [i[0] for i in neighbour_counter.items() if i[1] > 5] # Removing neighbours that repeated less than 5 time in our large corpus. They are not useful.\n",
        "\n",
        "len(neighbours)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "897084"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Q81Z4-ulUNu",
        "outputId": "a7fee168-69f0-4b3f-8b04-9d13fdfbe49e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from collections import defaultdict\n",
        "\n",
        "dict_of_neighbours = defaultdict(set) # This dict contains any naighbour that a word has in our data\n",
        "for i, j in neighbours: \n",
        "    dict_of_neighbours[i].add(j) \n",
        "\n",
        "len(dict_of_neighbours['cat'])"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "48"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rGTkFrdnm0yf",
        "outputId": "c9886dc9-e464-4e76-f40b-c8dfcfabf1d8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "words_set = list(set([j for i in neighbours for j in (i[0], i[1])])) # list of all tokens\n",
        "\n",
        "print(len(words_set))\n",
        "\n",
        "index2word = dict(enumerate(list(words_set))) # A dict that maps an index to a word, e.g. d[3] = 'cat'\n",
        "word2index = {v: k for k, v in index2word.items()} # Reverse of the dict above. e.g. d['cat'] = 3"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "63258\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-ZGsCdscVuIE"
      },
      "source": [
        "Now we should make negative smapling. Which means we should produce fake nighbours. We then lable the true tuple with 1 and fake tuples with 0.\n",
        "\n",
        "For producing fake neighbours we just randomly sample from our `words_lst`. Do not worry about it randomly choose a true nighbour and not a fake one. We do not need to be perfect for now."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5mxpWqMCXmCS"
      },
      "source": [
        "negative_samples = 5"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oQgv-3ZjoSme"
      },
      "source": [
        "import random\n",
        "\n",
        "def negative_sampling(true_neighbours): # We lable true_neighbours with 1, and we produce fake neighbours and lable them with 0\n",
        "  center_word = true_neighbours[0]\n",
        "  neighbours_labeled = [(true_neighbours[1], 1)]\n",
        "\n",
        "  for random_word in random.sample(words_set, negative_samples):\n",
        "    neighbours_labeled.append((random_word , 0))\n",
        "\n",
        "  random.shuffle(neighbours_labeled) # Shuffle to make the order random\n",
        "  x, y = zip(*neighbours_labeled) # Now Xs are neighbour and Ys are thir lables (0 or 1)\n",
        "\n",
        "  return center_word, x, y"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ja1PuZWxkSRf",
        "outputId": "ff21591c-f0fe-4d40-b680-4ef70cea9943",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "negative_sampling((\"cool\", \"code\"))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('cool',\n",
              " ('code', 'baroque', 'incumbent', 'res', 'gamers', 'acted'),\n",
              " (1, 0, 0, 0, 0, 0))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x6HHhgUSbQeA"
      },
      "source": [
        "Now The Final step to have our data ready! (Yoo Hoo!).\n",
        "\n",
        "We should make a function that return a batch of data everytime we call it. we rather replacethe words with thir indices here (by using `word2index`)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YIgMBlkYac3s"
      },
      "source": [
        "def give_us_data(batch_size):\n",
        "  batched_center = []\n",
        "  batched_x = []\n",
        "  batched_y = []\n",
        "\n",
        "  for random_neighbour in random.sample(neighbours, batch_size):\n",
        "    center_word, x , y = negative_sampling(random_neighbour)\n",
        "    \n",
        "    batched_y.append(y)\n",
        "    batched_center.append(word2index[center_word]) # Convert the center word to its index (e.g. \"cat\" to 3)\n",
        "    batched_x.append([word2index[i] for i in x]) # Convert words to its index (e.g. \"cat\" to 3)\n",
        "\n",
        "  return batched_center, batched_x, batched_y"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UbaWcUj39TLD",
        "outputId": "ae439a82-ac61-475c-ea57-e4268bb2fc2a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "give_us_data(2)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "([1364, 55498],\n",
              " [[35557, 15774, 33734, 46291, 10516, 39514],\n",
              "  [54854, 4541, 53446, 3848, 19714, 30488]],\n",
              " [(0, 0, 1, 0, 0, 0), (0, 0, 1, 0, 0, 0)])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7ru-jZjURy9n"
      },
      "source": [
        "# MODEL\n",
        "\n",
        "Now is the time to code the model with pure TF 2"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pXwD9S2OrTEI"
      },
      "source": [
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "\n",
        "embeding_size = 50\n",
        "batch_size = 128"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kGRBpAK9rgaf"
      },
      "source": [
        "target_amb = tf.keras.layers.Embedding(len(words_set), embeding_size) # This will be our Word2Vec\n",
        "context_amb = tf.keras.layers.Embedding(len(words_set), embeding_size) # This is just for the learning phase\n",
        "\n",
        "optimizer = tf.keras.optimizers.Adam()"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W1mk45uH1GHb",
        "outputId": "63d5c872-7f65-4857-eed2-8c59b8d440d6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 449,
          "referenced_widgets": [
            "fefb7410cb9d42f6b2e7175f047fae02",
            "60203681914241159d9f211802bf4ca0",
            "28c2b357616f4b528b46297b252a2e76",
            "833251bdd02046ba8593f25f5d214a5f",
            "74bac74892064d91a6aeb4bc65525919",
            "561496653b3248d28fef46009bd2bcd1",
            "ee6951d02d04410abc3c8fa3273fe254",
            "117c67d957074b1ebc66c70f88e7cf24"
          ]
        }
      },
      "source": [
        "for _ in tqdm(range(10001)): # tqdm is just for a nice progress bar.\n",
        "\n",
        "  center, x, y = give_us_data(batch_size)\n",
        "  center = np.asarray(center)\n",
        "  x = np.asarray(x)\n",
        "  y = np.asarray(y)\n",
        "\n",
        "  with tf.GradientTape() as t:\n",
        "    center_embs = target_amb(center)\n",
        "    neighbor_choices = context_amb(x)\n",
        "\n",
        "    scores = tf.keras.backend.batch_dot(neighbor_choices, center_embs, axes=(2,1)) # dot prudoct vectors of naighbours together\n",
        "    prediction = tf.nn.sigmoid(scores) # we want the model to give us a probability of that two, being real naighbours\n",
        "\n",
        "    loss = tf.keras.losses.categorical_crossentropy(y, prediction)\n",
        "\n",
        "    if not _ % 2000:\n",
        "      print(\"batch:\", _ , \" - The mean loss is: \" ,tf.reduce_mean(loss).numpy())\n",
        "      print(y[1], prediction[1].numpy())\n",
        "      print(\"------------------------------------------------\")\n",
        "\n",
        "    g_embed, g_context = t.gradient(loss, [target_amb.embeddings, context_amb.embeddings])\n",
        "    optimizer.apply_gradients(zip([g_embed, g_context], [target_amb.embeddings, context_amb.embeddings]))"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fefb7410cb9d42f6b2e7175f047fae02",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=10001.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "batch: 0  - The mean loss is:  1.7918224\n",
            "[1 0 0 0 0 0] [0.49952218 0.49837837 0.502265   0.49912843 0.49756768 0.50099564]\n",
            "------------------------------------------------\n",
            "batch: 2000  - The mean loss is:  1.1438766\n",
            "[0 0 1 0 0 0] [0.03187576 0.01108352 0.98943454 0.05731562 0.05656805 0.00725603]\n",
            "------------------------------------------------\n",
            "batch: 4000  - The mean loss is:  0.83899343\n",
            "[0 0 0 0 1 0] [2.6515016e-01 7.0336723e-04 7.1048317e-04 2.0742374e-04 9.7770542e-01\n",
            " 1.4255049e-03]\n",
            "------------------------------------------------\n",
            "batch: 6000  - The mean loss is:  0.66536206\n",
            "[0 0 0 1 0 0] [5.5141826e-03 2.3234300e-03 9.1780163e-04 9.8258972e-01 2.5561482e-03\n",
            " 3.7092983e-03]\n",
            "------------------------------------------------\n",
            "batch: 8000  - The mean loss is:  0.6653873\n",
            "[0 0 0 0 1 0] [0.38478294 0.47002935 0.25358015 0.39610022 0.9862511  0.3963444 ]\n",
            "------------------------------------------------\n",
            "batch: 10000  - The mean loss is:  0.5969314\n",
            "[0 0 0 1 0 0] [4.1194609e-01 1.9202128e-06 1.5215265e-06 9.9334502e-01 1.2881511e-04\n",
            " 2.0822688e-05]\n",
            "------------------------------------------------\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KSs_zCkqSK9F"
      },
      "source": [
        "Wooo Hooo! Congratulation!\n",
        "\n",
        "Now we have our Word2Vec ready in `target_amb`. Lets analyse is a little bit."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gnhdgX38SjxQ"
      },
      "source": [
        "# Model analysis\n",
        "\n",
        "Here I want to write a function that finds closest words to a word "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vxTzKaq_6rIp"
      },
      "source": [
        "def find_closest(embeds, word, n=1): # n is for \"n closest words\"\n",
        "  n = n + 1 # This is becuse the most similar word is definatly that word itself. like the most similar word for \"apple\" is \"apple\". so we should look for top n+1 words\n",
        "  main_vec = embeds(word2index[word])\n",
        "\n",
        "  similarities = -tf.keras.losses.cosine_similarity(embeds.embeddings, main_vec)\n",
        "  top_n = tf.math.top_k(similarities, n).indices\n",
        "  words = [index2word[i] for i in top_n.numpy()]\n",
        "\n",
        "  return words[1:] # I did [1:] to remove the word as I mentioned in `n = n + 1` comments\n",
        "\n"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yyJ0FPcPd0yd",
        "outputId": "674c8caf-3fec-4827-c7ce-596b11e49841",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "find_closest(target_amb, \"two\", 10)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['five', 'three', 'six', 'four', 'seven', 'zero', 'eight', 'nine', 'e', 'all']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    }
  ]
}