sentiment_analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "bert_fine_tuning.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "TPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vi0OsYO1eRK7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "843cbe89-dffa-442c-b42e-4fa0ffe2146f"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (4.3.2)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.10.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from transformers) (3.4.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.0.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "cJtY9zIsrWMP",
        "outputId": "9616828d-ff9d-4e29-97d2-0294161e5a99"
      },
      "source": [
        "!pip install tensorflow==2.4.1"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tensorflow==2.4.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c2/14/3724e7316a065b79174b10e34471e21ebb6896efbb82f689651584e6f5da/tensorflow-2.4.1-cp36-cp36m-manylinux2010_x86_64.whl (394.3MB)\n",
            "\u001b[K     |████████████████████████████████| 394.3MB 44kB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf>=3.9.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.4.1) (3.12.4)\n",
            "Requirement already satisfied: six~=1.15.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.4.1) (1.15.0)\n",
            "Requirement already satisfied: absl-py~=0.10 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.4.1) (0.10.0)\n",
            "Requirement already satisfied: opt-einsum~=3.3.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.4.1) (3.3.0)\n",
            "Requirement already satisfied: grpcio~=1.32.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.4.1) (1.32.0)\n",
            "Requirement already satisfied: wrapt~=1.12.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.4.1) (1.12.1)\n",
            "Requirement already satisfied: numpy~=1.19.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.4.1) (1.19.5)\n",
            "Requirement already satisfied: typing-extensions~=3.7.4 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.4.1) (3.7.4.3)\n",
            "Collecting tensorboard~=2.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/64/21/eebd23060763fedeefb78bc2b286e00fa1d8abda6f70efa2ee08c28af0d4/tensorboard-2.4.1-py3-none-any.whl (10.6MB)\n",
            "\u001b[K     |████████████████████████████████| 10.6MB 32.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: termcolor~=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.4.1) (1.1.0)\n",
            "Collecting tensorflow-estimator<2.5.0,>=2.4.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/7e/622d9849abf3afb81e482ffc170758742e392ee129ce1540611199a59237/tensorflow_estimator-2.4.0-py2.py3-none-any.whl (462kB)\n",
            "\u001b[K     |████████████████████████████████| 471kB 42.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.4.1) (1.1.2)\n",
            "Requirement already satisfied: flatbuffers~=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.4.1) (1.12)\n",
            "Requirement already satisfied: wheel~=0.35 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.4.1) (0.36.2)\n",
            "Requirement already satisfied: gast==0.3.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.4.1) (0.3.3)\n",
            "Requirement already satisfied: google-pasta~=0.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.4.1) (0.2.0)\n",
            "Requirement already satisfied: astunparse~=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.4.1) (1.6.3)\n",
            "Requirement already satisfied: h5py~=2.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==2.4.1) (2.10.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.9.2->tensorflow==2.4.1) (53.0.0)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (1.25.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (3.3.3)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (1.0.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (0.4.2)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (1.8.0)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard~=2.4->tensorflow==2.4.1) (2.23.0)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.1) (4.2.1)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.6\" in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.1) (4.7)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.1) (0.2.8)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tensorboard~=2.4->tensorflow==2.4.1) (3.4.0)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow==2.4.1) (1.3.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.1) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.1) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.1) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow==2.4.1) (2020.12.5)\n",
            "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.6/dist-packages (from rsa<5,>=3.1.4; python_version >= \"3.6\"->google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow==2.4.1) (0.4.8)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard~=2.4->tensorflow==2.4.1) (3.4.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow==2.4.1) (3.1.0)\n",
            "Installing collected packages: tensorboard, tensorflow-estimator, tensorflow\n",
            "  Found existing installation: tensorboard 2.2.2\n",
            "    Uninstalling tensorboard-2.2.2:\n",
            "      Successfully uninstalled tensorboard-2.2.2\n",
            "  Found existing installation: tensorflow-estimator 2.2.0\n",
            "    Uninstalling tensorflow-estimator-2.2.0:\n",
            "      Successfully uninstalled tensorflow-estimator-2.2.0\n",
            "  Found existing installation: tensorflow 2.2.0\n",
            "    Uninstalling tensorflow-2.2.0:\n",
            "      Successfully uninstalled tensorflow-2.2.0\n",
            "Successfully installed tensorboard-2.4.1 tensorflow-2.4.1 tensorflow-estimator-2.4.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "tensorboard",
                  "tensorflow"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v_wz8OiRia85",
        "outputId": "5e8ae640-0fa0-4e7e-fbe6-765660ea14e7"
      },
      "source": [
        "!pip install tensorflow_addons"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: tensorflow_addons in /usr/local/lib/python3.6/dist-packages (0.12.1)\n",
            "Requirement already satisfied: typeguard>=2.7 in /usr/local/lib/python3.6/dist-packages (from tensorflow_addons) (2.7.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yi1H_f8YebIZ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 202
        },
        "outputId": "bc75497a-0e2d-49de-aa71-ef1ea743b493"
      },
      "source": [
        "!pip install kaggle"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: kaggle in /usr/local/lib/python3.6/dist-packages (1.5.6)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from kaggle) (1.24.3)\n",
            "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.6/dist-packages (from kaggle) (2.8.1)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from kaggle) (4.41.1)\n",
            "Requirement already satisfied: six>=1.10 in /usr/local/lib/python3.6/dist-packages (from kaggle) (1.15.0)\n",
            "Requirement already satisfied: python-slugify in /usr/local/lib/python3.6/dist-packages (from kaggle) (4.0.1)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.6/dist-packages (from kaggle) (2020.6.20)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from kaggle) (2.23.0)\n",
            "Requirement already satisfied: text-unidecode>=1.3 in /usr/local/lib/python3.6/dist-packages (from python-slugify->kaggle) (1.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->kaggle) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->kaggle) (3.0.4)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kTpSd6LMTum0",
        "outputId": "00300a75-50dc-4053-e0c9-0b1c46bf9468"
      },
      "source": [
        "from google.colab import drive\r\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 62,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "96d6zUtpefSM"
      },
      "source": [
        "from google.colab import files"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HKeVHBGFehtS",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCkgewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwogICAgICBwZXJjZW50LnRleHRDb250ZW50ID0KICAgICAgICAgIGAke01hdGgucm91bmQoKHBvc2l0aW9uIC8gZmlsZURhdGEuYnl0ZUxlbmd0aCkgKiAxMDApfSUgZG9uZWA7CiAgICB9CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 73
        },
        "outputId": "5b458015-47e6-4652-8fb1-8aa6e201a520"
      },
      "source": [
        "uploaded = files.upload()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-297a152f-91b8-4f9a-b629-ae8d41dd0702\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-297a152f-91b8-4f9a-b629-ae8d41dd0702\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving kaggle.json to kaggle.json\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "296oVF7kekgi"
      },
      "source": [
        "!mkdir ~/.kaggle"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "srwDm6tberPH"
      },
      "source": [
        "!cp /content/kaggle.json ~/.kaggle/kaggle.json"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eBzSo5zaettH",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 504
        },
        "outputId": "b70feff4-f782-4781-f147-eebdd8de49c4"
      },
      "source": [
        "!kaggle competitions download -c jigsaw-multilingual-toxic-comment-classification"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Warning: Your Kaggle API key is readable by other users on this system! To fix this, you can run 'chmod 600 /root/.kaggle/kaggle.json'\n",
            "Warning: Looks like you're using an outdated API Version, please consider updating (server 1.5.6 / client 1.5.4)\n",
            "Downloading validation.csv.zip to /content\n",
            "  0% 0.00/1.35M [00:00<?, ?B/s]\n",
            "100% 1.35M/1.35M [00:00<00:00, 43.9MB/s]\n",
            "Downloading test-processed-seqlen128.csv.zip to /content\n",
            " 94% 28.0M/29.8M [00:00<00:00, 48.7MB/s]\n",
            "100% 29.8M/29.8M [00:00<00:00, 50.8MB/s]\n",
            "Downloading sample_submission.csv to /content\n",
            "  0% 0.00/612k [00:00<?, ?B/s]\n",
            "100% 612k/612k [00:00<00:00, 83.2MB/s]\n",
            "Downloading test.csv.zip to /content\n",
            " 97% 12.0M/12.4M [00:00<00:00, 39.6MB/s]\n",
            "100% 12.4M/12.4M [00:00<00:00, 41.4MB/s]\n",
            "Downloading validation-processed-seqlen128.csv.zip to /content\n",
            "  0% 0.00/3.44M [00:00<?, ?B/s]\n",
            "100% 3.44M/3.44M [00:00<00:00, 56.7MB/s]\n",
            "Downloading jigsaw-unintended-bias-train.csv.zip to /content\n",
            " 95% 278M/292M [00:02<00:00, 174MB/s]\n",
            "100% 292M/292M [00:02<00:00, 131MB/s]\n",
            "Downloading jigsaw-toxic-comment-train-processed-seqlen128.csv.zip to /content\n",
            " 88% 70.0M/79.6M [00:00<00:00, 78.0MB/s]\n",
            "100% 79.6M/79.6M [00:01<00:00, 82.9MB/s]\n",
            "Downloading jigsaw-unintended-bias-train-processed-seqlen128.csv.zip to /content\n",
            "100% 647M/650M [00:05<00:00, 84.2MB/s]\n",
            "100% 650M/650M [00:05<00:00, 114MB/s] \n",
            "Downloading jigsaw-toxic-comment-train.csv.zip to /content\n",
            " 97% 36.0M/37.3M [00:00<00:00, 46.3MB/s]\n",
            "100% 37.3M/37.3M [00:00<00:00, 45.3MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C3lKIYtMewhO"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras.layers import LSTM,Bidirectional,Input,Dense,Dropout,BatchNormalization\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras.callbacks import ReduceLROnPlateau,EarlyStopping\n",
        "from tensorflow.keras.models import Sequential,Model,save_model,load_model\n",
        "import transformers\n",
        "from tqdm import tqdm\n",
        "import re\n",
        "from tokenizers import BertWordPieceTokenizer\n",
        "from transformers import AdamW,get_linear_schedule_with_warmup, BertModel, BertConfig, TFBertModel\n",
        "from sklearn.model_selection import train_test_split\n",
        "from tensorflow_addons.optimizers import AdamW\n",
        "from tensorflow.keras import backend as K"
      ],
      "execution_count": 169,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "S8XYUDMysl6U",
        "outputId": "5b4e5fc0-7990-42a6-de96-d337928c8f57"
      },
      "source": [
        "tf.__version__"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'2.4.1'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CUEXTtzooVi9"
      },
      "source": [
        "K.clear_session()"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pP09cRBoTGr9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "12a56957-34cb-4626-bd0d-9b2e9312b8e1"
      },
      "source": [
        "!jupyter notebook --NotebookApp.iopub_data_rate_limit = 1e10"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The Jupyter HTML Notebook.\n",
            "\n",
            "This launches a Tornado based HTML Notebook Server that serves up an\n",
            "HTML5/Javascript Notebook client.\n",
            "\n",
            "Subcommands\n",
            "-----------\n",
            "\n",
            "Subcommands are launched as `jupyter-notebook cmd [args]`. For information on\n",
            "using subcommand 'cmd', do: `jupyter-notebook cmd -h`.\n",
            "\n",
            "stop\n",
            "    Stop currently running notebook server for a given port\n",
            "password\n",
            "    Set a password for the notebook server.\n",
            "list\n",
            "    List currently running notebook servers.\n",
            "\n",
            "Options\n",
            "-------\n",
            "\n",
            "Arguments that take values are actually convenience aliases to full\n",
            "Configurables, whose aliases are listed on the help line. For more information\n",
            "on full configurables, see '--help-all'.\n",
            "\n",
            "--script\n",
            "    DEPRECATED, IGNORED\n",
            "--pylab\n",
            "    DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib.\n",
            "--debug\n",
            "    set log level to logging.DEBUG (maximize logging output)\n",
            "--no-browser\n",
            "    Don't open the notebook in a browser after startup.\n",
            "--allow-root\n",
            "    Allow the notebook to be run from root user.\n",
            "-y\n",
            "    Answer yes to any questions instead of prompting.\n",
            "--no-mathjax\n",
            "    Disable MathJax\n",
            "    \n",
            "    MathJax is the javascript library Jupyter uses to render math/LaTeX. It is\n",
            "    very large, so you may want to disable it if you have a slow internet\n",
            "    connection, or for offline use of the notebook.\n",
            "    \n",
            "    When disabled, equations etc. will appear as their untransformed TeX source.\n",
            "--no-script\n",
            "    DEPRECATED, IGNORED\n",
            "--generate-config\n",
            "    generate default config file\n",
            "--certfile=<Unicode> (NotebookApp.certfile)\n",
            "    Default: u''\n",
            "    The full path to an SSL/TLS certificate file.\n",
            "--ip=<Unicode> (NotebookApp.ip)\n",
            "    Default: 'localhost'\n",
            "    The IP address the notebook server will listen on.\n",
            "--pylab=<Unicode> (NotebookApp.pylab)\n",
            "    Default: 'disabled'\n",
            "    DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib.\n",
            "--log-level=<Enum> (Application.log_level)\n",
            "    Default: 30\n",
            "    Choices: (0, 10, 20, 30, 40, 50, 'DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL')\n",
            "    Set the log level by value or name.\n",
            "--port-retries=<Integer> (NotebookApp.port_retries)\n",
            "    Default: 50\n",
            "    The number of additional ports to try if the specified port is not\n",
            "    available.\n",
            "--notebook-dir=<Unicode> (NotebookApp.notebook_dir)\n",
            "    Default: u''\n",
            "    The directory to use for notebooks and kernels.\n",
            "--keyfile=<Unicode> (NotebookApp.keyfile)\n",
            "    Default: u''\n",
            "    The full path to a private key file for usage with SSL/TLS.\n",
            "--client-ca=<Unicode> (NotebookApp.client_ca)\n",
            "    Default: u''\n",
            "    The full path to a certificate authority certificate for SSL/TLS client\n",
            "    authentication.\n",
            "--config=<Unicode> (JupyterApp.config_file)\n",
            "    Default: u''\n",
            "    Full path of a config file.\n",
            "--port=<Integer> (NotebookApp.port)\n",
            "    Default: 8888\n",
            "    The port the notebook server will listen on.\n",
            "--transport=<CaselessStrEnum> (KernelManager.transport)\n",
            "    Default: 'tcp'\n",
            "    Choices: [u'tcp', u'ipc']\n",
            "--browser=<Unicode> (NotebookApp.browser)\n",
            "    Default: u''\n",
            "    Specify what command to use to invoke a web browser when opening the\n",
            "    notebook. If not specified, the default browser will be determined by the\n",
            "    `webbrowser` standard library module, which allows setting of the BROWSER\n",
            "    environment variable to override it.\n",
            "\n",
            "To see all available configurables, use `--help-all`\n",
            "\n",
            "Examples\n",
            "--------\n",
            "\n",
            "    jupyter notebook                       # start the notebook\n",
            "    jupyter notebook --certfile=mycert.pem # use SSL/TLS certificate\n",
            "    jupyter notebook password              # enter a password to protect the server\n",
            "\n",
            "[C 20:51:23.567 NotebookApp] Bad config encountered during initialization:\n",
            "[C 20:51:23.567 NotebookApp] Invalid argument: '--NotebookApp.iopub_data_rate_limit'\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MqYDTBhYIBph",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e16571dd-1a1e-40cf-9a17-60d0d811e41c"
      },
      "source": [
        "!pip install -q --upgrade ipython\n",
        "!pip install -q --upgrade ipykernel"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[31mERROR: Operation cancelled by user\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HfwakatPe0nW",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "1d53455f-014c-45e9-faf3-5906522e08ea"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('gdrive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at gdrive; to attempt to forcibly remount, call drive.mount(\"gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZGaGwAoGe6HV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "72f65b4a-4bff-422d-84bc-d2c466b5104b"
      },
      "source": [
        "# Detect hardware, return appropriate distribution strategy\n",
        "try:\n",
        "    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n",
        "    # set: this is always the case on Kaggle.\n",
        "    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
        "    print('Running on TPU ', tpu.master())\n",
        "except ValueError:\n",
        "    tpu = None\n",
        "\n",
        "if tpu:\n",
        "    tf.config.experimental_connect_to_cluster(tpu)\n",
        "    tf.tpu.experimental.initialize_tpu_system(tpu)\n",
        "    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
        "else:\n",
        "    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n",
        "    strategy = tf.distribute.get_strategy()\n",
        "\n",
        "print(\"REPLICAS: \", strategy.num_replicas_in_sync)"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Running on TPU  grpc://10.113.11.186:8470\n",
            "WARNING:tensorflow:TPU system grpc://10.113.11.186:8470 has already been initialized. Reinitializing the TPU can cause previously created variables on TPU to be lost.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:TPU system grpc://10.113.11.186:8470 has already been initialized. Reinitializing the TPU can cause previously created variables on TPU to be lost.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.113.11.186:8470\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Initializing the TPU system: grpc://10.113.11.186:8470\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Clearing out eager caches\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Finished initializing TPU system.\n",
            "WARNING:absl:`tf.distribute.experimental.TPUStrategy` is deprecated, please use  the non experimental symbol `tf.distribute.TPUStrategy` instead.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Found TPU system:\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Workers: 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Num TPU Cores Per Worker: 8\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:localhost/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:CPU:0, CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:0, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:1, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:2, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:3, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:4, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:5, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:6, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU:7, TPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:TPU_SYSTEM:0, TPU_SYSTEM, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:*** Available Device: _DeviceAttributes(/job:worker/replica:0/task:0/device:XLA_CPU:0, XLA_CPU, 0, 0)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "REPLICAS:  8\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qFPcgOkEfDGr"
      },
      "source": [
        "train_data = pd.read_csv('jigsaw-toxic-comment-train.csv.zip',compression='zip', header=0, sep=',', quotechar='\"')\n",
        "#val_data = pd.read_csv('validation.csv.zip',compression='zip', header=0, sep=',', quotechar='\"')\n",
        "#test_data = pd.read_csv('test.csv.zip',compression='zip', header=0, sep=',', quotechar='\"')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V6oRWJAm-SqZ"
      },
      "source": [
        "train_2 = pd.read_csv('jigsaw-unintended-bias-train.csv.zip',compression='zip', header=0, sep=',', quotechar='\"')\n",
        "train_2.toxic = train_2.toxic.round().astype(int)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F6etMSl1-ZE3"
      },
      "source": [
        "train_data = pd.concat([train_data[['comment_text','toxic']],\n",
        "                        train_2[['comment_text','toxic']].query('toxic == 1'),\n",
        "                        train_2[['comment_text','toxic']].query('toxic == 0').sample(n = 20000, random_state = 0)\n",
        "                      ])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s86kDe6Z-cux",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67
        },
        "outputId": "1c3e0374-b723-4bc3-b4b9-4bf113bd4f1c"
      },
      "source": [
        "train_data['toxic'].value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    222165\n",
              "1    133610\n",
              "Name: toxic, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sEwhsfjyfH-S"
      },
      "source": [
        "y_train = train_data.toxic.values\n",
        "#y_val = val_data.toxic.values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_cr-ri_0O7b-"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "train_data,val_data = train_test_split(train_data,test_size = 0.1, random_state = 42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qnEpuo_fPRRD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 185
        },
        "outputId": "6231ccc0-b445-48ed-cde1-9afed4be448e"
      },
      "source": [
        "print(train_data.info())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 320197 entries, 12147 to 121958\n",
            "Data columns (total 2 columns):\n",
            " #   Column        Non-Null Count   Dtype \n",
            "---  ------        --------------   ----- \n",
            " 0   comment_text  320197 non-null  object\n",
            " 1   toxic         320197 non-null  int64 \n",
            "dtypes: int64(1), object(1)\n",
            "memory usage: 7.3+ MB\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UdLsJH4BUlBC",
        "outputId": "e06cf626-631f-4e03-9a54-07fc7d4cae8e"
      },
      "source": [
        "#loading other dataset\r\n",
        "\r\n",
        "df1 = pd.read_csv('/content/gdrive/MyDrive/twitter_sentiment_analysis/Twitter_Data.csv')\r\n",
        "print(df1['category'].value_counts())\r\n",
        "df2 = pd.read_csv('/content/gdrive/MyDrive/twitter_sentiment_analysis/Reddit_Data.csv')\r\n",
        "print(df2['category'].value_counts())\r\n",
        "df = pd.concat([df1, df2])\r\n",
        "print(df['category'].value_counts())\r\n",
        "\r\n",
        "df.drop(columns=['clean_comment'], axis = 1, inplace = True)\r\n",
        "df.dropna(inplace = True)\r\n",
        "print(df['category'].value_counts())\r\n",
        "\r\n",
        "df['category'] = df['category'].apply(lambda x: 2 if x == -1 else x)\r\n",
        "print(df.head(5))\r\n",
        "print(df['category'].value_counts())"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " 1.0    72250\n",
            " 0.0    55213\n",
            "-1.0    35510\n",
            "Name: category, dtype: int64\n",
            " 1    15830\n",
            " 0    13142\n",
            "-1     8277\n",
            "Name: category, dtype: int64\n",
            " 1.0    88080\n",
            " 0.0    68355\n",
            "-1.0    43787\n",
            "Name: category, dtype: int64\n",
            " 1.0    72249\n",
            " 0.0    55211\n",
            "-1.0    35509\n",
            "Name: category, dtype: int64\n",
            "                                          clean_text  category\n",
            "0  when modi promised “minimum government maximum...       2.0\n",
            "1  talk all the nonsense and continue all the dra...       0.0\n",
            "2  what did just say vote for modi  welcome bjp t...       1.0\n",
            "3  asking his supporters prefix chowkidar their n...       1.0\n",
            "4  answer who among these the most powerful world...       1.0\n",
            "1.0    72249\n",
            "0.0    55211\n",
            "2.0    35509\n",
            "Name: category, dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pD2a76sQXMwa",
        "outputId": "b35a9b0d-334c-442b-f5eb-d7aac34fa9fd"
      },
      "source": [
        "train_x, val_x, train_y, val_y = train_test_split(df['clean_text'], df['category'], test_size = 0.2, random_state = 42)\r\n",
        "print(len(train_x))\r\n",
        "print(len(val_x))"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "130375\n",
            "32594\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rUDenD96b0Q1",
        "outputId": "c2d84994-ccce-4068-ce49-2e3ddce0b5cc"
      },
      "source": [
        "print(train_x[:5])\r\n",
        "print(train_y[:5])"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "63425     some media guys cant even think straight forwa...\n",
            "78455                                             only namo\n",
            "70129     does this add the space debris problem the awa...\n",
            "157073    today’ interaction modi phenomenal looking fre...\n",
            "120885    more modi bjp wants change nitin gadkari will ...\n",
            "Name: clean_text, dtype: object\n",
            "63425     1.0\n",
            "78455     0.0\n",
            "70129     1.0\n",
            "157073    1.0\n",
            "120885    1.0\n",
            "Name: category, dtype: float64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "79YAan0qfPgy"
      },
      "source": [
        "AUTO = tf.data.experimental.AUTOTUNE\n",
        "max_len = 256\n",
        "BATCH_SIZE = 16*strategy.num_replicas_in_sync\n",
        "EPOCHS = 20"
      ],
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GhLn7bb7fdRZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b5211b01-5ff0-480a-f3aa-3e86be4dc1a8"
      },
      "source": [
        "from tokenizers import BertWordPieceTokenizer\n",
        "\n",
        "tokenizer = transformers.DistilBertTokenizer.from_pretrained('bert-base-uncased',lowercase = True)\n",
        "tokenizer.save_pretrained('.')\n",
        "# Reload it with the huggingface tokenizers library\n",
        "fast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=True)\n",
        "fast_tokenizer"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Tokenizer(vocabulary_size=30522, model=BertWordPiece, unk_token=[UNK], sep_token=[SEP], cls_token=[CLS], pad_token=[PAD], mask_token=[MASK], clean_text=True, handle_chinese_chars=True, strip_accents=None, lowercase=True, wordpieces_prefix=##)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gn8H54ywgerG"
      },
      "source": [
        "def convert_example_to_feature(sentence):\n",
        "  return tokenizer.encode_plus(\n",
        "      sentence,\n",
        "      add_special_tokens = True,\n",
        "      max_length = max_len,\n",
        "      padding = True,\n",
        "      truncation = True,\n",
        "      return_attention_mask = True,\n",
        "      return_token_type_ids = True\n",
        "  )"
      ],
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vucAdSAKh1Vf"
      },
      "source": [
        "def map_example_to_dict(input_ids, attention_masks, token_type_ids, label):\n",
        "  return {\n",
        "      \"input_ids\": input_ids,\n",
        "      \"token_type_ids\": token_type_ids,\n",
        "      \"attention_mask\": attention_masks,\n",
        "  }, label"
      ],
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2drQ2lxjh8eV"
      },
      "source": [
        "def encode_example(train_x, train_y):\n",
        "  input_ids_list = []\n",
        "  token_type_ids_list = []\n",
        "  attention_mask_list = []\n",
        "  label_list = []\n",
        "\n",
        "\n",
        "  for sentence,label in tqdm(zip(train_x.astype(str),train_y.astype(np.int64))):\n",
        "    bert_input = convert_example_to_feature(sentence)\n",
        " \n",
        "    #     bert_input['token_type_ids'] = bert_input['token_type_ids'] + ([0] * (max_len - len(bert_input['token_type_ids'])))\n",
        "    #     bert_input['attention_mask'] = bert_input['attention_mask'] + ([0] * (max_len - len(bert_input['attention_mask'])))\n",
        "  \n",
        "      \n",
        "    if(len(bert_input['input_ids']) > max_len):\n",
        "      for i in range(len(bert_input['input_ids']) - max_len):\n",
        "        del bert_input['input_ids'][-1]\n",
        "    else:\n",
        "      bert_input['input_ids'] = bert_input['input_ids'] + [0]*(max_len - len(bert_input['input_ids']))\n",
        "    \n",
        "    if(len(bert_input['token_type_ids']) > max_len):\n",
        "      for i in range(len(bert_input['token_type_ids']) - max_len):\n",
        "        del bert_input['token_type_ids'][-1]\n",
        "    else:\n",
        "      bert_input['token_type_ids'] = bert_input['token_type_ids'] + [0]*(max_len - len(bert_input['token_type_ids']))\n",
        "\n",
        "    if(len(bert_input['attention_mask']) > max_len):\n",
        "      for i in range(len(bert_input['attention_mask']) - max_len):\n",
        "        del bert_input['attention_mask'][-1]\n",
        "    else:\n",
        "      bert_input['attention_mask'] = bert_input['attention_mask'] + [0]*(max_len - len(bert_input['attention_mask']))\n",
        "\n",
        "    if(len(bert_input['attention_mask']) < max_len or len(bert_input['attention_mask']) > max_len or len(bert_input['token_type_ids']) < max_len or len(bert_input['token_type_ids']) > max_len or len(bert_input['input_ids']) != max_len):\n",
        "        print('bad')\n",
        "\n",
        "    input_ids_list.append(bert_input['input_ids'])\n",
        "    # print(input_ids_list)\n",
        "    token_type_ids_list.append(bert_input['token_type_ids'])\n",
        "    attention_mask_list.append(bert_input['attention_mask'])\n",
        "    label_list.append([label])\n",
        "\n",
        "  return tf.data.Dataset.from_tensor_slices((input_ids_list, attention_mask_list, token_type_ids_list, label_list)).map(map_example_to_dict) "
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XcGGEUG2jc1h",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "732dbb5f-8f08-4d70-ccf8-04bad926586f"
      },
      "source": [
        "train_data_encode = encode_example(train_x, train_y).repeat().shuffle(32768).batch(BATCH_SIZE).prefetch(AUTO)\n",
        "print('done')\n",
        "#print(train_data_encode)"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "130375it [01:39, 1309.55it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "done\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RsjswszSoWdV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "70855850-56ba-4eec-b391-693b0e4fe775"
      },
      "source": [
        "val_data_encode = encode_example(val_x, val_y).batch(BATCH_SIZE).cache().prefetch(AUTO)"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "32594it [00:23, 1399.89it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "muAOMbH1letb",
        "outputId": "dd6ef8f4-0a0c-46ac-f943-250d9224ec9b"
      },
      "source": [
        "print(val_data_encode)"
      ],
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<PrefetchDataset shapes: ({input_ids: (None, 256), token_type_ids: (None, 256), attention_mask: (None, 256)}, (None, 1)), types: ({input_ids: tf.int32, token_type_ids: tf.int32, attention_mask: tf.int32}, tf.int32)>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V44JIR59kmok",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f3d0f624-70fe-4942-b6a6-014d116fc175"
      },
      "source": [
        "#fine tuning of the model.\n",
        "\n",
        "from transformers import TFBertForSequenceClassification, BertConfig\n",
        "\n",
        "def build_model():\n",
        "  config = BertConfig.from_pretrained('bert-base-uncased', num_labels=3)\n",
        "  model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased', config = config)\n",
        "\n",
        "  #model.summary()\n",
        "\n",
        "  # choosing Adam optimizer\n",
        "  lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(2e-5, 2, 0.96)\n",
        "  wd_schedule = tf.keras.optimizers.schedules.ExponentialDecay(1e-2, 2, 0.9)\n",
        "  optimizer = AdamW(learning_rate=lr_schedule, weight_decay=lambda : None)\n",
        "  optimizer.weight_decay = lambda : wd_schedule(optimizer.iterations)\n",
        "  # optimizer = tf.keras.optimizers.Adam(learning_rate=2e-5, epsilon=1e-08)\n",
        "\n",
        "  # total_steps = int((len(train_data) / BATCH_SIZE)*EPOCHS)\n",
        "  # warm_up_steps = int((len(train_data) / BATCH_SIZE)*5) \n",
        "  # scheduler = linear_schedule_with_warmup(\n",
        "  #     optimizer,\n",
        "  #     num_warmup_steps = warm_up_steps,\n",
        "  #     num_training_steps = total_steps\n",
        "  # )\n",
        "\n",
        "  # we do not have one-hot vectors, we can use sparce categorical cross entropy and accuracy\n",
        "  loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "  metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy')\n",
        "\n",
        "  model.compile(optimizer=optimizer, loss=loss, metrics=[metric])\n",
        "  return model\n",
        "\n",
        "with strategy.scope():\n",
        "  model = build_model()\n",
        "\n",
        "model.summary()\n"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "All model checkpoint layers were used when initializing TFBertForSequenceClassification.\n",
            "\n",
            "Some layers of TFBertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Model: \"tf_bert_for_sequence_classification_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "bert (TFBertMainLayer)       multiple                  109482240 \n",
            "_________________________________________________________________\n",
            "dropout_75 (Dropout)         multiple                  0         \n",
            "_________________________________________________________________\n",
            "classifier (Dense)           multiple                  2307      \n",
            "=================================================================\n",
            "Total params: 109,484,547\n",
            "Trainable params: 109,484,547\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pIfla8QZm2fg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3ad4317c-9aef-48ac-9b4a-1b79e7a711d0"
      },
      "source": [
        "callbacks = [ReduceLROnPlateau(factor = 0.2,patience = 4),\n",
        "             #EarlyStopping(patience = 7)\n",
        "            ]\n",
        "\n",
        "EPOCHS = 2\n",
        "e_steps = int(len(train_x) // BATCH_SIZE)\n",
        "v_steps = int(len(val_x) // BATCH_SIZE)\n",
        "bert_history = model.fit(train_data_encode , epochs=EPOCHS, validation_data = val_data_encode, \n",
        "                         steps_per_epoch = e_steps, validation_steps = v_steps)"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/2\n",
            "1018/1018 [==============================] - 218s 214ms/step - loss: 0.8298 - accuracy: 0.6328 - val_loss: 0.8246 - val_accuracy: 0.6336\n",
            "Epoch 2/2\n",
            "1018/1018 [==============================] - 218s 214ms/step - loss: 0.8311 - accuracy: 0.6318 - val_loss: 0.8246 - val_accuracy: 0.6336\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Y5GAOnCnMBs"
      },
      "source": [
        "model.save_pretrained('gdrive/MyDrive/sentiment_analysis_v1')"
      ],
      "execution_count": 60,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zGow14jbnX4L"
      },
      "source": [
        "drive.flush_and_unmount()"
      ],
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MUD0EMKX7STa"
      },
      "source": [
        "import json\n",
        "\n",
        "with open(\"/var/log/colab-jupyter.log\", \"r\") as fo:\n",
        "  for line in fo:\n",
        "    print(json.loads(line)['msg'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4wKJom50SDwU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "35a55ab6-30b1-4026-da29-35fbc0ec4ae3"
      },
      "source": [
        "config = BertConfig.from_json_file('/content/gdrive/MyDrive/sentiment_analysis_v1/config.json')\r\n",
        "# config.output_hidden_states = False\r\n",
        "model = TFBertModel.from_pretrained('/content/gdrive/MyDrive/sentiment_analysis_v1', config = config)\r\n",
        "# model = build_model(transformer_layer, max_len=MAX_LEN)\r\n",
        "model.summary()"
      ],
      "execution_count": 172,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some layers from the model checkpoint at /content/gdrive/MyDrive/sentiment_analysis_v1 were not used when initializing TFBertModel: ['dropout_75', 'classifier']\n",
            "- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the layers of TFBertModel were initialized from the model checkpoint at /content/gdrive/MyDrive/sentiment_analysis_v1.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Model: \"tf_bert_model_10\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "bert (TFBertMainLayer)       multiple                  109482240 \n",
            "=================================================================\n",
            "Total params: 109,482,240\n",
            "Trainable params: 109,482,240\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RsZEFJkpH_Ye"
      },
      "source": [
        "def convert_example_to_feature(sentence):\r\n",
        "  return tokenizer.encode_plus(\r\n",
        "      sentence,\r\n",
        "      add_special_tokens = True,\r\n",
        "      max_length = max_len,\r\n",
        "      padding = True,\r\n",
        "      truncation = True,\r\n",
        "      return_attention_mask = True,\r\n",
        "      return_token_type_ids = True\r\n",
        "  )"
      ],
      "execution_count": 66,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8H_96em2LonW"
      },
      "source": [
        "def map_example_to_dict(input_ids, attention_masks, token_type_ids, label):\r\n",
        "  return {\r\n",
        "      \"input_ids\": input_ids,\r\n",
        "      \"token_type_ids\": token_type_ids,\r\n",
        "      \"attention_mask\": attention_masks,\r\n",
        "  }, label"
      ],
      "execution_count": 101,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JB7qI5kLV__e"
      },
      "source": [
        "import torch"
      ],
      "execution_count": 125,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_NHRi5LoJthk"
      },
      "source": [
        "sentence = \"You are a fucking crap person\"\r\n",
        "\r\n",
        "input_ids_list = []\r\n",
        "token_type_ids_list = []\r\n",
        "attention_mask_list = []\r\n",
        "\r\n",
        "bert_input = convert_example_to_feature(sentence)\r\n",
        "\r\n",
        "if(len(bert_input['input_ids']) > max_len):\r\n",
        "  for i in range(len(bert_input['input_ids']) - max_len):\r\n",
        "    del bert_input['input_ids'][-1]\r\n",
        "else:\r\n",
        "  bert_input['input_ids'] = bert_input['input_ids'] + [0]*(max_len - len(bert_input['input_ids']))\r\n",
        "\r\n",
        "if(len(bert_input['token_type_ids']) > max_len):\r\n",
        "  for i in range(len(bert_input['token_type_ids']) - max_len):\r\n",
        "    del bert_input['token_type_ids'][-1]\r\n",
        "else:\r\n",
        "  bert_input['token_type_ids'] = bert_input['token_type_ids'] + [0]*(max_len - len(bert_input['token_type_ids']))\r\n",
        "\r\n",
        "if(len(bert_input['attention_mask']) > max_len):\r\n",
        "  for i in range(len(bert_input['attention_mask']) - max_len):\r\n",
        "    del bert_input['attention_mask'][-1]\r\n",
        "else:\r\n",
        "  bert_input['attention_mask'] = bert_input['attention_mask'] + [0]*(max_len - len(bert_input['attention_mask']))\r\n",
        "\r\n",
        "if(len(bert_input['attention_mask']) < max_len or len(bert_input['attention_mask']) > max_len or len(bert_input['token_type_ids']) < max_len or len(bert_input['token_type_ids']) > max_len or len(bert_input['input_ids']) != max_len):\r\n",
        "    print('bad')\r\n",
        "\r\n",
        "input_ids_list.append(bert_input['input_ids'])\r\n",
        "# print(input_ids_list)\r\n",
        "token_type_ids_list.append(bert_input['token_type_ids'])\r\n",
        "attention_mask_list.append(bert_input['attention_mask'])\r\n",
        "\r\n",
        "# input_ids_list = np.expand_dims(np.array(input_ids_list), axis = 0)\r\n",
        "# attention_mask_list = np.expand_dims(np.array(attention_mask_list), axis = 0)\r\n",
        "# token_type_ids_list = np.expand_dims(np.array(token_type_ids_list), axis = 0)\r\n",
        "# label = [None]\r\n",
        "# ds = tf.data.Dataset.from_tensor_slices((input_ids_list, attention_mask_list, token_type_ids_list, label))\r\n",
        "input_ids_list = torch.LongTensor(input_ids_list)\r\n",
        "token_type_ids_list = torch.LongTensor(token_type_ids_list)\r\n",
        "attention_mask_list = torch.LongTensor(attention_mask_list)\r\n",
        "# print(input_ids_list.size())\r\n",
        "# print(attention_mask_list)\r\n",
        "# print(token_type_ids_list)\r\n",
        "\r\n",
        "# predict_dict = {\"input_ids\": input_ids_list,\r\n",
        "#                 \"token_type_ids\": token_type_ids_list,\r\n",
        "#                 \"attention_mask\": attention_mask_list}"
      ],
      "execution_count": 157,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P5gbq-dmKUPP"
      },
      "source": [
        "output = model(input_ids = input_ids_list, attention_mask = attention_mask_list, token_type_ids = token_type_ids_list)\r\n",
        "# output = model(*ds)"
      ],
      "execution_count": 158,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ux--pTVpKrR4"
      },
      "source": [
        "sequence_output = output[0]\r\n",
        "cls_token = sequence_output[:, 0, :]"
      ],
      "execution_count": 160,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2g6TJwnOWe4W",
        "outputId": "6cb8a6df-d711-4076-add7-3646e6f9093c"
      },
      "source": [
        "cls_token.shape"
      ],
      "execution_count": 162,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1, 768])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 162
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1fp05nPjb9FH"
      },
      "source": [
        "#training on our twitter sentiment analysis part\r\n",
        "\r\n",
        "def build_model(transformer, max_len=256):\r\n",
        "    \"\"\"\r\n",
        "    function for training the BERT model\r\n",
        "    \"\"\"\r\n",
        "    input_ids = Input(shape=(max_len,), dtype='int32', name=\"input_ids\")\r\n",
        "    token_type_ids = Input(shape=(max_len,), dtype='int32', name=\"token_type_ids\")\r\n",
        "    attention_mask = Input(shape=(max_len,), dtype='int32', name=\"attention_mask\")\r\n",
        "\r\n",
        "    sequence_output = transformer(input_ids, token_type_ids, attention_mask)[0]\r\n",
        "    cls_token = sequence_output[:, 0, :]\r\n",
        "    out = Dense(3, activation='softmax')(cls_token)\r\n",
        "    \r\n",
        "    model = Model(inputs=[input_ids, token_type_ids, attention_mask] , outputs=out)\r\n",
        "    model.compile(Adam(lr=2e-5), loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])\r\n",
        "    \r\n",
        "    return model"
      ],
      "execution_count": 181,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N3WXBBqgdvP8",
        "outputId": "38bbc54f-65e4-4f5f-b2f3-d125ea65efa2"
      },
      "source": [
        "with strategy.scope():\r\n",
        "    config = BertConfig.from_json_file('/content/gdrive/MyDrive/sentiment_analysis_v1/config.json')\r\n",
        "    # config.output_hidden_states = False\r\n",
        "    transformer_layer = TFBertModel.from_pretrained('/content/gdrive/MyDrive/sentiment_analysis_v1', config = config)\r\n",
        "    model = build_model(transformer_layer, max_len=256)\r\n",
        "model.summary()"
      ],
      "execution_count": 182,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some layers from the model checkpoint at /content/gdrive/MyDrive/sentiment_analysis_v1 were not used when initializing TFBertModel: ['dropout_75', 'classifier']\n",
            "- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the layers of TFBertModel were initialized from the model checkpoint at /content/gdrive/MyDrive/sentiment_analysis_v1.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_2\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_ids (InputLayer)          [(None, 256)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "token_type_ids (InputLayer)     [(None, 256)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "attention_mask (InputLayer)     [(None, 256)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "tf_bert_model_14 (TFBertModel)  TFBaseModelOutputWit 109482240   input_ids[0][0]                  \n",
            "                                                                 token_type_ids[0][0]             \n",
            "                                                                 attention_mask[0][0]             \n",
            "__________________________________________________________________________________________________\n",
            "tf.__operators__.getitem_2 (Sli (None, 768)          0           tf_bert_model_14[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, 3)            2307        tf.__operators__.getitem_2[0][0] \n",
            "==================================================================================================\n",
            "Total params: 109,484,547\n",
            "Trainable params: 109,484,547\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 928
        },
        "id": "lPK4YM7rd_C4",
        "outputId": "8e854d24-0486-4923-9573-2e4f003e22b7"
      },
      "source": [
        "callbacks = [ReduceLROnPlateau(factor = 0.2,patience = 3),\r\n",
        "             #EarlyStopping(patience = 7)\r\n",
        "            ]\r\n",
        "\r\n",
        "EPOCHS = 10\r\n",
        "e_steps = int(len(train_x) // BATCH_SIZE)\r\n",
        "v_steps = int(len(val_x) // BATCH_SIZE)\r\n",
        "bert_history = model.fit(train_data_encode , epochs=EPOCHS, validation_data = val_data_encode, \r\n",
        "                         steps_per_epoch = e_steps, validation_steps = v_steps, callbacks = callbacks)"
      ],
      "execution_count": 183,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_14/bert/pooler/dense/kernel:0', 'tf_bert_model_14/bert/pooler/dense/bias:0'] when minimizing the loss.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_14/bert/pooler/dense/kernel:0', 'tf_bert_model_14/bert/pooler/dense/bias:0'] when minimizing the loss.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_14/bert/pooler/dense/kernel:0', 'tf_bert_model_14/bert/pooler/dense/bias:0'] when minimizing the loss.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_model_14/bert/pooler/dense/kernel:0', 'tf_bert_model_14/bert/pooler/dense/bias:0'] when minimizing the loss.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "1018/1018 [==============================] - ETA: 0s - loss: 0.5493 - sparse_categorical_accuracy: 0.7595WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r1018/1018 [==============================] - 300s 225ms/step - loss: 0.5491 - sparse_categorical_accuracy: 0.7596 - val_loss: 0.1157 - val_sparse_categorical_accuracy: 0.9636\n",
            "Epoch 2/10\n",
            "1018/1018 [==============================] - 217s 213ms/step - loss: 0.1145 - sparse_categorical_accuracy: 0.9642 - val_loss: 0.0743 - val_sparse_categorical_accuracy: 0.9773\n",
            "Epoch 3/10\n",
            "1018/1018 [==============================] - 217s 213ms/step - loss: 0.0709 - sparse_categorical_accuracy: 0.9783 - val_loss: 0.0684 - val_sparse_categorical_accuracy: 0.9785\n",
            "Epoch 4/10\n",
            "1018/1018 [==============================] - 217s 214ms/step - loss: 0.0528 - sparse_categorical_accuracy: 0.9831 - val_loss: 0.0595 - val_sparse_categorical_accuracy: 0.9813\n",
            "Epoch 5/10\n",
            "1018/1018 [==============================] - 217s 213ms/step - loss: 0.0421 - sparse_categorical_accuracy: 0.9864 - val_loss: 0.0564 - val_sparse_categorical_accuracy: 0.9830\n",
            "Epoch 6/10\n",
            "1018/1018 [==============================] - 218s 214ms/step - loss: 0.0356 - sparse_categorical_accuracy: 0.9886 - val_loss: 0.0685 - val_sparse_categorical_accuracy: 0.9809\n",
            "Epoch 7/10\n",
            "1018/1018 [==============================] - 217s 214ms/step - loss: 0.0286 - sparse_categorical_accuracy: 0.9912 - val_loss: 0.0624 - val_sparse_categorical_accuracy: 0.9838\n",
            "Epoch 8/10\n",
            "1018/1018 [==============================] - 218s 214ms/step - loss: 0.0251 - sparse_categorical_accuracy: 0.9926 - val_loss: 0.0634 - val_sparse_categorical_accuracy: 0.9825\n",
            "Epoch 9/10\n",
            " 164/1018 [===>..........................] - ETA: 2:48 - loss: 0.0170 - sparse_categorical_accuracy: 0.9947"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-183-01d395b43b6f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0mv_steps\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_x\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m//\u001b[0m \u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m bert_history = model.fit(train_data_encode , epochs=EPOCHS, validation_data = val_data_encode, \n\u001b[0;32m----> 9\u001b[0;31m                          steps_per_epoch = e_steps, validation_steps = v_steps, callbacks = callbacks)\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1103\u001b[0m               \u001b[0mlogs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtmp_logs\u001b[0m  \u001b[0;31m# No error, now safe to assign to logs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1104\u001b[0m               \u001b[0mend_step\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep_increment\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1105\u001b[0;31m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mend_step\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1106\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstop_training\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1107\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/callbacks.py\u001b[0m in \u001b[0;36mon_train_batch_end\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m    452\u001b[0m     \"\"\"\n\u001b[1;32m    453\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_should_call_train_batch_hooks\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 454\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mModeKeys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTRAIN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'end'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    455\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    456\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mon_test_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_hook\u001b[0;34m(self, mode, hook, batch, logs)\u001b[0m\n\u001b[1;32m    294\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_begin_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    295\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'end'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 296\u001b[0;31m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_end_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    297\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    298\u001b[0m       \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Unrecognized hook: {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_end_hook\u001b[0;34m(self, mode, batch, logs)\u001b[0m\n\u001b[1;32m    314\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_times\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_time\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    315\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 316\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_batch_hook_helper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhook_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    317\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    318\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_times\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_batches_for_timing_check\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/callbacks.py\u001b[0m in \u001b[0;36m_call_batch_hook_helper\u001b[0;34m(self, hook_name, batch, logs)\u001b[0m\n\u001b[1;32m    354\u001b[0m       \u001b[0mhook\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    355\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'_supports_tf_logs'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 356\u001b[0;31m         \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    357\u001b[0m       \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    358\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mnumpy_logs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Only convert once.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/callbacks.py\u001b[0m in \u001b[0;36mon_train_batch_end\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m   1018\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1019\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mon_train_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1020\u001b[0;31m     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_update_progbar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1021\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1022\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mon_test_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlogs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/callbacks.py\u001b[0m in \u001b[0;36m_batch_update_progbar\u001b[0;34m(self, batch, logs)\u001b[0m\n\u001b[1;32m   1082\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mverbose\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1083\u001b[0m       \u001b[0;31m# Only block async when verbose = 1.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1084\u001b[0;31m       \u001b[0mlogs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_numpy_or_python_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1085\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprogbar\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mseen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfinalize\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1086\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/utils/tf_utils.py\u001b[0m in \u001b[0;36mto_numpy_or_python_type\u001b[0;34m(tensors)\u001b[0m\n\u001b[1;32m    512\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m  \u001b[0;31m# Don't turn ragged or sparse tensors to NumPy.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    513\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 514\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_structure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_to_single_numpy_or_python_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    515\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    516\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/nest.py\u001b[0m in \u001b[0;36mmap_structure\u001b[0;34m(func, *structure, **kwargs)\u001b[0m\n\u001b[1;32m    657\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    658\u001b[0m   return pack_sequence_as(\n\u001b[0;32m--> 659\u001b[0;31m       \u001b[0mstructure\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mentries\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    660\u001b[0m       expand_composites=expand_composites)\n\u001b[1;32m    661\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/util/nest.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    657\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    658\u001b[0m   return pack_sequence_as(\n\u001b[0;32m--> 659\u001b[0;31m       \u001b[0mstructure\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mentries\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    660\u001b[0m       expand_composites=expand_composites)\n\u001b[1;32m    661\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/utils/tf_utils.py\u001b[0m in \u001b[0;36m_to_single_numpy_or_python_type\u001b[0;34m(t)\u001b[0m\n\u001b[1;32m    508\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_to_single_numpy_or_python_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    509\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 510\u001b[0;31m       \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    511\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    512\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m  \u001b[0;31m# Don't turn ragged or sparse tensors to NumPy.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36mnumpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1069\u001b[0m     \"\"\"\n\u001b[1;32m   1070\u001b[0m     \u001b[0;31m# TODO(slebedev): Consider avoiding a copy for non-CPU or remote tensors.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1071\u001b[0;31m     \u001b[0mmaybe_arr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1072\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mmaybe_arr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmaybe_arr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mmaybe_arr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1073\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36m_numpy\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1035\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_numpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1036\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1037\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_numpy_internal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1038\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1039\u001b[0m       \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_from\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_status_to_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BnRTuQG_g6st"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uEMnvyDtrh40"
      },
      "source": [
        "#Modi government is dedicated to ‘Jan Kalyan’ but Mamata Didi’s government is busy with ‘Bhatija Kalyan’\r\n",
        "#Amazing energy and enthusiasm here at Science City Auditorium, Kolkata. Interacting with our Social Media warriors from all over West Bengal.\r\n",
        "#Godi media is not investigated. Planned, voluntary compromise of journalistic ethics isn't investigated... A toolkit is. This is Animal Farm in real life."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NjRetBtWj3lh"
      },
      "source": [
        "#sentence = \"Godi media is not investigated. Planned, voluntary compromise of journalistic ethics isn't investigated... A toolkit is. This is Animal Farm in real life.\"\r\n",
        "sentence = \"this is a shit thing\"\r\n",
        "input_ids_list = []\r\n",
        "token_type_ids_list = []\r\n",
        "attention_mask_list = []\r\n",
        "\r\n",
        "bert_input = convert_example_to_feature(sentence)\r\n",
        "\r\n",
        "if(len(bert_input['input_ids']) > max_len):\r\n",
        "  for i in range(len(bert_input['input_ids']) - max_len):\r\n",
        "    del bert_input['input_ids'][-1]\r\n",
        "else:\r\n",
        "  bert_input['input_ids'] = bert_input['input_ids'] + [0]*(max_len - len(bert_input['input_ids']))\r\n",
        "\r\n",
        "if(len(bert_input['token_type_ids']) > max_len):\r\n",
        "  for i in range(len(bert_input['token_type_ids']) - max_len):\r\n",
        "    del bert_input['token_type_ids'][-1]\r\n",
        "else:\r\n",
        "  bert_input['token_type_ids'] = bert_input['token_type_ids'] + [0]*(max_len - len(bert_input['token_type_ids']))\r\n",
        "\r\n",
        "if(len(bert_input['attention_mask']) > max_len):\r\n",
        "  for i in range(len(bert_input['attention_mask']) - max_len):\r\n",
        "    del bert_input['attention_mask'][-1]\r\n",
        "else:\r\n",
        "  bert_input['attention_mask'] = bert_input['attention_mask'] + [0]*(max_len - len(bert_input['attention_mask']))\r\n",
        "\r\n",
        "if(len(bert_input['attention_mask']) < max_len or len(bert_input['attention_mask']) > max_len or len(bert_input['token_type_ids']) < max_len or len(bert_input['token_type_ids']) > max_len or len(bert_input['input_ids']) != max_len):\r\n",
        "    print('bad')\r\n",
        "\r\n",
        "input_ids_list.append(bert_input['input_ids'])\r\n",
        "# print(input_ids_list)\r\n",
        "token_type_ids_list.append(bert_input['token_type_ids'])\r\n",
        "attention_mask_list.append(bert_input['attention_mask'])\r\n",
        "\r\n",
        "# input_ids_list = np.expand_dims(np.array(input_ids_list), axis = 0)\r\n",
        "# attention_mask_list = np.expand_dims(np.array(attention_mask_list), axis = 0)\r\n",
        "# token_type_ids_list = np.expand_dims(np.array(token_type_ids_list), axis = 0)\r\n",
        "# label = [None]\r\n",
        "# ds = tf.data.Dataset.from_tensor_slices((input_ids_list, attention_mask_list, token_type_ids_list, label))\r\n",
        "input_ids_list = tf.constant(input_ids_list)\r\n",
        "token_type_ids_list = tf.constant(token_type_ids_list)\r\n",
        "attention_mask_list = tf.constant(attention_mask_list)\r\n",
        "# print(input_ids_list.size())\r\n",
        "# print(attention_mask_list)\r\n",
        "# print(token_type_ids_list)\r\n",
        "\r\n",
        "# predict_dict = {\"input_ids\": input_ids_list,\r\n",
        "#                 \"token_type_ids\": token_type_ids_list,\r\n",
        "#                 \"attention_mask\": attention_mask_list}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-DRTWEvwqD7D"
      },
      "source": [
        "output = model.predict([input_ids_list, token_type_ids_list, attention_mask_list])"
      ],
      "execution_count": 242,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WOnaveSTsALg"
      },
      "source": [
        "classes = ['neutral', 'positive', 'negative']"
      ],
      "execution_count": 243,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ailyFEUGqPOn",
        "outputId": "bf7e51c0-e948-4fed-dfb1-1b815e8b0840"
      },
      "source": [
        "val = np.argmax(output, axis = 1)\r\n",
        "print(val)\r\n",
        "print(output)"
      ],
      "execution_count": 244,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[2]\n",
            "[[3.5443401e-04 3.1661220e-05 9.9961394e-01]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8FyZ1AiZqkHL",
        "outputId": "45221979-15b0-4a2f-82df-b8ca51db305b"
      },
      "source": [
        "print(classes[val[0]])\r\n",
        "print(output[0][val[0]])"
      ],
      "execution_count": 245,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "negative\n",
            "0.99961394\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8-WJADxBsOcT"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}