NLP_Project.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOUK1TaLOEE8qSKUidcWFqM",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "2bc67705c5b741f0ad67f5ad719b6b29": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_02164e85528045f097929cc66f9461d5",
              "IPY_MODEL_e2372f1f43bc46edaf26b8a215d74e78",
              "IPY_MODEL_d8130841bf144ca5862f0b89d41b0396"
            ],
            "layout": "IPY_MODEL_18b216cc41204e9c88e5631fcd712980"
          }
        },
        "02164e85528045f097929cc66f9461d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0068d13cb3254e5b99a0ce675544561c",
            "placeholder": "​",
            "style": "IPY_MODEL_609f5ad5779546eba5febf722d701add",
            "value": "100%"
          }
        },
        "e2372f1f43bc46edaf26b8a215d74e78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dd0cd9d5931a4379bb4d63bccffb16d6",
            "max": 3,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_256ce81fbf26431b98a739a57ed92656",
            "value": 3
          }
        },
        "d8130841bf144ca5862f0b89d41b0396": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_35de890b27894c3d9dacb36068785b01",
            "placeholder": "​",
            "style": "IPY_MODEL_102c10ad4fb34716b567fa746fa3c2d1",
            "value": " 3/3 [00:00&lt;00:00, 34.75it/s]"
          }
        },
        "18b216cc41204e9c88e5631fcd712980": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0068d13cb3254e5b99a0ce675544561c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "609f5ad5779546eba5febf722d701add": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dd0cd9d5931a4379bb4d63bccffb16d6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "256ce81fbf26431b98a739a57ed92656": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "35de890b27894c3d9dacb36068785b01": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "102c10ad4fb34716b567fa746fa3c2d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/folaAj/AdventureWorks/blob/master/NLP_Project.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install datasets\n",
        "import pyarrow\n",
        "if int(pyarrow.__version__.split('.')[1]) < 16 and int(pyarrow.__version__.split('.')[0]) == 0:\n",
        "    import os\n",
        "    os.kill(os.getpid(), 9)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bifFPLTRehwU",
        "outputId": "de0da76c-5714-4c39-e651-9ed6d4448b15"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.7/dist-packages (2.6.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (2022.10.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.21.6)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.3.5)\n",
            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.2.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.10.1)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Requirement already satisfied: dill<0.3.6 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.5.1)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.64.1)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.7/dist-packages (from datasets) (3.8.3)\n",
            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.18.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (4.13.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.13)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.7/dist-packages (from datasets) (3.1.0)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (4.1.1)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.1.1)\n",
            "Requirement already satisfied: asynctest==0.13.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (0.13.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (22.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (6.0.2)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.8.1)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.3.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (3.8.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (3.0.9)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.25.11)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2022.9.24)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.10.0)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2022.6)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oUwiCHsvz7hF",
        "outputId": "4d44381e-9e3e-4196-9fcf-e6d897c252a6"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.24.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.13.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.10.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.1)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.13.2)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.10.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.9.24)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.25.11)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q -U \"tensorflow-text==2.8.*\""
      ],
      "metadata": {
        "id": "JIJc7ulVlIB6"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -q tf-models-official==2.7.0"
      ],
      "metadata": {
        "id": "bPQXXmQWjjHE"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "DOakyv61eAG3"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "import tensorflow_hub as hub\n",
        "import tensorflow_text as text\n",
        "import unicodedata\n",
        "\n",
        "from datasets import load_dataset\n",
        "from transformers import AutoTokenizer\n",
        "\n",
        "tf.get_logger().setLevel('ERROR')"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Read dataset"
      ],
      "metadata": {
        "id": "LoMkwEpviU8V"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = load_dataset(\"rotten_tomatoes\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87,
          "referenced_widgets": [
            "2bc67705c5b741f0ad67f5ad719b6b29",
            "02164e85528045f097929cc66f9461d5",
            "e2372f1f43bc46edaf26b8a215d74e78",
            "d8130841bf144ca5862f0b89d41b0396",
            "18b216cc41204e9c88e5631fcd712980",
            "0068d13cb3254e5b99a0ce675544561c",
            "609f5ad5779546eba5febf722d701add",
            "dd0cd9d5931a4379bb4d63bccffb16d6",
            "256ce81fbf26431b98a739a57ed92656",
            "35de890b27894c3d9dacb36068785b01",
            "102c10ad4fb34716b567fa746fa3c2d1"
          ]
        },
        "id": "FE3bgSOvfBWW",
        "outputId": "64484637-6a52-472a-b319-8dfe23978b2d"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:datasets.builder:Found cached dataset rotten_tomatoes (/root/.cache/huggingface/datasets/rotten_tomatoes/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/3 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "2bc67705c5b741f0ad67f5ad719b6b29"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-uncased\")"
      ],
      "metadata": {
        "id": "Q3oMz41082HM"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kF5IOT2rqSgU",
        "outputId": "16f59d59-53a7-4aa3-a42e-035a13f93a1b"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['text', 'label'],\n",
              "        num_rows: 8530\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['text', 'label'],\n",
              "        num_rows: 1066\n",
              "    })\n",
              "    test: Dataset({\n",
              "        features: ['text', 'label'],\n",
              "        num_rows: 1066\n",
              "    })\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from tokenizers import normalizers\n",
        "from tokenizers.normalizers import NFKD, StripAccents \n",
        "normalizer = normalizers.Sequence([NFKD(), StripAccents()])"
      ],
      "metadata": {
        "id": "2hhNZe2YC4Wc"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "BATCH_SIZE = 32\n",
        "def normalize(x):\n",
        "  x['text'] = unicodedata.normalize('NFKD', x['text']).encode('ascii', 'ignore')\n",
        "  return x\n",
        "train = dataset['train'].map(normalize).to_tf_dataset(batch_size=BATCH_SIZE, shuffle=True).map(lambda x: (x['text'], x['label']))\n",
        "val = dataset['validation'].map(normalize).to_tf_dataset(batch_size=BATCH_SIZE, shuffle=True).map(lambda x: (x['text'], x['label']))\n",
        "test = dataset['test'].map(normalize).to_tf_dataset(batch_size=BATCH_SIZE, shuffle=True).map(lambda x: (x['text'], x['label']))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G_MyrLFier9n",
        "outputId": "4b444392-8a9b-445c-aca0-331e34074dfc"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/rotten_tomatoes/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/cache-ee551f5b56588dd5.arrow\n",
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/rotten_tomatoes/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/cache-7eb96e05f8aec454.arrow\n",
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/rotten_tomatoes/default/1.0.0/40d411e45a6ce3484deed7cc15b82a53dad9a72aafd9f86f8f227134bec5ca46/cache-43388f5fb654ff9a.arrow\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Finetune  a BERT Model For Sentiment Analysis on Rotten Tomaotoes Dataset."
      ],
      "metadata": {
        "id": "4GvgdRo9iE21"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def build_classifier_model(should_finetune):\n",
        "  tfhub_handle_encoder = 'https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-4_H-512_A-8/1'\n",
        "  tfhub_handle_preprocess =  'https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3'\n",
        "  text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')\n",
        "  preprocessing_layer = hub.KerasLayer(tfhub_handle_preprocess, name='preprocessing')\n",
        "  encoder_inputs = preprocessing_layer(text_input)\n",
        "  encoder = hub.KerasLayer(tfhub_handle_encoder, trainable=should_finetune, name='BERT_encoder')\n",
        "  outputs = encoder(encoder_inputs)\n",
        "  net = outputs['pooled_output']\n",
        "  net = tf.keras.layers.Dropout(0.1)(net)\n",
        "  net = tf.keras.layers.Dense(1, activation=None, name='classifier')(net)\n",
        "  return tf.keras.Model(text_input, net)"
      ],
      "metadata": {
        "id": "H78Pfit7iexd"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "strategy = tf.distribute.MirroredStrategy()"
      ],
      "metadata": {
        "id": "vHNMsQmY5Ker"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with strategy.scope():\n",
        "  metrics = tf.metrics.BinaryAccuracy()\n",
        "  loss = tf.keras.losses.BinaryCrossentropy(from_logits=True)\n",
        "  optimizer = tf.keras.optimizers.Adam(learning_rate=1e-5)  \n",
        "  bert_finetuned_model = build_classifier_model(should_finetune=True)\n",
        "  bert_finetuned_model.compile(optimizer=optimizer,\n",
        "                         loss=loss,\n",
        "                         metrics=metrics)"
      ],
      "metadata": {
        "id": "XcZNP4xljHnS"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "bert_finetuned_model.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YMuDZxkxmW3-",
        "outputId": "1fb1c7ab-cbc3-497b-a6df-a0a0523b0dbf"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " text (InputLayer)              [(None,)]            0           []                               \n",
            "                                                                                                  \n",
            " preprocessing (KerasLayer)     {'input_type_ids':   0           ['text[0][0]']                   \n",
            "                                (None, 128),                                                      \n",
            "                                 'input_mask': (Non                                               \n",
            "                                e, 128),                                                          \n",
            "                                 'input_word_ids':                                                \n",
            "                                (None, 128)}                                                      \n",
            "                                                                                                  \n",
            " BERT_encoder (KerasLayer)      {'encoder_outputs':  28763649    ['preprocessing[0][0]',          \n",
            "                                 [(None, None, 512)               'preprocessing[0][1]',          \n",
            "                                , (None, None, 512)               'preprocessing[0][2]']          \n",
            "                                , (None, None, 512)                                               \n",
            "                                , (None, None, 512)                                               \n",
            "                                ],                                                                \n",
            "                                 'default': (None,                                                \n",
            "                                512),                                                             \n",
            "                                 'pooled_output': (                                               \n",
            "                                None, 512),                                                       \n",
            "                                 'sequence_output':                                               \n",
            "                                 (None, None, 512)}                                               \n",
            "                                                                                                  \n",
            " dropout (Dropout)              (None, 512)          0           ['BERT_encoder[0][5]']           \n",
            "                                                                                                  \n",
            " classifier (Dense)             (None, 1)            513         ['dropout[0][0]']                \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 28,764,162\n",
            "Trainable params: 28,764,161\n",
            "Non-trainable params: 1\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "EPOCH = 10\n",
        "bert_finetuned_model.fit(x=train, validation_data=val, epochs=EPOCH)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vu-sE6WLp-PL",
        "outputId": "860673a4-56f5-422a-a212-a1db7f71f9fa"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/10\n",
            "267/267 [==============================] - 72s 240ms/step - loss: 0.5772 - binary_accuracy: 0.6733 - val_loss: 0.4987 - val_binary_accuracy: 0.7514\n",
            "Epoch 2/10\n",
            "267/267 [==============================] - 61s 230ms/step - loss: 0.4627 - binary_accuracy: 0.7761 - val_loss: 0.4387 - val_binary_accuracy: 0.7899\n",
            "Epoch 3/10\n",
            "267/267 [==============================] - 64s 242ms/step - loss: 0.3892 - binary_accuracy: 0.8216 - val_loss: 0.4299 - val_binary_accuracy: 0.7974\n",
            "Epoch 4/10\n",
            "267/267 [==============================] - 64s 238ms/step - loss: 0.3420 - binary_accuracy: 0.8450 - val_loss: 0.4268 - val_binary_accuracy: 0.8049\n",
            "Epoch 5/10\n",
            "267/267 [==============================] - 64s 240ms/step - loss: 0.2834 - binary_accuracy: 0.8750 - val_loss: 0.4744 - val_binary_accuracy: 0.8039\n",
            "Epoch 6/10\n",
            "267/267 [==============================] - 63s 236ms/step - loss: 0.2281 - binary_accuracy: 0.8984 - val_loss: 0.4846 - val_binary_accuracy: 0.8096\n",
            "Epoch 7/10\n",
            "267/267 [==============================] - 61s 229ms/step - loss: 0.1891 - binary_accuracy: 0.9250 - val_loss: 0.5259 - val_binary_accuracy: 0.8124\n",
            "Epoch 8/10\n",
            "267/267 [==============================] - 64s 239ms/step - loss: 0.1510 - binary_accuracy: 0.9368 - val_loss: 0.6260 - val_binary_accuracy: 0.7983\n",
            "Epoch 9/10\n",
            "267/267 [==============================] - 61s 229ms/step - loss: 0.1152 - binary_accuracy: 0.9544 - val_loss: 0.6380 - val_binary_accuracy: 0.8068\n",
            "Epoch 10/10\n",
            "267/267 [==============================] - 62s 234ms/step - loss: 0.0930 - binary_accuracy: 0.9649 - val_loss: 0.6790 - val_binary_accuracy: 0.8039\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7fb5b1b28b50>"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    }
  ]
}