Seq2Seq_transformer.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "transformer_workshop.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "s_qNSzzyaCbD"
      ],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/daquarti/AI/blob/main/Seq2Seq_transformer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J0Qjg6vuaHNt"
      },
      "source": [
        "# Transformer model for language understanding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7OPBSzL62Pmb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "740f432a-9aea-465c-d515-87904a737d29"
      },
      "source": [
        "# Load the Drive helper and mount\n",
        "from google.colab import drive\n",
        "\n",
        "# This will prompt for authorization.menos de 48hs de evolución\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "22Fc9Ymhn0Bn"
      },
      "source": [
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tn6Z8BL22f9Q"
      },
      "source": [
        "import pathlib\n",
        "from shutil import copyfile"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y54ZuVr12qgP"
      },
      "source": [
        "path = \"/content/drive/My Drive/NLP/GPT2/autonomous\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gOzKNCUn2h_C"
      },
      "source": [
        "data_dir =pathlib.Path (path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6OFnPCLADfll"
      },
      "source": [
        "file_name = \"conducta_diagnostico_stratificado_finetuning_shuffle_1000.txt\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mQEOKObvI7Vw"
      },
      "source": [
        "file = str (data_dir / file_name)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NUAjjhfeBXje"
      },
      "source": [
        "df = pd.read_csv (file, header=None)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2wkov21GDIRJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e9e9e3c0-935f-4705-b677-c0cce61343b3"
      },
      "source": [
        "df.columns"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Int64Index([0], dtype='int64')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c-UE-dz3Rr07",
        "outputId": "0c42f1df-6b8e-43a0-f8ef-38805bb959f1"
      },
      "source": [
        "df[0].str[17:]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0        Refiere 5 vomitos el dia de hoy asociado a cef...\n",
              "1         Paciente masculino de 39 años de edad, refier...\n",
              "2        Antecedente de broncoespasmo, refiere tos, nie...\n",
              "3        Paciente Refiere molestia en epiglotis \\n\\nAP:...\n",
              "4        Paciente Femenino de 70 años de edad consulta ...\n",
              "                               ...                        \n",
              "33180    Refiere T*C 38.4*C y mialgias.Congestión nasal...\n",
              "33181    LITIASIS RENAL IZQUIERDA [DIAGNOSTICO] LOCOM  ...\n",
              "33182    DOLOR EN FLANCO DERECHO...VARIOS DIAS DE EVOLU...\n",
              "33183    Refiere TA 140 90 en dos episodios, refiere ho...\n",
              "33184    Paciente Masculino de 24 años de edad consulta...\n",
              "Name: 0, Length: 33185, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ucFSe-0gDr7U"
      },
      "source": [
        "lista_2 = df[0].str[17:].to_list()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CvFcHlWsFD0g",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "25294b89-87f6-4756-86ef-911d1e32a4df"
      },
      "source": [
        "lista_2[1].split( ' [CONDUCTA] ')[1][:-15]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'Indico diclofenac 50 mg cada 8 hs ante dolor'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-aUPhZuCGk_e"
      },
      "source": [
        "epicrisis_list = []\n",
        "conducta_list = []\n",
        "for x in lista_2:\n",
        "  epicrisis = x.split( ' [CONDUCTA] ')[0]\n",
        "  conducta = x.split( ' [CONDUCTA] ')[1][:-15]\n",
        "  epicrisis_list.append (epicrisis)\n",
        "  conducta_list.append (conducta)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 416
        },
        "id": "2VEh0kVYTrZa",
        "outputId": "ebcbf4b7-cf29-4b88-b2cd-ee5fb48cd489"
      },
      "source": [
        "data = {'epicrisis': epicrisis_list, 'conducta': conducta_list}\n",
        "pd.DataFrame.from_dict(data)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>epicrisis</th>\n",
              "      <th>conducta</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Refiere 5 vomitos el dia de hoy asociado a cef...</td>\n",
              "      <td>Indico metoclopramida 10 mg cada 8hs. Abundant...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Paciente masculino de 39 años de edad, refier...</td>\n",
              "      <td>Indico diclofenac 50 mg cada 8 hs ante dolor</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Antecedente de broncoespasmo, refiere tos, nie...</td>\n",
              "      <td>Continuar manejo con salbutamol, pautas de ala...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Paciente Refiere molestia en epiglotis \\n\\nAP:...</td>\n",
              "      <td>ECO partes blandas tiroides</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Paciente Femenino de 70 años de edad consulta ...</td>\n",
              "      <td>PARACETAMOL 1 GRAMO CADA 8 HS POR 3 DIAS. CALO...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33180</th>\n",
              "      <td>Refiere T*C 38.4*C y mialgias.Congestión nasal...</td>\n",
              "      <td>['Mantener buena hidratación (Aproximandamente...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33181</th>\n",
              "      <td>LITIASIS RENAL IZQUIERDA [DIAGNOSTICO] LOCOM  ...</td>\n",
              "      <td>AINES/ ANTIESPASMODICOS</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33182</th>\n",
              "      <td>DOLOR EN FLANCO DERECHO...VARIOS DIAS DE EVOLU...</td>\n",
              "      <td>DIETA Y SERTAL\\nCONTROL EN 24 HS\\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33183</th>\n",
              "      <td>Refiere TA 140 90 en dos episodios, refiere ho...</td>\n",
              "      <td>continuar tratamiento habitual, evitar tabaco.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33184</th>\n",
              "      <td>Paciente Masculino de 24 años de edad consulta...</td>\n",
              "      <td>Bronaxf flex</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>33185 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                               epicrisis                                           conducta\n",
              "0      Refiere 5 vomitos el dia de hoy asociado a cef...  Indico metoclopramida 10 mg cada 8hs. Abundant...\n",
              "1       Paciente masculino de 39 años de edad, refier...       Indico diclofenac 50 mg cada 8 hs ante dolor\n",
              "2      Antecedente de broncoespasmo, refiere tos, nie...  Continuar manejo con salbutamol, pautas de ala...\n",
              "3      Paciente Refiere molestia en epiglotis \\n\\nAP:...                        ECO partes blandas tiroides\n",
              "4      Paciente Femenino de 70 años de edad consulta ...  PARACETAMOL 1 GRAMO CADA 8 HS POR 3 DIAS. CALO...\n",
              "...                                                  ...                                                ...\n",
              "33180  Refiere T*C 38.4*C y mialgias.Congestión nasal...  ['Mantener buena hidratación (Aproximandamente...\n",
              "33181  LITIASIS RENAL IZQUIERDA [DIAGNOSTICO] LOCOM  ...                            AINES/ ANTIESPASMODICOS\n",
              "33182  DOLOR EN FLANCO DERECHO...VARIOS DIAS DE EVOLU...                 DIETA Y SERTAL\\nCONTROL EN 24 HS\\n\n",
              "33183  Refiere TA 140 90 en dos episodios, refiere ho...    continuar tratamiento habitual, evitar tabaco. \n",
              "33184  Paciente Masculino de 24 años de edad consulta...                                       Bronaxf flex\n",
              "\n",
              "[33185 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AOpGoE2T-YXS"
      },
      "source": [
        "<table class=\"tfo-notebook-buttons\" align=\"left\">\n",
        "  <td>\n",
        "    <a target=\"_blank\" href=\"https://www.tensorflow.org/tutorials/text/transformer\">\n",
        "    <img src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" />\n",
        "    View on TensorFlow.org</a>\n",
        "  </td>\n",
        "  <td>\n",
        "    <a target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/docs/blob/master/site/en/tutorials/text/transformer.ipynb\">\n",
        "    <img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />\n",
        "    Run in Google Colab</a>\n",
        "  </td>\n",
        "  <td>\n",
        "    <a target=\"_blank\" href=\"https://github.com/tensorflow/docs/blob/master/site/en/tutorials/text/transformer.ipynb\">\n",
        "    <img src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />\n",
        "    View source on GitHub</a>\n",
        "  </td>\n",
        "  <td>\n",
        "    <a href=\"https://storage.googleapis.com/tensorflow_docs/docs/site/en/tutorials/text/transformer.ipynb\"><img src=\"https://www.tensorflow.org/images/download_logo_32px.png\" />Download notebook</a>\n",
        "  </td>\n",
        "</table>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XFG0NDRu5mYQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e4226ded-cca9-40b3-b332-e1e84a9f6a16"
      },
      "source": [
        "!pip install -q tfds-nightly\n",
        "\n",
        "# Pin matplotlib version to 3.2.2 since in the latest version\n",
        "# transformer.ipynb fails with the following error:\n",
        "# https://stackoverflow.com/questions/62953704/valueerror-the-number-of-fixedlocator-locations-5-usually-from-a-call-to-set\n",
        "!pip install matplotlib==3.2.2"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: matplotlib==3.2.2 in /usr/local/lib/python3.6/dist-packages (3.2.2)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.2.2) (1.3.1)\n",
            "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.2.2) (1.18.5)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.2.2) (2.8.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.2.2) (2.4.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib==3.2.2) (0.10.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.1->matplotlib==3.2.2) (1.15.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JjJJyJTZYebt"
      },
      "source": [
        "import tensorflow_datasets as tfds\n",
        "import tensorflow as tf\n",
        "\n",
        "import time\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fd1NWMxjfsDd"
      },
      "source": [
        "## Setup input pipeline"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T0XnXQkM2ZNq"
      },
      "source": [
        "dataset = tf.data.Dataset.from_tensor_slices((epicrisis_list, conducta_list))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bo65COjy3m7-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6a4559cc-1e8f-43db-eb07-f2d9d729950e"
      },
      "source": [
        "[x for x in dataset][5]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(<tf.Tensor: shape=(), dtype=string, numpy=b'CONSTIPACION 7 DIAS DE EVOLUCION//FIEBRE//VOMITOS DE MENOS DE UN DIA DE EVOLUCION, HACE 3 DIAS CON BAREX [DIAGNOSTICO] DIGES   Dispepsia indigesti\\xc3\\xb3n   (318)'>,\n",
              " <tf.Tensor: shape=(), dtype=string, numpy=b'ANTITERMICOS// BAREX-- SI NO TIENE DEPOSICIONES: IR A LA GUARDIA'>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RCEKotqosGfq"
      },
      "source": [
        "Create a custom subwords tokenizer from the training dataset. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KVBg5Q8tBk5z"
      },
      "source": [
        "tokenizer_en = tfds.deprecated.text.SubwordTextEncoder.build_from_corpus(\n",
        "    (en.numpy() for pt, en in dataset), target_vocab_size=2**13)\n",
        "\n",
        "tokenizer_pt = tfds.deprecated.text.SubwordTextEncoder.build_from_corpus(\n",
        "    (pt.numpy() for pt, en in dataset), target_vocab_size=2**13)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4DYWukNFkGQN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9728a2a6-4cfe-4abf-c9ab-58b8ddeaa0a1"
      },
      "source": [
        "sample_string = 'paciente con disnea y cefalea y dolor de panza'\n",
        "\n",
        "tokenized_string = tokenizer_en.encode(sample_string)\n",
        "print ('Tokenized string is {}'.format(tokenized_string))\n",
        "\n",
        "original_string = tokenizer_en.decode(tokenized_string)\n",
        "print ('The original string: {}'.format(original_string))\n",
        "\n",
        "assert original_string == sample_string"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tokenized string is [354, 5, 7282, 3, 929, 3, 119, 1, 1822, 2399]\n",
            "The original string: paciente con disnea y cefalea y dolor de panza\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o9KJWJjrsZ4Y"
      },
      "source": [
        "The tokenizer encodes the string by breaking it into subwords if the word is not in its dictionary."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bf2ntBxjkqK6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "868054af-e28b-4ae4-d57f-13928d3f30c7"
      },
      "source": [
        "for ts in tokenized_string:\n",
        "  print ('{} ----> {}'.format(ts, tokenizer_en.decode([ts])))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "354 ----> paciente \n",
            "5 ----> con \n",
            "7282 ----> disnea \n",
            "3 ----> y \n",
            "929 ----> cefalea \n",
            "3 ----> y \n",
            "119 ----> dolor \n",
            "1 ----> de \n",
            "1822 ----> pan\n",
            "2399 ----> za\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bcRp7VcQ5m6g"
      },
      "source": [
        "BUFFER_SIZE = 20000\n",
        "BATCH_SIZE = 64"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kGi4PoVakxdc"
      },
      "source": [
        "Add a start and end token to the input and target. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UZwnPr4R055s"
      },
      "source": [
        "def encode(lang1, lang2):\n",
        "  lang1 = [tokenizer_pt.vocab_size] + tokenizer_pt.encode(\n",
        "      lang1.numpy()) + [tokenizer_pt.vocab_size+1]\n",
        "\n",
        "  lang2 = [tokenizer_en.vocab_size] + tokenizer_en.encode(\n",
        "      lang2.numpy()) + [tokenizer_en.vocab_size+1]\n",
        "  \n",
        "  return lang1, lang2"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tx1sFbR-9fRs"
      },
      "source": [
        "You want to use `Dataset.map` to apply this function to each element of the dataset.  `Dataset.map` runs in graph mode.\n",
        "\n",
        "* Graph tensors do not have a value. \n",
        "* In graph mode you can only use TensorFlow Ops and functions. \n",
        "\n",
        "So you can't `.map` this function directly: You need to wrap it in a `tf.py_function`. The `tf.py_function` will pass regular tensors (with a value and a `.numpy()` method to access it), to the wrapped python function."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mah1cS-P70Iz"
      },
      "source": [
        "def tf_encode(pt, en):\n",
        "  result_pt, result_en = tf.py_function(encode, [pt, en], [tf.int64, tf.int64])\n",
        "  result_pt.set_shape([None])\n",
        "  result_en.set_shape([None])\n",
        "\n",
        "  return result_pt, result_en"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6JrGp5Gek6Ql"
      },
      "source": [
        "Note: To keep this example small and relatively fast, drop examples with a length of over 40 tokens."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2QEgbjntk6Yf"
      },
      "source": [
        "MAX_LENGTH = 40"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c081xPGv1CPI"
      },
      "source": [
        "def filter_max_length(x, y, max_length=MAX_LENGTH):\n",
        "  return tf.logical_and(tf.size(x) <= max_length,\n",
        "                        tf.size(y) <= max_length)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9mk9AZdZ5bcS"
      },
      "source": [
        "train_dataset = dataset.map(tf_encode)\n",
        "train_dataset = train_dataset.filter(filter_max_length)\n",
        "# cache the dataset to memory to get a speedup while reading from it.\n",
        "train_dataset = train_dataset.cache()\n",
        "train_dataset = train_dataset.shuffle(BUFFER_SIZE).padded_batch(BATCH_SIZE)\n",
        "train_dataset = train_dataset.prefetch(tf.data.experimental.AUTOTUNE)\n",
        "\n",
        "\n",
        "#val_dataset = val_examples.map(tf_encode)\n",
        "#val_dataset = val_dataset.filter(filter_max_length).padded_batch(BATCH_SIZE)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_fXvfYVfQr2n"
      },
      "source": [
        "#pt_batch, en_batch = next(iter(val_dataset))\n",
        "#pt_batch, en_batch"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nBQuibYA4n0n"
      },
      "source": [
        "## Positional encoding\n",
        "\n",
        "Since this model doesn't contain any recurrence or convolution, positional encoding is added to give the model some information about the relative position of the words in the sentence. \n",
        "\n",
        "The positional encoding vector is added to the embedding vector. Embeddings represent a token in a d-dimensional space where tokens with similar meaning will be closer to each other. But the embeddings do not encode the relative position of words in a sentence. So after adding the positional encoding, words will be closer to each other based on the *similarity of their meaning and their position in the sentence*, in the d-dimensional space.\n",
        "\n",
        "See the notebook on [positional encoding](https://github.com/tensorflow/examples/blob/master/community/en/position_encoding.ipynb) to learn more about it. The formula for calculating the positional encoding is as follows:\n",
        "\n",
        "$$\\Large{PE_{(pos, 2i)} = sin(pos / 10000^{2i / d_{model}})} $$\n",
        "$$\\Large{PE_{(pos, 2i+1)} = cos(pos / 10000^{2i / d_{model}})} $$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WhIOZjMNKujn"
      },
      "source": [
        "def get_angles(pos, i, d_model):\n",
        "  angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n",
        "  return pos * angle_rates"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Rz82wEs5biZ"
      },
      "source": [
        "def positional_encoding(position, d_model):\n",
        "  angle_rads = get_angles(np.arange(position)[:, np.newaxis],\n",
        "                          np.arange(d_model)[np.newaxis, :],\n",
        "                          d_model)\n",
        "  \n",
        "  # apply sin to even indices in the array; 2i\n",
        "  angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
        "  \n",
        "  # apply cos to odd indices in the array; 2i+1\n",
        "  angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
        "    \n",
        "  pos_encoding = angle_rads[np.newaxis, ...]\n",
        "    \n",
        "  return tf.cast(pos_encoding, dtype=tf.float32)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1kLCla68EloE",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "outputId": "dd0dbb92-1ed9-4340-e9ba-d8298fab6208"
      },
      "source": [
        "pos_encoding = positional_encoding(50, 512)\n",
        "print (pos_encoding.shape)\n",
        "\n",
        "plt.pcolormesh(pos_encoding[0], cmap='RdBu')\n",
        "plt.xlabel('Depth')\n",
        "plt.xlim((0, 512))\n",
        "plt.ylabel('Position')\n",
        "plt.colorbar()\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1, 50, 512)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEKCAYAAAD+XoUoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOydd5xU1fmHn/femdneKywsvSpSRBCxYe8aE1s0lhhNYokaE2OKJjHFmKIxicaoMZpmjwb8YbCAoiDFQkfaUndh2b47u9PunfP7494ZZpeFHWAXWTzP53Oc2++ZdThz5/ue9/uKUgqNRqPRfD4wPusOaDQajebgoQd9jUaj+RyhB32NRqP5HKEHfY1Go/kcoQd9jUaj+RyhB32NRqP5HNGjg76IbBKR5SKyREQ+dLfli8ibIrLOfc3ryT5oNBrNZ4WIPCUiO0VkxR72i4j8QUTWi8gyEZmQsO8ad5xcJyLXdFefDsaT/jSl1Dil1ER3/W7gbaXUMOBtd12j0WgOR54GztrL/rOBYW67EfgzOA/HwI+BycAk4Mfd9YD8Wcg7FwLPuMvPABd9Bn3QaDSaHkcpNReo38shFwJ/Vw4LgFwR6QOcCbyplKpXSjUAb7L3L4+k8XTHRfaCAt4QEQX8RSn1OFCilNru7t8BlHR2oojciPPNR0Z62tFtKp1xowawZPVmxo0sZ+snKxlwxCCWbGkiIy+Hfi3baWgMUTr+CJat2443LZ0jCk2UbbGmxUNbQx1FfUsoU01Ubawh1RAKRw5kY6vQsLMO0+ujsCiXmuo6VDRKVkE+QwrSCG2toL4ugK0gN91LZnkJbd5sNlW3UJKfTkEKWDU7aN3ZQosVBSDdNMjITSGlqAA7LYfKT1biEyEjxSQ1Nw1vXh7R1CxawjYNrWHaAhZWKIgdCUPUZsKQIqKtzYRb2oi0hgmHo4SiClspooAApoBHhIKyXKxACCtoYYdswtEoVpT4sbF8awPIPmIkYVsRtqKELZuwFSVqK6dFo6io7TZneWypD/F4EdMLhokyTOcVIarAVqCU0681FdsRERBBcF8NY9e6YSBiICJ4U0yUApRCuddw1kE5/yGWKa5UlMysVEQEAQwR3NsgCIbg7HO3VVXWxd+1ci7Q4RO5a33IoD5I7PPm/kfctfbrDqvXb0vmMw/AkcP6d7pdZPdty9dsSfq6AEeNLO/82p1sW/pp8tcet4frdsaSfbiuc+0B+3Dtzclfd1T76y5ZvRkVqKtVShUlfZEOGNn9FFYwqWNVoG4lkHjw4+44lyxlwNaE9W3utj1tP2B6etA/XilVKSLFwJsi8mniTqWUcr8QdsP9wz0OcPRRR6jl5mTmzXuUnCk3Mff9R/hOxigeeekJCm6ZxXFfOof7Z/+MV2as43vz5tH3gvspPeJoFl6fgd1Ux0nvFvDRi//i8nvv4P7wa/z0K08wPNPHtS89wVcWpfLKI0+TWTqQa288lz8/9G8iwVZOuPpSXrrySDbe9hX+9c/lNEWiXDyqlOP++B2Wlp3C1Q+9x51XjOXqwSa1j/2ChX+ay5yaNgAm5KQy+fxhDLnxGlqOPJsfZo+mb4qHKQNzGHHRUfT90pdoHXkK725u4rkPt7JsWTU7N6zFv2MTVtDPohdvpG3hG1S+u4SqxZVs3tLMprYI9WGbcFRhCuR4TQp9JtfcdSG1yzZQt6aWhopGKv1hakI2DRGbgB3Fdv+6PkM4+6U32NIUZFNtK5vrWqmqa6O1OURbU4hgW5hQSyPhtiasgB8r2Mq875RjFpRi5hVDRi7RlCyiablEzBTaIlFaI1EClqI5ZHHK5T/B9PowPD4MjxfD48NMScP0+OLLhseHx+el37ACrHAUK2JjRWxsK4oViRK1oth2FNuKErWj2JZF1Aoz5eQR+DwGPo/pvJoGKR7D3da+3fPjp1FR2/kMuV9ezrLzGnVfAR792w8wBEwRDBFMw/lS6bguAgbC0Rfc1e5ae2PGGw8Buwb52E9qcTcYCSP0gGm3dnm9RN5+90+dDvBGJxuLT7gl6eu++/4j7dY7u0eM/Kk3J31dgPfnPZr0sbnH3ZT0sfM6XDdnyk1Elvwt+W+NzrCCeEZckNShkSV/CyZI172CHpV3lFKV7utO4BUcbara/fmC+7qzJ/ug0Wg0+4QIYphJtW6gEkj8WdjP3ban7QdMjw36IpIhIlmxZeAMYAUwHYhFoq8B/ttTfdBoNJp9R9xfrF23bmA6cLU7i+dYoMmVv2cBZ4hInhvAPcPddsD0pLxTArzi/pz1AP9WSv1PRBYDL4jI9cBm4NIe7INGo9HsG+6TfvdcSp4FTgYKRWQbzowcL4BS6jFgJnAOsB5oA65z99WLyM+Axe6l7lNK7S0gnDQ9NugrpSqAsZ1srwNO3ZdrrdoZZsp3r+adkZOZcuvDLDjmRC4dU8yl851v2unn53HbTav50S/O5ew/LyTYVMvf7ziBOWefQeTl11g281eUTzmPB84czNsjniVgK07/+hQWpo7m3ddfRkVtRp94DC/NXENbXRUDjjufu04bTvTNJ1k6fS01IZsJuamMunQi1rhz+ef/1jF+fB/OGFpAdNG/2fTmSpY3hQhHFf3TvAwemkfZieNg2GRW1QTI9BgMyvBSPKaYwolHoMrHsKU5zEdbG9m4rZnm2gaCDdVYQT8A4YqVNK7dSuPGBhq3+6kJ2fitKOGoI9D7DCHDNMj3mbRW1tK2009bbYCmoIXfitJqO8fG9HxTnNYQiNDQFqauNUydP0woYBEOWIRDFpFgG3Y4gB0KELXCqKiNkZ6FkZqB+NKIelJRvnSUJ4WwpZyAcFQRtqOErChixn7yGohhYnh9GO5PYMPjQwwT0+NBRLBj2r0dRUWdQLKKKqLKeVVKEY2quHZuGoJpGM6riLveSUuIkqpodO+fT9u9dpJ6/q7rdq3n7wudBXb3h870fDmAi3dTt3olAojZPYO+UuqKLvYroNMAiVLqKeCpbulIAj0dyNVoNJrehQhGNz3pH4roQV+j0Wg60F3yzqGIHvQ1Go0mkW7U9A9F9KCv0Wg0CQiC4fF+1t3oMXqFy2aopZG3TwnyxrZmZp9r8srqGo5dOJfXHnmSn//set468csck5dG7bW/ZOFzLzDp0ksYPe8RXlldw+2PfICybe65/hhqH7idmZXNnN0/mz7f/infe3EZtWsXU3zEVO654AgqP55DRlF/zjltKMemN7Ly8ddY3BAkx2swYUoZRRdfyVsbG3l38VYun9ifstaNVL4+m7UraqgOWaSZwuhsH/2OG0jG5FPYYeQyb3M9JSke+g/MpXTiUFLHTKHBV8CS7S18vLmB+u0ttO7cQri1CQDTl0Zw0wYa11fRvK2ZmpBNs+UkWoETxM30GOR43UDujjr81a201QdoikTjAV87IfPUFMFnCPXBCDubQ9T5QwQDEcKBCOGQ5SRIhQJYYSeIG7Ui2JEwRkY2RkYWUV8aUW8aypNCROEEcKMKy4a2iE3QisaDtrEgriQGcc1YMFcwPQYqihOwjSpsO+oGbVU8OUu5QVwVtVG2HQ/U+kwnASuWmNUxkGuItAu07i0xCzoPfu6JfYmJxu6XTGLW/vB5DrIeFA7uPP2Djn7S12g0mg701gE9GfSgr9FoNImIdNuUzUMRPehrNBpNAsLh/aTfKzT9/uV9+OVxt3DvHy/jwYnX893vnsQxP3yTPuNP4/rqV3m1ooEvT/8pF/98NukFfXntm5N57uZ/MjwzhY3vT+eocy/gqvwaZjz8Hvk+kxPvv4RnNipWvv0evowcTj/rSE5Or8UOBxg8eQq3nTCQxmf/xIJ52/BbUY7NT2PUV6ZRlX8kT83fRNXqT5k2MIfA3FfY+NYG1vrD2AoGpvvoP6GUPtOOxRpwNB9VtTBn9U6GZnrpM6EPOePGEelzBOsbgny4uYGqrU207NxO2N9A1AojhokvI4eGtVtp2txMfV0gnpiVaJwWS8xKz0+jZbuftroA9WGbpohN0NXbExOzfIaQZhrU+8PUt4ZpjCVmhWwiIQsr4McOB4hGXD0/lpyVkYXyZaC86eBNJepJIRhLzLIVQStK0IrSFrHbGa2JYWIkJGUZHh+GIZimgWka8cQs24qiosS1/Li2H9P0bUfXjxmtmYbgSdDw2+n6Ipiu2N3diVkSv27yiVl70vM7O+ZA0YlZ3YwYmB5fUq03op/0NRqNJhE5vJ/09aCv0Wg0CQh6nr5Go9F8rjicB/1eoelnN1RSmurh0eFfBWDZNQ+wbs4rzP7VOTx85aNcfWI5D1sT2Dx/Bt++8xKqbr+SxQ1Brrj3LLL7DefpGybxyc3fZWlTkAtPGUjrOXfwu2eX4q/exMBjp/Gj04ay/c+/pWDoBL55/ijKKz9g6ZPvs7olRP80L0d+YRS+067m1TU1rPxkO83b1pK27j02/PcDlm1poj5sk+8zGVmaQf+TR+MbP431zYp31tVSWdFA3zHFlE4ejWf0sVSGTD7e3szSTfXUV/sJNOyIz9H3pGWSklNI4/pqmrc1syMYm6O/y2gt0+Po+TmpHjJK0mmtbqWlKURTJEowqgjYu4zZYuf4DCHVkPgc/VDAIhSIEAlZRIJB7LAzR9925+nHtHRJy0L50lDeFKLeNEJWNK7nh21FW8SmLRIlZEfjRmsx47W4nu/O2TdMA8NjIIYQtZyCKUopp2BKB6O1mOFbrHVptObO0TcMiev5Xc3Rj7EnPb8jyc6t70r3j13nUNXzP2sOia7refoajUbzeULLOxqNRvO5QUQwvL1zZk4y6EFfo9FoEtGGaxqNRvP5Qg/6nzE7qv1ct+NDsi/4Lf5Fj1N4+5+ZdsP1tN56Ga12lAmvv865F/2aISdfxN19qvjB00v44sgCgl/9BZcPqqB87mP89K2NHJOXyvgHf8K1M1azaf4scspHccsXj6Rs9f8x/YkFjPvp9Vx1ZCEbbr+D9ysaMEU4bkQ+A666lCWBLJ599xNq1nxE1Aqzc8YrVMzdwtZABJ8hDM/0UT61H3knnExj7hDeX1XDh2tqaKisos/EgWSMm0IgfzArNjUxf10ttZUttNZsIdTS4CRCeXz40rNJLyij8aMmqlvCNER2BXFNgUyPQbYbyM0oySCzJIP6dQ3Uh50ErsTqWtA+iJtmGtS3hmhpDRMKRggHLCIha5fRmpuYFU0IoCqfY7KmvOlYGITtqNucIG7IjhKybKdyVoLBmtkhiGt6DEyP4QRKPUY8Ecu2VNx4LZaYlWi01i6Q2yExq12ClpuYFauctbcgbiwxCzoP2MZITMza3yBudxutHQx6QRcPCkZv+J+1n/SK2TsajUZzsBARxEiuJXm9s0RkjYisF5G7O9n/kIgscdtaEWlM2Gcn7JveHe+vVzzpazQazcHENLvneVhETOAR4HRgG7BYRKYrpVbFjlFK3ZFw/K3A+IRLBJRS47qlMy76SV+j0WgSEbrzSX8SsF4pVaGUCgPPARfu5fgrgGe74V3skV4x6JcUpDH+NyspO/pUTp0lmClpvH4aPPLcKr7zyBWc9Nv5WMFWXv3+yfzvzG/hM4RTXvw1lz22kAdPKWbmLc8QjirO+e6pvCUjePOVeQCMPX0K143MYNn9TzC3to2fnTsa+78Pseg/q9kRtJiQm8qY646nbex5PLFgMxs/WUNbXRVpeaWsn7GUpU0hAraib6qHYaMK6X/aRBg5lU92tPLGyh1Ub2nEX72JoinjiQ4az4aGEIs2N7BhUyNN1bUEG6qxgn4AfBk5pOWVkpWfSeN2PzuCVjuNPs004kZrOfmpZBank1GaS1PQoikSpdWO7ma0lmi2lukR6mJGawGLcMgiEmzDDgewQ4F4QlQ0sisxKupNR/nSiXpTCcWSsqKKsB0l5BqtxV5jGr5htE/OMj0eTNNJynKSs5wCKlHb1fLdBK1YYlaing+OTm6K7LFwSiypynATtPZGop4Pe07Miun5iexr0tDe/mElXutA/gEebkZrh0RiFjGXzW4b9MuArQnr29xtu99XZAAwCJidsDlVRD4UkQUictF+vqV2aHlHo9Fo2tH1A0QChSLyYcL640qpx/fzxpcDLymlEp9OBiilKkVkMDBbRJYrpTbs5/UBPehrNBpNe1x5J0lqlVIT97K/EuifsN7P3dYZlwM3J25QSlW6rxUi8g6O3n9Ag36vkHc0Go3mYNKN8s5iYJiIDBIRH87AvtssHBEZCeQBHyRsyxORFHe5EJgKrOp47r7SKwb9QMkA1r/7GssfPIf5f3+G6X+8gScnX88XRxbw+sRv8skrz3LFLVeR8cidzNjWzFdvOY7H/UNY8t//sP62G3hrZytfOLoPObf/jrv//hH1FUspn3Q6D3/xKBqf+BlvvbsFgPHhtXz0+5ksbghSmuph4lmDyf3i1/jPp7W898EWGjatwPD4yB86gZVr6tkRtMjxGozJT2PAqSNJn3IOmyMZvL22hg3r62jcupZgUw2+o05kB9ks3NbEB+tqqdvhzNGPG62lOkZrGYWl5BalUxmwaLaiuxVDz/eZFKZ4yCjOILNvFpllRdSHbVrt6G5Ga6Y4Wn6qYZDpcVpba5hwIOKarYWxAv7diqHH9HzANVtL21U0pZ3R2q4WiNi7jNU8Pqd53VdXzzdNI15IJT5P325vvJZospbYErV8Xwdt30iYo29K8kZrKmp3abR2IHP0d11jz3P0u1vP780cKno+OH0xPZJU6wqllAXcAswCVgMvKKVWish9InJBwqGXA88ppVTCtlHAhyKyFJgD/Cpx1s/+ouUdjUaj6UB3OpUqpWYCMztsu7fD+k86OW8+MKbbOuKiB32NRqNJQNzZYIcretDXaDSaDuxDILfXoQd9jUaj6cDhPOj3ikDups07+MEv7+CdkZOZctXVFPzyBja1RThpwSxu+dE/KJ9yHo9NtPjLA7O5cEAOafc8xs8eeh1fRg7PvrCKsTmpHPfYvdw+41PWzH6d7H7Duenyoxi+ZTYLfvc2m9oiTC1Io+Kh3/LOsp0AnDgsn2E3fJlV0pen3t7AjpUfYYcDZPcbzpCjSlnrD2EKDM/0MWjaAIpPO5Xm4tG8u6me91ZUU7NxG221VaioTaB4BMuqW3l/XQ07tzXTvH0TwaZaolYYw+MjJSuP9IIycoszGNgni1rXQM1WToJVmilkewyKUkwyStLJ6ptJZlkRGWVFNEU6C+I656S6AeBMj5CZ4iHYFiHkGq3Fgrh2KIAdDmJ3qFYFoLzpRMRDyIoSdKtmBSNR2iK7ErOCdpRA2HYTsXY3WosFb02PgWE6CVq2pZwA7h6M1oB2/ejMaC1eTUuIJ2bFfpJ3FlRNTMzaW3WrRKO1jtv2xL4EcXsyYHmwKmbtwxz23ok47zGZ1hvRT/oajUaTgOA8nByu6EFfo9FoEpHD21pZD/oajUbTgd5cXL4resVvGG96Fjetfpw3tjUz+2x46ImPufuxK5n6+48JNdXy+k9OZ+bx12GKcMbMh7nojx9Qu3YxJ15+Pn4rysU/OJ0308Yz/fl3UVGbo88+gW8ekcnSn/6Rt3a20j/Ny+TrjuH9Z5dT5Rqtjb3xJAITv8DDcyuo+PhTWmu2kpZXSv8jR/OVKQMI2Ir+aV5GjSlmwNmTYcwpfLi9ldeWbWf7xnr81Zuwgn4Mj4/1DSHmVdSxtqKBhsodnRqtZRfmUFSSyVH9c3czWsv2mHGjtaw+mWSU5pJZVoSnqMxNzGpvtBYrnhIzWsvxmqRkpxAOWE5iVoLRmh0O7ma0FiNmtBZMMFqLJWTFjNYCYafF9fwORmuGx4gbrcUKqezJaC2xD3HTtw7JWfEkLdOI6/hew3C0/Q7/UGOJWXvS87syWjOkezX4Q9Vo7bPmUOu6Y7iWXOuN9Hi3RcQUkU9E5DV3fZCILHQLCjzvpiZrNBrNoUFsckASrTdyML6rbsNJP47xAPCQUmoo0ABcfxD6oNFoNEkiGKaRVOuN9GivRaQfcC7wpLsuwCnAS+4hzwDd4hGt0Wg03YHoJ/0D4vfAXUDUXS8AGl0TIth7QYEb3eIBH5akBPnx7S/z40ev4MFJN3LlsWX8feR1LH31OW77/vXIfdfz2vYWvn7vmfxqe1+W/PclBp94IS9cPZ7Lpw3E880H+O6Ti6mvWMrgqWfxyCVHUfPwPcycsxlT4LSp/eh/07dZ3BCgf5qXY78wguxLbuLZFTt5f95mGjatwPSlUTRyImccW87ZQ/PJ95mMK81k0FljSD3ufNYHU5m5qpoNa+to3PIpwaYaxDBJyyvhg62NLFhXS21Vs1sMvR5wjNZS80rIKu5DfkkGR5blMKIoczejtaIUk6J0LxnFGWT1yyGrvISU0lI8peW7zdGPafkZpmOyluM18aV7Scn2EQpECAcCWAE/kaDfNVoL72a0FsOZn++YrIUsRUtod6M1f9CiLWzv1WjN9LivrsafrNFarEh7MkZrhtHecG1PRmuJdGW0Ftvccd5+IgdqtNYdWvzB1PO7e276oabnx+jOGrmHGj026IvIecBOpdRH+3O+UupxpdREpdTEwoKCbu6dRqPRdI4InScDdtJ6Iz05ZXMqcIGInAOkAtnAw0CuiHjcp/29FRTQaDSaz4TeOqAnQ4896Sulvq+U6qeUGojjFT1bKXUlji/0l9zDrgH+21N90Gg0mn1FSO4pv7d+MXwWyVnfA54TkZ8DnwB//Qz6oNFoNJ0iAj5tw3BgKKXeAd5xlyuASftyfu2KNVwyYTyPDLkWHy8x7H9vcM75P2bMeZdyT9rH3P3YYq48toz6a+/nwev/SGbpQJ6643gqv3M1E5/8Pef/awnr332NgqET+PG1R9Nv8T958Y9zqQpanN8vm3Hfv475dj98hnDyhFKG3vwN5vmzeGrWx2xf/gF2OEDh8GMYO7GMqyb0o7B6CUdmpzDkjCEUnXEONblDeXNFNfOX76B240ba6hyjtZSsfDJLBvHWqmp2bG6keXsFwaZaJzjpSyM1p5CMonLySjIZ0T+XMWU5DM1PZ6ZrtJbpMcjzmhSlmGT1zSS7n1MtK6NvMZ6Scsgp3i2I6zN2Ga3leA3SfCapeamk5aUSCkR2VcuKhB2jtYgTzO0skOtUyooSsnavltWakJgViNjtgrimxzFYixmtiUg8Scs0jd2M1qJWGGW3D+LCLtO1dklZHiMevPUmJGglGmAlBnH3x2gt8QFuf4zW4ud2YrTW3UHcZO/fPdf7nARxxTH5O1zRNgwajUaTgHB4a/p60NdoNJpEpPfq9clw+ApXGo1Gsx84T/pGUi2p64mcJSJrXOuZuzvZf62I1IjIErd9LWHfNSKyzm3XdMf76xVP+raCAf97g7PPvRv/oscZcudrZBT1Z/73pvBY30mMykph8uuvMOae2bTVVXHXfbcy7qO/8eu/fkz2ZZnMf+lFfBk5XPrlk/hi9k7evftvzKsLMDYnlWO/dyY14y7m3r9/zJ3FmYy/40K29p/KAy8tZ+OHHxNsqiGrzxAGTxjJ144byHCjjtrpLzDi2DL6n38q1uhTmLuugekfVbK9Yif+6k3Y4QCe1EwySwZSWF5CxYZ6mqoqaaurwg4HEMPEl5FDRlE5uUUZlPXN4qj+OYwszKA0w/lfkqjn5xRnkN0vi+zyYrLKSzBLyjGKy7GzSnYzWktzk7IyPQbZXpM0V89PzUvFCvixwwH3tfPCKYmELOUYrlnRBD0/SshyCqfEErNiRVQMj29X0ZSY2ZopcX3fMATTI9hWlKgdxbaseOGUzhKzYrQzXBPBa+zS8WNGa6bs/pN8b3q+Eytob7S2p8IpHXX+feWzKJxyqOv5hzrd9aQvIibwCHA6TjLqYhGZrpRa1eHQ55VSt3Q4Nx/4MTARUMBH7rkNB9In/aSv0Wg0CRiyKwO8q5YEk4D1SqkKpVQYeA64MMmunAm8qZSqdwf6N4Gz9utNJaAHfY1Go+mAM0Os6wYUxuxi3HZjh0uVAVsT1vdkPfNFEVkmIi+JSP99PHef6BXyjkaj0RwspBOpcC/UKqUmHuAtZwDPKqVCIvJ1HCPKUw7wmnukVzzplx4xmClff4qyo0/l1FlC9fK5zHjoauYddzpVwQjXvnYfZz79KRvfn87kyy/lx8P8vHDjX2mK2Pzu0bcJNFQz/vyzeeDMway46/vMXF1LaaqH0686isxrf8QvZm9g9XufcPStJ8I5t/D79zax7L3VNG9bS2pOEf2OGs9XTh7MtPJMwrP/xbpXP2bYxVMwJp3P4u1tvLqkki1ramnasopQSz2Gx0d6YV9y+5UzeEg+dZW1+Ks3EWltApzCKekFfckpKaS4LJsJA/IYXZRJv2wfmaF6txC6o+cX5KWS2TeTrH55ZJWX4CsbgLfvQKKZhYR8WUCini9kmM78/ByvQUqOj1RXz0/Ny8AK+okEdhmtdVY4JYYYJkHbKYjuD1v43fn4bREbf8japedHbAJhC8Prw/R4HMvZ2Jx8j7Sbr2+YjmVtYuGUvRmtxfT+uOFawrz8jkZrsXn6nRVO2RPJFE7ZV6O1xOt0PP9gGa31hoknh3qIoBszciuB/gnru1nPKKXqlFIhd/VJ4Ohkz90fesWgr9FoNAeLWHJWMi0JFgPD3OJRPhxLmunt7yd9ElYvYFf9kVnAGSKSJyJ5wBnutgNCyzsajUaTgCDdZsOglLJE5BacwdoEnlJKrRSR+4APlVLTgW+JyAWABdQD17rn1ovIz3C+OADuU0rVH2if9KCv0Wg0Ceyjpt8lSqmZwMwO2+5NWP4+8P09nPsU8FS3dQY96Gs0Gk07Dncbhl6h6a+qiRBqqWf5g+cw/+/PcNd9t5J7/w28sHwn3/75ufyqbSwf/OvfDD7xQv73jUm8c9FNLKgPcMVpg6j5dAHDpl3I3645mtoHbue/M9ZhK8U5J5Uz6Hv38MTyembOXEl9xVIKr/8uTy3Zzsy31lO7djGmL43i0ZM5/6RBfGFkITL/BdY+P5dlK2rIOOWLrLeyeWlpFctX7KR+4yoCDdXxalm5/YfTd1Ae00YV01K1vl21rLSCvmSX9qOgTyYTBuQxpk82g3JTyTdCeOo3k+MmZRWle8nul0VOeS7ZA/uQ2r8/3r4DsbNLsTOLaBIaYMMAACAASURBVAg6wcTOqmWlZ6eQmusmZuWmkZKbRSToJGfFjNb2FsQVw+y0WlZrePcgbrxylplotLaHJC2P0a5aVmIwubMgbqLhWmK1rFiClje23Q3odkZniVm7vee9VMsyhHa2a10FcROvGWNPQdz9HVt0taweRBdR0Wg0ms8PMT/9wxU96Gs0Gk0H9KCv0Wg0nxOMw7yISq94Z8HmRt548nbeGTmZKVddzV31L/H7v3zINy4ewfIv3Mvv7n+G/MFj+e8Pp7Huq1/kheU7uWhwHhOe+jOlY6fx+69PpuTNh5nx8HtUBS3OGZbP+J/dzuv+Yv784gqql8/Fm5HD67WpPDljNVVL56KiNoXDj+H44wdyzdH9yN80j00vzGDF+1tZ6w9RmTWE6aurmbekiup1a2it2RovnJLdbwSlA/I45YgSpvTLI9BQHS+ckpZXQlbJAArLshkzMJ+x/XIYUZhOn3QDT90mwhUrKfSZlKZ6HD2/XzbZg/qQUV6Gt89AVF5folnFNISiNAbteOGUXXq+QUaap53RWlpBDqkF2dihAFErstfCKTE9XwwzruO3hHcVTvEHLcdsLWThD0bihmsxvd7jNXcZrCUUTolr/YbssXBKRz0/hs9j4DWMPRZOMY32RVS6MlqLv9e9FE5J1PP3dH6y6MIpuzjk9XzQmr5Go9F8nhDivjqHJXrQ12g0mg4czlbSetDXaDSaBAT2OP33cKBXDPr9+pdi3Hwpb2xrZvbZ8INxT3HB0Hzyn3iZs274K2IYPHLPRWQ8ciePvbiaY/PTOO2l+/nJ0ig//OaJnLhzDv93x7MsbQpyWnEGxz9wDSv7nshPn1zEpkWzEcOk/JhpPDB9FZsWzSfS2kT+4LGMnjKMW08YzODWdVQ+9yxrZqxlRXOIgK14fV0dMxZuZfvazfh3bCJqhfFm5JBdNpzSgUUcN7qY4wfmM6IghagVxvD4SM0pJLN0EPl9shhWnsuEAbkcUZxJWaYXb916rI0raF2/ztHzy7LIHZhD9qBSsgf2wdN3EFLYDyurhCbLoCFos70lFDdZi+n5OameuMlaemE6qa6en1qQ48zR30vhlEQ9XwwTf9jGH7Z2Ga0FnTn6LSErPj8/ELaxIraj5Scaq8U0/IQ5+x7Xgzym50e7KOISL4yeYK5miOA1pV3hlMTlZPV82F3P78x8DZxBwBDZJz2/swfFjnp+d8/RP9T1/F6D+1k7XOkVg75Go9EcLATwJlkKsTeiB32NRqNJQMs7Go1G83nCnRJ8uKIHfY1Go0kgFsM5XOkVwlVu03aeeG0dP370Ch6cdCOjslI4+eM5TPvBLJqrNvDDe67l9KVP8JcHZtM31ctlf/smf7dH85fHXuOGwmre/doDzKpuZUJuKqf97EJ2Tv0qtz23hLXvvoMV8FM6dhpXnTeST+d+QFtdFVl9hjDs2KP49qnDGOupofbFv7HqhSUsbgjQFIlSlGLy3Aeb2fppJY1bV2MF/XhSM8nuM4SSwWVMGF3MtGGFHFmcTtrONYhhOkHckkEUluUzeEAukwfnM7Ykm/5ZXlIbt2BvWU1g/ac0rttKfkkGuQOyyRlYQs6QMrz9hmKWDsLO6UOrpNIQsqlqCVHZEiQtIYib53OSstIL00kvSCO1ICsexPXk5mO71bJiAdTOiAVxTa+PlpBTMavFNVmLG62F7fhrOGxjW9HdjdViCVkep1qWJ6GYdMekrD0ZrcXaLnM1I14lKzFRa1flrF3vI1mTtcTlWBC3XXD3wD66e/wH1v1B1+6+XvcPer1pHHWM/bpuvRH9pK/RaDQJiPtAcbiiB32NRqNJ4HCXd/Sgr9FoNB3ordJNMvSK3zDbd7TwvTtP4JEh1wJw1dKXmPTzeWxb/D+uuuOrfMuez59u/AemCDc8+CXeGX4Z9z70Jk1bVvPBNXfy6po6hmf6OP+uU7Gv+BG3vLyc5W/OJdCwg+LRU7ng7BHcPLkfLds3kFHUn6HHTuL2s0Ywrcii5dW/svKfC1lU1UJNyCbHazAhN5WNK7bTuGkFkdYmTF8amaUDKR4ymDGjijljZDHj+2SS07SR8Ip5pOYUkVkyiIL+xfQfkMtxwwoZ3yebgbk+MlqrUVtXE1y7goa1W2lYV0Pe4FxyBhWTM7QMX7/BePoOxs4ppc2TSV3AZkdLmO0tIbY1BMj2GOT7TPJ9pmOuVphGemEaaYVZpBbkkF6chzcvDyO7YK96fmJSlun1xZOz2iVlBS38IYuWYCSu51sRGysSxfAYeLztTdc8XiNeWCWm56d4jF2Ga13o+TES9XyPaSRo+Lv0fK+5yy8lGT0/fm3pWs83RPZLj+7uwil7vE8vGKB604OzsMvAr6uW1PVEzhKRNSKyXkTu7mT/t0VklYgsE5G3RWRAwj5bRJa4bXrHc/cH/aSv0Wg0iXRjjVwRMYFHgNOBbcBiEZmulFqVcNgnwESlVJuIfBP4NXCZuy+glBrXLZ1x6RVP+hqNRnOwcDT95FoSTALWK6UqlFJh4DngwsQDlFJzlFJt7uoCoF83vp3d0IO+RqPRJBCzYUimAYUi8mFCu7HD5cqArQnr29xte+J64PWE9VT3ugtE5KLueH+9Qt4pzkvl/S/fz8+/eT/+RY9z/N93sHrWS5z5zRt4dMROnjjplzREbG7/6dmsO+u7fPO+N9i5ah5DTr6I5/9wG31TvVx80xRybv8dX395BR/MeBd/9SYKhx/DGeeO5funDCZl9pOk5ZUyePIUbjp3JOcNSCX48u9Z/vRcPljfQFXQItPj6PnDTh1IQ8VSgk01GB6fq+cPZ+TIQs46ooRj+mZR2FaFtWIetQs+JqNoInllpfQtz2XqsEIm9MlhcG4K2cFaZNsqguuXUf/pZurX7KBhYyNDzhpB3vD+pA4Ygrd8OFZOXwIpedS2Wezwh6lsDrKloY3NdW0c53Xm6KfnO1p+emE6aQWZpBXlOXp+bi5GbjFmXlGXhdANjw8xY8te/GHLLZayS88PhJ0iKoGgFdfzrYjtmKolzM83TInr+Wk+M67n+zxmUvPzIWa4Fu1Uz/cmLDtF0WWfTNFU1G5XCB32rOfvD8nq+QecB9ADWvnhPHMlKQT2YcZmrVJqYrfcVuQqYCJwUsLmAUqpShEZDMwWkeVKqQ0Hcp8ee9IXkVQRWSQiS0VkpYj81N0+SEQWukGN50XE11N90Gg0mn0lNmWzmwK5lUD/hPV+7rb29xQ5DfghcIFSKhTbrpSqdF8rgHeA8fv9xlx6Ut4JAacopcYC44CzRORY4AHgIaXUUKAB5+eMRqPRHCKIa+fddUuCxcAw92HXB1wOtJuFIyLjgb/gDPg7E7bniUiKu1wITAUSA8D7RY8N+srB76563aaAU4CX3O3PAN2iU2k0Gk130J1P+kopC7gFmAWsBl5QSq0UkftE5AL3sN8AmcCLHaZmjgI+FJGlwBzgVx1m/ewXParpu9OVPgKG4kxb2gA0un8I2EtQww2I3AjQJz21J7up0Wg0cRwbhu6LayilZgIzO2y7N2H5tD2cNx8Y020dcenR2TtKKdudY9oPZ+rSyH0493Gl1ESl1MSMQcP5xrd+R9nRp3LqLOGjF//F1Guu5b+nmvzzlNtY6w9x850nUX/t/VzxqzlUfTSLAcedz+O3TiXfZ3LZV8fT554/cOf/rWHWy3Np3raW/MFjOfncifz4jGHkfvAvPv71iww69nhuPG8Ul4/Mxfq/R1n+1znMX1HD1kCETI/B2JwURp48gMEXTyPQsCMhiDuSEaOLuHBsX47rn0NJuBpr+VxqP1hM1cIK8vv3p+9AJ4h7dFkOQ/NTyY00INtWEVr7CfUrNtKwZjsNFY3U1AbIG15O6kAniGvn9CWUXkBdwGJna5itTQG2NAbYXNfGtvo28n0mmXmppBemkVGSQUZxFmlFeaQV5OAryMfMK8bMKcDIyidqhXf7O3cM4poeH4bHi+Hx4Q9ZNLVF2gVxW4IWoYSkLCtiE7Wi8cpZHp+JYUo8QSsxKcvnMXdVzkoyiAvEq2btKYjrjVXP6uTTvKeKXLAriBuroBX/m7ivsSe5A4lrfpZB3P25/uc+iOsiklzrjRyU2TtKqUYRmQNMAXJFxOM+7Xca1NBoNJrPEuOAv5IPXXpy9k6RiOS6y2k4GWmrcbSpL7mHXQP8t6f6oNFoNPuKoJ/095c+wDOurm/gBDBeE5FVwHMi8nOc9OO/9mAfNBqNZp/pDX5G+0uPDfpKqWV0MqfUnW86aV+uVbFpB+VfnsryB88hZ8pNTLnqat66KJt/TbyCpU1BvnX78bTd/jAX/3w2Wz54jfIp5/GXO45n4qrnKL12HP3vf5w7Z23mlWffoXHTCnIHHslJ50/h/nNHUfzh83x8/z95+6PtfP23o7lmTCH2jD+w5JFZzF9Szaa2CGmmMDYnhTEnlTPskml4T/gShudXZJYOpGjoaIaNLuKicWVMLc+lT6SG6Iq51M5bQNXCDVSvqKH0wlxOHFHElAF5jCpMp8BqwKhcRXjtJ9Qt20Dd6krq1jWwc2crO4IWaUOG4Rs4EjuvP6GMonhS1pamIFsaA1TUtLK5tpXmxiCZealkFGc4mr6r56cX55FSXOjo+XnFGDmFRNNydvu77k3PN7y+dnq+PxiJ6/nhkIUViRK1nGZFoqSmezvV89N8Zjs932ca+6Tnq6jtGq5Jl3p+Rz16b3p+jEQ935A96/n785NY6/m9lF78FJ8MSX2WReRiEVknIk0i0iwiLSLS3NOd02g0moONdO88/UOOZJ/0fw2cr5Ra3ZOd0Wg0mkMBLe9AtR7wNRrN54XDeMxPetD/UESeB17FsVcAQCn1nx7plUaj0XxG6HKJDtlAG3BGwjYFHJRB35OWyeqHz2XOyMlMufVh5nwhk78f7QRxb7vzRNru+CMX3vc2m+fPoHzKefz1zhOZtPLfTP/aY1y0YT63u0Hc+oql5A8ey0nnT+E3F4x2gri/eIa3FlVRFbS466giojP+wCd/nMl7n+yIB3En5KZy1CkDGXbZqXhPvJRPrZx4EHf0mBIuGlfGiQNy6WvVEF3+DjXvzady/nqqV9SwpiXMtFHFuwVxQ6sWdRrErQ3b8SBuMKOIGjeIu6khsFsQt605REZxRrukrD0FcTsGcvcWxDVT0jA9vi6DuLEELduKJh3E9XmMfQriAkkHcRM1Vh3E1RwIh/GYn9ygr5S6rqc7otFoNIcKh3OhkWRn7/QTkVdEZKfbXhaRHq3uotFoNJ8F4pZLTKb1RpL9Qvsbjh1oX7fNcLdpNBrNYYfOyIUipVTiIP+0iNzeEx3qjCP7ZfH6oInMrW1j9tnw5NFXsdYf5jv3nEHN1x7gC/e+QeXimQw+8UL+ceeJHPHBY7x089+ZVxfg9RkV/N/zb9O0ZTUFQydw5heO4xdnjyB/3tMs/sW/eXtJNTuCFgPTvURe+g2fPPIG7y3fZbI2ITeVMacNZOhlp2OeeBmrgxm8sLSKkmFHcORRJXxhfBnH98+hNLQda+kcat5fyLb569mxqpb1/gjVIYvzB+YzojCNgnCdUylr1SJql22gblUV9evrqakNUBmwaIjY+K0oVv4AQukF1LRZVDY7Jmsb651KWYl6fltLqJ2en9GnYJfJWkEpkl1INDXL0fRTsuJ/z2T0/JjhWmd6fsxkLabn23Y0aT0/xWPsk57vVLhKTs+PafHJ6Pmw90pZHfV82c9/4V3p+d39sNhLx6FDCkHLOwB1InKViJhuuwqo68mOaTQazWeFiCTVeiPJDvpfBS4FdgDbcQzTdHBXo9Ecfri/AJNpvZFkZ+9sBi7o8kCNRqPp5QhODYfDlb0O+iJyl1Lq1yLyR5x5+e1QSn2rx3qWQP3yNSww+vDjR6/gwUk30mzZfP+hL/LJmXdx3d2vUr1iLqPO/BLP33E8pf/9Ff/83n/4uDHItKJ0vvn31/BXb6J49FQuuvgY7jtjKKn/+xMLfvkys1fXUhOyGZLh48QpZSz+7UzeX1dPVdAix2twTF4aR5wzhEGXnYcx5WKWNpk8+8lW3ltSxYQJfbjILZpS1LqFyCezqX5vEVULNlL1aR3r/WGqQxYBWzG6KJ3cQDVsWU5g9cdxPb9ufQM76wPsCNpxPT8cVbSl5lPbalHZHGJLU5CNda1xPd/fGKS1OUTAHyLU0kxmn5wEPb8AI7cYM6/I0fPTclBpOdgpmbRFHK08Uc83vT532YvpS8Pw+jA9PmfZ46OxLUwgbBMIWu2Kplhhm6itsN25+nasiIqr5ScWTUnzmfjM2LrT9kXPB9rp+V5zl37fUc83jeT1fOhcz+8uLT/x+jG0nt976K3STTJ0Je/ErBc+xCl72LFpNBrNYYWTkdt98o6InCUia0RkvYjc3cn+FBF53t2/UEQGJuz7vrt9jYic2R3vb69P+kqpGe5im1LqxQ4dvaQ7OqDRaDSHGt31nO/WE3kEp4jUNmCxiEzvUOD8eqBBKTVURC4HHgAuE5HRwOXAEThT5d8SkeFKqc5/uiZJsoHc7ye5TaPRaHo5jlyYTEuCScB6pVSFUioMPAdc2OGYC4Fn3OWXgFPF0ZcuBJ5TSoWUUhuB9exjLZLO6ErTPxs4BygTkT8k7MoGrAO9uUaj0Rxy7FviVaGIfJiw/rhS6vGE9TJga8L6NmByh2vEj1FKWSLSBBS42xd0OLcs6Z7tga5m71Th6PkX0F7DbwHuONCbJ0s4qvjpa3fzO+/J+HiJH7xwG8/2vYi77/oHzdvWcvQlV/LKzcdi//7bPPGbOWxoDXN+v2xOeeRrXP3T5ZQdcw7XXTKGu44vJ/iPnzH3gdd5a0sTfivKkdkpTJ02gFHf+BK/uOgBakI2RSkmk/PTGXnxKPp/6ULUpIt4v6qN5z7ezKIlVVSvq+C+yy5hUlkWuXVrCX30Ftvnfkjlgi1sq2hkvT9MbdgmHFWYAnn+rUQ3LqVt5RLqVlZQu6qahopGdjQG2RHclZRlu6Hy6jYniLupMcDmujYqavxU1QfiQdxgW5hQSzORtibSRxeQUVqAtyBmslYEmQVxkzXbm05rOEpbJLorIcswMb1OAlZipSyPG8CNJWn5gxbhsN1pENcK29i2k5wVtaP43ACuz2OQ7jPbJWUlBnF9HqNdEHdX0HZXEDcx8BqN2kkHcTt78tpTEDdGskHcfQ266iBu70WUQrr43CRQq5Sa2JP96W660vSXAktF5F9KKf1kr9FoPheIinbXpSqB/gnr/dxtnR2zTUQ8QA5O8msy5+4ze9X0ReQFd/ETEVmW0JaLyLIDvblGo9EceihQ0eRa1ywGhonIIBHx4QRmp3c4Zjpwjbv8JWC2Ukq52y93Z/cMAoYBiw703XUl79zmvp53oDfSaDSaXoPaLS1pPy+jLBG5BZgFmMBTSqmVInIf8KFSajrwV+AfIrIeqMf5YsA97gVgFU4M9eYDnbkDXcs7293FWiCglIqKyHBgJPD6gd48WfocMYgrq8Yy8y8P41/0ON9dV8hf73qMqBXmrK9fy/OXj6Li1iv497Mr8VtRvjypL1P+cBeLS05g6EnzuevKcXy5XLHz17fzwaPvM7e2DYCpBWkcc9EIhtxwLY2jzqAm9Ev6p3mZNCCbkV8cR58vXkLr8JOYXdHIcx9uZcWyanZu+BT/jk2cNCCH1K0f0brgTSrnLqFqURUbtzWzNWBRn6Dn53hN7E8X0rJiKXUrNlK3ppaGikYq/WFqQk5SVsDepef7DKGiPsCWpiCbalupqPFT3RCgtTlEW1OINn+ISGsT4bYmrICfzLIivIUlGG7RFDJyiabmEE3NJmKm0Ba2aY1EaYuoPer5iSZrZoqj63t8XkIhCyscK5Ziu8lYTgGVRD3ftqwELd/Yq57vSzRcS9DzOyZkRRM0Va9pYAhd6vkdJf1k9PyuDNYOVHvv7HSt5x/iKJXsU3ySl1MzgZkdtt2bsBwEOp0Cr5T6BfCLbusMyU/ZnAukikgZ8AbwFeDp7uyIRqPRHCqIiibVeiPJDvqilGoDLgYeVUpdgpMwoNFoNIcZCqJWcq0XkqyfvojIFOBKnOwxcPQpjUajObxQdKu8c6iR7KB/O04G7itucGEwMKfnutWe1XU2y/74F8qnnMeps4QF//4TmaUDueP2i7l7sJ/5p5/Di4uqyPeZfPWSUYz+9QP8uyaPX/1hPo/ePIUTqGDt93/JnJc+ZWlTkByvwQmFGYz96iTKrvs6FdmjeHreZkZlpTDxqGJGXDqJvPOvZHvOcP63cifPLdrKplU7qa9YQVtdFVErjG/V29TPm03l+6vY/tEO1te2URW0aIrY2MrR5nO8Bn1TvdQvXEjdik3Ur2+gbnMTlQGnAHpTxNH+E/X8TI/B2rpWKna2srmulbqGAG3NIWd+fmuQcEs9kaAfK+DHDgfxFg/BLCjFzCuOF0tRaTkE8dAWjtIaiRKworSErLihWuLcfEe/T2uv57vmaZGQHZ+P72j6Kl5AJabpq6hN1ArH9fw0n6ddwZSYjm8aspumD13r+cq22+n5Hefqwy4930hQt7vS82PnQXJ6/v74b/X03PzO7qHpDhREP+eDvlLqXeBdEckUkUylVAVwUBw2NRqN5mDTW/X6ZEi2MPoYEfkEWAmsEpGPRERr+hqN5vCk++bpH3IkK+/8Bfi2UmoOgIicDDwBHNdD/dJoNJrPBqUgeRuGXkeyg35GbMAHUEq9IyIZPdQnjUaj+Uw5nOWdZAf9ChG5B/iHu34VUNEzXdqdQFMDU+/4Cm/cOoWcKTdRPuU8Hv/2CUz59AVeOfZR3trZyticVC78zjTyvvMQ3521nhdfeovqFXM59pxaFv7yad78oJKqoEXfVA8nH1XM2BtPIf2CG5nXksHjs9awaOE2nj99IMMvPxXvSZfyqZ3Pyx9V8vribVSuraJpy2oCDTsASMnKp/q16VR+sI4dS3eypsWpkuW3nA9KmikU+jyUpXnoU5DGjoXrqFvXwM6drXGDtaaIUyULnNJssSButsdkZWUzm2tbaW4M0tYcoq0lRKjVT6S1qV0Q1woF8JSUY+QUxg3WoilZtFmKtohNqxUlEInSFLRoClm7BXHjAVy3apbHl4JhGnh8Jh6viZVgtma7wdt2iVlW2AnkRsJOANdNyOoYxI0308AQ2WuVrFgQV9kJyVmGsdeErFgAVyS5AG7i/boK4u5vAaVkgrgHUp1JB3B7ku5NzjrU2JfC6EXAf4CXgUJ3m0aj0Rx+fF41fRFJBb4BDAWWA3cqpSIHo2MajUbzmdDNNgyHGl3JO88AEeA94GxgFM6cfY1GozksET7fmv5opdQYABH5K91g67k/9O1XypwzIrw5cjLH3fYHXr3hGBp+8nUefHQBVcEIXxiWz0l/upmKoy7hy39eyLJZ7+Kv3kRO+Sjeuu4h5uzwE7CjTMhNZcpZgxl+w+WEj72Ef6+u5al3lrPh4w00bF7B6N/ciD3+XGZvbuaFTzbw4ZLtVK9bR3PVBqygH8PjIzWnkOx+I1g341G2bmpkY2ukXcGUTI9BSYqj5xeXZZE/LJ+qRVVUNYfYEbRpttoXTDEF0kyDTI9Bntck32cws6oZf1OQQEuYgD9EqKUxbrBmh4NY4QDRSJioFUby+2CnuQZrnjTawlEClqI1EqU1bNMUsmgKRvCHbUxf6u56foLBmmkaeLwmhsfA4zUIh6w9GqzFtPxYclaaz9yjwZppCD7TwGsIhiF7LZgC7fV8FbW71PPjunySQneinr+3gimJknuyOmhnHG56/gF0vZegwD58Z+909VmOSzn7WkRFRPqLyBwRWSUiK0XkNnd7voi8KSLr3Ne8/ei3RqPR9AwxG4bDVNPvatAfKyLNbmsBjooti0hzF+daODGA0cCxwM1udfe7gbeVUsOAt911jUajOWQ4nF02u/LT329TNdeLf7u73CIiq3GK+l4InOwe9gzwDvC9/b2PRqPRdC+f70ButyAiA4HxwEKgJKE4yw6gZA/n3AjcCFCWk8kDU26mNmzx9pmKd487mZdX7KQkxcOtXx3HkJ//jqc2e3jwF7PZsuhNAMqnnMcl545kxnmPkO8zOa08j6OuO5aSq77OurTBPP7mBt6ct5mqFUvwV29CRW22Dz+TmUt28MKCLWz5tIb6imW01VWhojae1EwyivuTXz6M0oG5rHilnq2BSFyf9xlCvs+kJMVDeZaPvMG5FIwoJG94f96bVRE3WAvYuyry7Jqbb5Dj6vk5WSk01rQS8IfjBmvhtibsUAAr2IrtavkxPdzOKkKlZBFQJoEEg7XGgIU/7MzP94csmkNWgn7fucGax2vi8Rlxbb+1ObTb3PyYhh/T8+OavtfcTc+Pmax5DQNTnGIoXkO6NFiLL7vbvYaxW/HzRD3fkOR05o5z+JMxWDuUtPx9v3/33uvw1/ITOIwH/QP5TCeFiGTizO2/XSnVThJy60B2WpdMKfW4UmqiUmpiQUZaT3dTo9FoHGI2DMm0XkiPDvoi4sUZ8P+llPqPu7laRPq4+/sAO3uyDxqNRrNvKJQVSaodCMlMahGRcSLygTsZZpmIXJaw72kR2SgiS9w2Lpn79tigL87v2L8Cq5VSDybsSqz8fg3w357qg0aj0ewzioP1pJ/MpJY24Gql1BHAWcDvRSQ3Yf93lVLj3LYkmZv2pKY/FaeW7nIRiXXmB8CvgBdE5HpgM3BpD/ZBo9Fo9gmFahdb6kG6nNSilFqbsFwlIjtxLHEa9/emPTboK6XeZ895JKfuy7Wqqpooys3n5t9fwoOTbmRDa5jz+2Vzyh+vo3Lq1zj7+aUsmfU+zdvWktVnCKOnHcePLjiCU7ObeCw7hanTBjDqG19CnXw1L66p4y+vLmHDks3Urf+YSGsTntRMcvoN51dzNrDgkyp2rN1A8/YNRFqbEMMkvaAvWX2GUjywlKFD8jl5ZDGr/eF4QlaO14gbrJX2LrxafAAAH7lJREFUySR/WB75w/uSN2oAKYNGsjUwY48JWdkeg3yfSWGKh/TCNDJKMmhpCBBqaSbS1kS4tWm3hKzEgKSVlk9rJEpbvEKWTUvYoilo4Q/bNIUi+IMWTW0RvKmZTnKWG8TtLCErFtA1PYZbLWvPCVnxQG7UjlfO2lNCViyY6zGNpBKyEukqIatj1azO6MyIbV8SsvY1APtZBnG7O4ALn7cgLvtSOatQRD5MWH9cKfV4kucmNaklhohMAnzAhoTNvxCRe3F/KSilQl3d9KDM3tFoNJrewz756dcqpSbuaaeIvAWUdrLrh+3uqJQSkU4ntbjX+f/2zjw8jrvM85+3qrullmTrlixbjuX4NgkJORxCBiYkgQSWHJsNIYFhmF0yHpb7AYYkZGFgnp1nAzObsCwsYG52MjAQyEOAgElCjuUIwUnsxI7t2PER35Zlqa2jpe7q+u0f9etWtdwttXxIavf7eZ56uupX1VX1s1tvV3/fq4OgyvF7jMmFFt1J8GURA9YQ/Er4x4luWI2+oihKGGNO2kk7eipzVbF9InJIRDqMMQfGC2oRkdnAL4G7jDFPhc6d/ZUwIiLfAT5Ryj2d9pBNRVGU8sLkpMuJlpNkwqAWEYkBDwDfN8bcP2ZfNgpSgBuAjaVctCye9FsbqvnPm3/JFzZliHE/n/zo6+j8zD3cs76fb3z6N+x75mGcSIyz33A9775uBR+4pJPqJ7/H+i/fzzs++1Yab17NizKXr/5iK0/+4RUOvPgcg917AKhr76J50bmc/ao2fvXrLfTteoFk7yGMnyFaW8+s9i4aOrvoWNjIZctauWxhE69qq2WDb4i7QmPUZU51hPn1VTQtaaJpcTONKxZQt3gx0a4V+M0LSKRH9cG4K8TdUS2/KeYyq76KurZaalri1HXMZqjnUF6BtbEJWWF6hzM5PT/bLGXAavqDqUDL7xtKMzDi4cbieQlZkZgbaPqhhKywtp/T9EPNUsIJWX7owx8Pafqu1fCjblAkbVTXl5zePFFCVng76oY0/AIJWWGNfywT/WGeai2/EOOdo9QicaWiCVmngGz0zumnYFCLiFwEvM8Yc5sdewPQLCJ/Y9/3NzZS5z4RaSXwna4nKIM/IWVh9BVFUaYOMxlH7olfxZgeCgS1GGPWAbfZ9X8F/rXI+684keuq0VcURQljmKqQzWlBjb6iKEoek4reKTvKwuh7nQu54J4tbH/iIQaeXsMj7krefs+zvPTEI6QGE7Qufy2XvelcPveW5SzpeZadd/w3nvnRRp46muQT9z3Ivc8f4P4nnmb3hhdJ7H2JTCpJdX0rDV3nMH/5PK56zVzetqKdN3zv38ikkrixODXNc5nduYw5XY2cv7SF1y9q5oK5s+maHSV6aOtocbWaCM0L6mlZ1kzD0k4ali0k2rUc6ViE19BJbyb4J445QtwVat1RLb+xNkq8pYa6thpq22uJtzVSO6eJ5K8O5hqfF9PyxXERx6VveDQuP6vn948EWv7AcLA+MJymf9gjWls/Goefa4DuWB1/jL4fcfBS6eD6mfy4fONnyGS37RNRVtPPavhR2wQ96gY6ftQRXKvplxKbH94eW1xt7Bgcr42X4mQbq+ePp+WfqPZeTM9XLX8Gcwqjd2YiZWH0FUVRpg590lcURakcpi56Z1pQo68oihLCYHJ9nM9E1OgriqKE0Sf96eflXQeJPvZzOi9+M1euFZ5f+zUGDu2i/qwVXHTjdXz22pVcVnWIQ1/7e379zT/y+8ODHE1lmFMd4Z3f/jM71u/i6M4NpAcTRGvraew6h7nLz+ay8+dy7TlzuKijltmHX8T4mZwDt21BG0sXNfGXy9q4pLOeRY1VxHt34a/bwLGN6zmvvoq2ebOChCxbXC3WtRx33lIyjZ0cc2roHvTYe2yIukhQXK3RdsdqjEWoba+hpqWG2rYaatrqqe1oJt7aSLS1ndRPthcsrpZFHBcnEkMclwMDI/Tbzlj9qVEHbl8yzcBwmqFUhoFhj1QqQ6wqkpeAlUvOirq4EcFxHWKhJKvMSLJgcbWcQzcTSs6KugWLq2U7ZjkiufVSHbhZ3FBnq3BxtTzHLqPOzFIzJUtJyKo0By5UuBMXAkduOjXdd3HaKAujryiKMnVMTXLWdKFGX1EUZSwq7yiKolQIxpyKYmozlrIw+pHqWm7/7x/ljr/sov7S9zOrYxGrbnk3d12/kqsaBjj6/c/x6Dd/z+9eSdA9kqG1yuVtHbNYfuMK/vmBn+UapTQvvoA5Sxdx8XkdXHduB5d2zqLh6DZG1j7MzifX0br8GlrOamfpkmYuX97GqnkNLGqMUde/D/+55xjc8jxHnt/OkRcPsez18/MapbidgZafcOvoTnrsOzbIrr4ku3uGmFsdOa5RSlbLDxKymok2t+A2tuE2tuIl10+o5bvRoBnKgf6RoMBaMk1iKEjCGhjx6B9O57R8L53BS/vE4tHjGqVEos5xWn5VxCEei5BJJSfU8rP3WRVxxtXys4labhHdvdgfmfEzE2r5MNpgZbJ/rKVq+Scrc6uWX15o9I6iKEqlYAwmo0ZfURSlIjDG4Ke96b6N04YafUVRlDAGfdKfbs6ZP5sPb/sWj//db3jdR77EZ65dyRviRzj8nc/yyDf/wP87MMDRVKDlX9s5mxU3nUPnTTfgX3At5orbaVl6MR1LF3Kpjcu/eG4d9Ue2MPLrR9j5xDr2/3kvu1/u5fX3fDwXl7+woYraxCv4zz3HwOZAy+/Z2s3RbUfZf2yEm794C1WLVubi8vucGrqHPPYeG+SVRJId3YPs7hlk75EhPj6r6jgtPxeX39yC2zwHt7ENahvx4/UFi6uN1fKdSBQnEuOV3qGgsNqwRyKZyovLT48Een4m4+OlMlTXRseNyw+am9tt1zmuUUohLT+rfVa7TtG4fEeCWPtsg/Pw/MbT8rNk/QDjafkw+TZw2eOnU8s/kfOfDj1fyUeNvqIoSoVgjMHXevqKoiiVw5kcvaON0RVFUcLY6J1SlpNBRJpE5GER2WZfG4sclxGR9XZ5MDS+UET+JCLbReTfbRP1CVGjryiKEiIbvVPKcpLcATxqjFkCPGq3C5E0xpxvl+tC458H7jXGLAZ6gfeWctGykHeOPr+Fz3y4l5gjPHq1YecXP8BPbGesZMbQVRPlqmXNLL/5QtpvfAd981fx0x29/PC+Dbzm+htynbHOba0muvNPDPzoEbY+sYH9zxxkx/5+9iTTHE1l+MzVy3KdsdK/f4beTRvp2bSTni099O7oY89Qmu4Rj2OeT/yKt5Np6KQ7E6F7yGN3Xz97Ekl2WgfuwZ4hBo+NMHhshDnnt+V1xoq3NRJpbMVpbCPSPAe/pgG/ahZ+vJ6UE3xZZztjiePiRGM41pmbdeC6VXHcSIy9vclcZ6yBYS9IxEr5NiHLOnI9g+/51M6uzuuMFY+5VFknbtiBmx3zUslccbRCDtzR9aDgWrYz1miXrHwHbuDcHb8oWsGktCKF1cY6cIsVOStGMQduobNMNrnqdDhwlanDnxpH7vXA5Xb9e8DjwO2lvFGCD+8VwDtD7/8s8NWJ3qtP+oqiKGFsyGaJ8k6LiKwLLasncaV2Y8wBu34QaC9yXLU991MicoMdawb6jDHZnxt7gXmlXLQsnvQVRVGmjMll5B4xxlxUbKeIPALMKbDrrvxLGiMipshpFhhj9onI2cBvReQFIFHqDY5Fjb6iKEoIw6mL3jHGXFVsn4gcEpEOY8wBEekADhc5xz77ukNEHgdeA/wEaBCRiH3a7wT2lXJPZWH0U77h7Rd0cP7qN3LPqtW8PJgi7grn1Vdz3uvns+zWNxK9/Ba2mWa+s+kgDz34FHu27CPxymbW/+hOOv0e/I0/58h3fs++P27j4IbDbB9IsX/YY8AL/nPjrrD44FOkHn+G/S9s58jGPfRs66Wne5B9SY/edIZE2iflB1/Ge6rP4lBPml19A+w6OsSO7kH2Hh0i0TfM4LFhkv0pkv39pAcTdFyymJq2RqpamnKJWE59C368Hq96Fn51PUOeYSjlk/Q8q93HENfFDen4TjRGJBYPNP1YHCcaY/eRQUZCRdW80HrG88lkfHz7Wl0bJZan5Y/q+NlCa7HQ4qdTebp99g8hPAbg+xmqIzYhy2r5UcfJ0/HDun6pxdayuFntfgIt/2R197FvP9VF0lTHLxOMwU9NSRmGB4H3AHfb15+NPcBG9AwZY0ZEpAW4DPiC/WXwGHAT8MNi7y+EavqKoihhDPi+X9JyktwNvElEtgFX2W1E5CIR+aY9ZgWwTkQ2AI8BdxtjXrT7bgc+JiLbCTT+b5Vy0bJ40lcURZkqDFNTZdMY0wNcWWB8HXCbXf8DcG6R9+8AVk32umr0FUVRwhjy+jifaZSF0e9YuYCFax/mK+v3E+N+brmwgxU3X0Trje/icOu5/OTlXn7wwCu8vGkDR17exGD3HnwvhRuL0/Djf2Lr717gwLMH2X5gkP3DQUx+xkDMEVqrXNqrIpxVE2XbF7/MkS099O5KsC/p5WLykxmfjPWrxxwh7gq/3HaEHYeDmPwjvUkG+oYZGkgxPJgi1X+U1FACLzlAJjVM8yUX5hqk+DUN+NX1ZKpnkXJiDKZ9hgY9kmlDYiRN/0iGaLwuF5PvVlkNP6Tju7F4rhHKscRIwZj8bKE14xsynofvpWiui+Vi8uNRN0/Hdx3J0/OjTlBwDY6PyYdAx89iMhmqIk7BmPzwdrgRSvhc4xE0UTm+qFohHf9E6pCVGpM/2RyAia6hzGSMlmE4EUTk2yJyWEQ2hsZKSjtWFEWZNiYXp192nE5H7neBa8aMlZp2rCiKMi0YY8ikvJKWcuS0GX1jzJPA0THD1xOkC2Nfb0BRFGVGYaykOfFSjky1pl9q2jE2nXk1wFkdRQ9TFEU5tWjnrNPDBGnHGGPWAGsAauctNZes/haJvS8x8PQa+uav4uEdvfzw8T1s3fQo3dtfZPDwHjKpJG4sTm3rfGZ3LqP9rAZ+/KkP5gqqZUyQ6FMfzTpvIzQvqKdlWTMNSzv52b88VtR5WxcRal2HpphLU8zl63/YnSuoVsh5640k8b0guSn6qr/Cr64nbQuqDaZ9hoZ9kuk0/SmPxLBHYsRjIOXRP+IRq2vMFVQr5Lx1XYdIzCUSdRhIJHPO20wmSMjyM37OeWsymdx9NNVV5RVUG7u4tlhatvOV76WD/4siztvcejg5q4jzNlw0bSIH7tj92eSs8Zy3J/KTNexgPZOct9pY6yQxYDJFTVPZM9VGv6S0Y0VRlOnCYKaqyua0MNUZudm0Y5hE2rCiKMqUYcD4pqSlHDltT/oi8gOCWtEtIrIX+AeCNOMfich7gd3Azafr+oqiKCeCMZBJaXLWpDHG3Fpk13FpxxOR7Osl1n+UeRdeyZVrhd2bH6Jv1wsM9ewPNPPaembNXUTTWYuY09XApUtbuezsZs5tq+V/fHrYJmFFmFsdYV5djKYljTQtbqZpxQLqliwm1rUcv6WLDZ/+Ve6acVeIuw6zIw71UZfWKpdZ9VXUNMepa69l16b9pAcTeTp+Jp3K6edhXbq/cRGDaUMy6TOUTuVp+AMjHsdGPBJDthHKiEe8cU4uKSsSdYnEsjq+bYASdXEiDpGow+FXEqNavr12tlCa8QM937frbbOqRvV7m4wVdRyiruT0fMexr7Yw2ng6fpiaqJtXEC2s44/q7lJUbx5P5xeR0SYqofc7Y46ZLMcVXBvnHKe6+JpzioV31fFPIcaopq8oilJJ+Gr0FUVRKgQN2VQURakcDOCXqZO2FNToK4qihDFGHbnTzZx57fz0Gx/hvPYa6i99P24sTryxnbkXXk37WQ28emkLr1/cwsXzZtM1O0r00FbSL/2C/l9v5Or2WpoX1NO0uJGmFWfRsGwh0a7lSMciMg2d9GYidA957O4dpj7q5CVgNdZGibfUUNdWQ217LfG2RmpaG6jpaKb3GxvyErDGOiLFcXPLlp5hEsOB4zYxEiRgJYbSDAwH6wPD1ok77OGlM9S1tOQlYDmhpCw3IoFz13bA2r1pb14CVnbJZLczo9UxW2dXHZeAFXUDp23UyXa9Gl3PpFO5+UzU7SrqOHkJWOGKmnnjRd4/Hq712I7nuD1RR2sx5606bisXo8lZiqIoFYQafUVRlEpCM3IVRVEqhynKyC2lv4iIvFFE1oeWYRG5we77rojsDO07v5TrlsWTfluym6qP3MJvnznI6z72v/OSrzoiw7gHNpPa8hhHf76FbVv2cGRLDz37B9iX9Fj9bx/OJV95DfPoHvLoHvTY1TvE7p2j3a96e4f5dFdDLvmqpq2OmrZGauY0EW9twmlsI9I8B6ehFT9ez/C/fD7vHsMavhMNOl05kShOJMYfXunNS77KavhDVsP30j5eKpPrdlXfXJNLvsoWWcsmVVVFHOKxSLDtOjzVfzSXfJXV8MNdroIleGppqo7mJV+5Y9YdIdD83dHkrCyFNPjwWMTNT75yZFS/DydtFTvXeDjka+/HJVVN6myh941zzuOOneS5T7WGH0b1/NOLYcri9LP9Re4WkTvs9u1592LMY8D5EHxJANuB34QO+XtjzP2TuWhZGH1FUZQpwxj8qYneuZ6gVA0E/UUeZ4zRH8NNwK+MMUMnc1GVdxRFUUIYEzzpl7KcJCX3F7HcAvxgzNg/icjzInKviFSVclF90lcURRnDJLpitYjIutD2GtsLBAAReQSYU+B9d+Vdb4L+IrYU/bnA2tDwnQRfFjGC3iO3A/840Q2XhdHft7ePr+99ibgrPHq1YWTzQxz94VZ6Nu/l5S09dB8c4OBwhiMpjwHPJxn6Bt746neysy/J7q1D7Ojeyu4jgyT6hhk8NkyyP8Xw4FCucNpFH76SeHsrbmMrbmNbTr/34/X4VbMY8IXBtGEo7eNEYgX1eycaIxILiqU5kRhuVZzHNh8uqt97qaD5SbgJyooL5hKLONTEXGIRN6ffZzX9cOOT1GACOF6/D+v6EDRAaYxH8/T7qOPkmp0Uan4ykaYfJuZkG5zk6/fZn5KFGqCUiht609i3n0w8fbH3qmRe4ZhJPcUfMcZcVPxU5qpi+0RkMv1FbgYeMMakQ+fO/koYEZHvAJ8o5YZV3lEURQlj4/RLWU6SyfQXuZUx0o79okCCJ6obgI2lXLQsnvQVRVGmCsOUFVwr2F9ERC4C3meMuc1udwHzgSfGvP8+EWkl+HG6HnhfKRdVo68oihLGGDKp02/0jTE9FOgvYoxZB9wW2t4FzCtw3BUncl01+oqiKCGMAd9oGYZppXV2FZ/8L6+jaUUX96xaTW86w4Dnk7IZca4EjsS6iMPc6ihNMYfWqgg1TXFu+z9/ZKh/hJHBgcBhO5jAGx7E91J4I8lcdymAWX91N5nq2QykfQbTPknPJ5n2SRz1SIz0MzBiO16NeMyau8g6cGO4sbh14FaFulq5ueJor+zoJWMdtV4qgzEm1+lqbJcr42c4Z96KvO5WuSVUJC3qOLgC3vAgkO+whcJdrhrj0YIO27GF0UpJojq+4Fr2HPkO22KdriaDUNjpeiLdssaet1S0YFplkVGjryiKUhkY4Ayut6ZGX1EUZSz6pK8oilIh+IacdHwmUhZG3z/rbJ766y+w8+gQMe5nUW2UppjLrOYaalri1LbXUts2i5o5zdS0NRJrbsJt7sBtbGXrbT/NafZhcsXRbAKVG4lx/84UiZGDgXZvC6QlkmmSKY/+YY9kKMFqzrKVOc0+2/DEce22bXCSTaZ6cu3zeZp9oQJp4WSq5R2zcpp9xA1eAy1/dD1bLC3rlxhLobHZVZE8zT5bIG1sg5Osfj2ZwmgRV4o2OTnZhiTumBOc6gYnwTlVs1dGUXlHURSlQjAYlXcURVEqBXXkKoqiVBhq9KeZbbsP8bcf+p/46RQDT6+B2sagCFr1bNKROENpn6Rn6Ev77EtlSIx4JIbTDKQyxBvbjyuE5lYFr5FYNNDjbWz9vQ++aIuh5RdA8zM+Gc8L9HgbV3/1tRfkYufHFkHLxdi7DlFHeOj7O/MKoYW18kJx9UuaascthBZuVpJJJUv6NzR+hrpYoLqPLYIGhePqJ0OsBN39ROPqww1ZTiWT0fFVo68cjNHoHUVRlIrBoNE7iqIoFYNq+oqiKBWGyjuKoigVQqDpT/ddnD7Kwui7sWraVl6GG3G4cq3gpY7ipbttolSGjGfwPT/Xjcr4hozn4Xsp3vzO/2Cdqy7xqJvXfWpsQbNP/8N3Q0lSfsHuU1luvfA6HOE4R2shx+tw4kjufaUkPJ1VH7S6LKX71GQSqGqjwZkK+SRPNuEp6uaf4FT6Pd3T5EVV56xSDH3SVxRFqRAMMCUtVKYJNfqKoighDEajdxRFUSqFIHpHjf60cs6CJn7/pbcBUH/p+yf13u9+7e0lH/ux7j0lH3vZ/FklH1uo4Nt4tNWenv+WmuiJtjGZmMjpqIJmUe1dmVLOcEfu6bMC4yAi14jIVhHZLiJ3TMc9KIqiFCL7pF/KcjKIyNtFZJOI+LYZerHjCtpLEVkoIn+y4/8uIrFSrjvlRl9EXOArwFuAlcCtIrJyqu9DURSlGBlT2nKSbARuBJ4sdsAE9vLzwL3GmMVAL/DeUi46HU/6q4DtxpgdxpgU8EPg+mm4D0VRlOPwCcowlLKcDMaYzcaYrRMcVtBeShC/fQVwvz3ue8ANpVxXzBQ7LETkJuAaY8xtdvvdwCXGmA+OOW41sNpunkPwrXim0AIcmfCo8uFMmw+ceXOqpPksMMa0nuiJReTX9vylUA0Mh7bXGGPWTPJ6jwOfMMasK7CvoL0EPgs8ZZ/yEZH5wK+MMedMdL0Z68i1/3BrAERknTGmqOZVbuh8Zj5n2px0PqVjjLnmVJ1LRB4B5hTYdZcx5men6jqTYTqM/j5gfmi7044piqKcURhjrjrJUxSzlz1Ag4hEjDEek7Cj06Hp/xlYYj3PMeAW4MFpuA9FUZSZTkF7aQJd/jHgJnvce4CSfjlMudG330ofBNYCm4EfGWM2TfC2SWlkZYDOZ+Zzps1J5zPDEJH/KCJ7gUuBX4rIWjs+V0Qeggnt5e3Ax0RkO9AMfKuk6061I1dRFEWZPqYlOUtRFEWZHtToK4qiVBAz2uiXa7kGEfm2iBwWkY2hsSYReVhEttnXRjsuIvIlO8fnReSC6bvzwojIfBF5TERetGnjH7HjZTknEakWkadFZIOdz+fseMG0dhGpstvb7f6u6bz/YoiIKyLPicgv7Ha5z2eXiLwgIutFZJ0dK8vP3Exixhr9Mi/X8F1gbKzvHcCjxpglwKN2G4L5LbHLauCrU3SPk8EDPm6MWQm8FviA/b8o1zmNAFcYY84DzgeuEZHXUjyt/b1Arx2/1x43E/kIgbMvS7nPB+CNxpjzQzH55fqZmzkYY2bkQuDRXhvavhO4c7rvaxL33wVsDG1vBTrsegew1a5/Hbi10HEzdSEIDXvTmTAnoAZ4liDL8QgQseO5zx9B5MSldj1ij5Ppvvcx8+gkMIJXAL8gaF5WtvOx97YLaBkzVvafueleZuyTPjAPCNc63mvHypV2Y8wBu34QaLfrZTVPKwW8BvgTZTwnK4WsBw4DDwMvA30mCJGD/HvOzcfuTxCEyM0kvgh8ktGmT82U93wgKHj5GxF5xpZlgTL+zM0UZmwZhjMZY4wRkbKLlRWROuAnwEeNMcckVOi+3OZkjMkA54tIA/AAsHyab+mEEZG3AYeNMc+IyOXTfT+nkL8wxuwTkTbgYRHZEt5Zbp+5mcJMftI/08o1HBKRDgD7etiOl8U8RSRKYPDvM8b81A6X9ZwAjDF9BJmNl2LT2u2u8D3n5mP31xOkwc8ULgOuE5FdBFUYrwD+F+U7HwCMMfvs62GCL+ZVnAGfuelmJhv9M61cw4MEqdKQnzL9IPDXNvrgtUAi9PN1RiDBI/23gM3GmHtCu8pyTiLSap/wEZE4gX9iM8XT2sPzvAn4rbHC8UzAGHOnMabTGNNF8HfyW2PMuyjT+QCISK2IzMquA28mqLRblp+5GcV0OxXGW4C3Ai8R6K13Tff9TOK+fwAcANIE2uJ7CTTTR4FtwCNAkz1WCKKUXgZeAC6a7vsvMJ+/INBXnwfW2+Wt5Ton4NXAc3Y+G4HP2PGzgaeB7cCPgSo7Xm23t9v9Z0/3HMaZ2+XAL8p9PvbeN9hlU/bvv1w/czNp0TIMiqIoFcRMlncURVGUU4wafUVRlApCjb6iKEoFoUZfURSlglCjryiKUkGo0VemHRHJ2EqKm2zly4+LyAl/NkXkU6H1LglVO1WUSkeNvjITSJqgkuKrCBKl3gL8w0mc71MTH6IolYkafWVGYYKU+9XAB212pSsi/ywif7Z10v8OQEQuF5EnReSXEvRc+JqIOCJyNxC3vxzus6d1ReQb9pfEb2wWrqJUJGr0lRmHMWYH4AJtBNnMCWPMxcDFwN+KyEJ76CrgQwT9FhYBNxpj7mD0l8O77HFLgK/YXxJ9wH+autkoysxCjb4y03kzQU2V9QTlnJsJjDjA08aYHSaomPkDgnIRhdhpjFlv158h6HWgKBWJllZWZhwicjaQIaigKMCHjDFrxxxzOUE9oDDFaoqMhNYzgMo7SsWiT/rKjEJEWoGvAV82QWGotcB/taWdEZGltuoiwCpbhdUB3gH8zo6ns8cripKPPukrM4G4lW+iBP14/y+QLeH8TQI55llb4rkbuMHu+zPwZWAxQRnhB+z4GuB5EXkWuGsqJqAo5YJW2VTKEivvfMIY87bpvhdFKSdU3lEURakg9ElfURSlgtAnfUVRlApCjb6iKEoFoUZfURSlglCjryiKUkGo0VcURakg/j+7fyjNRp+DjgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a_b4ou4TYqUN"
      },
      "source": [
        "## Masking"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s42Uydjkv0hF"
      },
      "source": [
        "Mask all the pad tokens in the batch of sequence. It ensures that the model does not treat padding as the input. The mask indicates where pad value `0` is present: it outputs a `1` at those locations, and a `0` otherwise."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U2i8-e1s8ti9"
      },
      "source": [
        "def create_padding_mask(seq):\n",
        "  seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n",
        "  \n",
        "  # add extra dimensions to add the padding\n",
        "  # to the attention logits.\n",
        "  return seq[:, tf.newaxis, tf.newaxis, :]  # (batch_size, 1, 1, seq_len)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A7BYeBCNvi7n",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "06fe9715-a176-4f1c-a1a9-ee3226c33360"
      },
      "source": [
        "x = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]])\n",
        "create_padding_mask(x)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(3, 1, 1, 5), dtype=float32, numpy=\n",
              "array([[[[0., 0., 1., 1., 0.]]],\n",
              "\n",
              "\n",
              "       [[[0., 0., 0., 1., 1.]]],\n",
              "\n",
              "\n",
              "       [[[1., 1., 1., 0., 0.]]]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z0hzukDBgVom"
      },
      "source": [
        "The look-ahead mask is used to mask the future tokens in a sequence. In other words, the mask indicates which entries should not be used.\n",
        "\n",
        "This means that to predict the third word, only the first and second word will be used. Similarly to predict the fourth word, only the first, second and the third word will be used and so on."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dVxS8OPI9uI0"
      },
      "source": [
        "def create_look_ahead_mask(size):\n",
        "  mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n",
        "  return mask  # (seq_len, seq_len)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yxKGuXxaBeeE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "52d40eff-3ff6-4157-fc36-2a2806c3b07a"
      },
      "source": [
        "x = tf.random.uniform((1, 3))\n",
        "temp = create_look_ahead_mask(x.shape[1])\n",
        "temp"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(3, 3), dtype=float32, numpy=\n",
              "array([[0., 1., 1.],\n",
              "       [0., 0., 1.],\n",
              "       [0., 0., 0.]], dtype=float32)>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xluDl5cXYy4y"
      },
      "source": [
        "## Scaled dot product attention"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vsxEE_-Wa1gF"
      },
      "source": [
        "<img src=\"https://www.tensorflow.org/images/tutorials/transformer/scaled_attention.png\" width=\"500\" alt=\"scaled_dot_product_attention\">\n",
        "\n",
        "The attention function used by the transformer takes three inputs: Q (query), K (key), V (value). The equation used to calculate the attention weights is:\n",
        "\n",
        "$$\\Large{Attention(Q, K, V) = softmax_k(\\frac{QK^T}{\\sqrt{d_k}}) V} $$\n",
        "\n",
        "The dot-product attention is scaled by a factor of square root of the depth. This is done because for large values of depth, the dot product grows large in magnitude pushing the softmax function where it has small gradients resulting in a very hard softmax. \n",
        "\n",
        "For example, consider that `Q` and `K` have a mean of 0 and variance of 1. Their matrix multiplication will have a mean of 0 and variance of `dk`. Hence, *square root of `dk`* is used for scaling (and not any other number) because the matmul of `Q` and `K` should have a mean of 0 and variance of 1, and you get a gentler softmax.\n",
        "\n",
        "The mask is multiplied with -1e9 (close to negative infinity). This is done because the mask is summed with the scaled matrix multiplication of Q and K and is applied immediately before a softmax. The goal is to zero out these cells, and large negative inputs to softmax are near zero in the output."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LazzUq3bJ5SH"
      },
      "source": [
        "def scaled_dot_product_attention(q, k, v, mask):\n",
        "  \"\"\"Calculate the attention weights.\n",
        "  q, k, v must have matching leading dimensions.\n",
        "  k, v must have matching penultimate dimension, i.e.: seq_len_k = seq_len_v.\n",
        "  The mask has different shapes depending on its type(padding or look ahead) \n",
        "  but it must be broadcastable for addition.\n",
        "  \n",
        "  Args:\n",
        "    q: query shape == (..., seq_len_q, depth)\n",
        "    k: key shape == (..., seq_len_k, depth)\n",
        "    v: value shape == (..., seq_len_v, depth_v)\n",
        "    mask: Float tensor with shape broadcastable \n",
        "          to (..., seq_len_q, seq_len_k). Defaults to None.\n",
        "    \n",
        "  Returns:\n",
        "    output, attention_weights\n",
        "  \"\"\"\n",
        "\n",
        "  matmul_qk = tf.matmul(q, k, transpose_b=True)  # (..., seq_len_q, seq_len_k)\n",
        "  \n",
        "  # scale matmul_qk\n",
        "  dk = tf.cast(tf.shape(k)[-1], tf.float32)\n",
        "  scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n",
        "\n",
        "  # add the mask to the scaled tensor.\n",
        "  if mask is not None:\n",
        "    scaled_attention_logits += (mask * -1e9)  \n",
        "\n",
        "  # softmax is normalized on the last axis (seq_len_k) so that the scores\n",
        "  # add up to 1.\n",
        "  attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)  # (..., seq_len_q, seq_len_k)\n",
        "\n",
        "  output = tf.matmul(attention_weights, v)  # (..., seq_len_q, depth_v)\n",
        "\n",
        "  return output, attention_weights"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FiqETnhCkoXh"
      },
      "source": [
        "As the softmax normalization is done on K, its values decide the amount of importance given to Q.\n",
        "\n",
        "The output represents the multiplication of the attention weights and the V (value) vector. This ensures that the words you want to focus on are kept as-is and the irrelevant words are flushed out."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n90YjClyInFy"
      },
      "source": [
        "def print_out(q, k, v):\n",
        "  temp_out, temp_attn = scaled_dot_product_attention(\n",
        "      q, k, v, None)\n",
        "  print ('Attention weights are:')\n",
        "  print (temp_attn)\n",
        "  print ('Output is:')\n",
        "  print (temp_out)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yAzUAf2DPlNt",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8a05704a-b01c-419e-d830-3d5b9c5bc376"
      },
      "source": [
        "np.set_printoptions(suppress=True)\n",
        "\n",
        "temp_k = tf.constant([[10,0,0],\n",
        "                      [0,10,0],\n",
        "                      [0,0,10],\n",
        "                      [0,0,10]], dtype=tf.float32)  # (4, 3)\n",
        "\n",
        "temp_v = tf.constant([[   1,0],\n",
        "                      [  10,0],\n",
        "                      [ 100,5],\n",
        "                      [1000,6]], dtype=tf.float32)  # (4, 2)\n",
        "\n",
        "# This `query` aligns with the second `key`,\n",
        "# so the second `value` is returned.\n",
        "temp_q = tf.constant([[0, 10, 0]], dtype=tf.float32)  # (1, 3)\n",
        "print_out(temp_q, temp_k, temp_v)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Attention weights are:\n",
            "tf.Tensor([[0. 1. 0. 0.]], shape=(1, 4), dtype=float32)\n",
            "Output is:\n",
            "tf.Tensor([[10.  0.]], shape=(1, 2), dtype=float32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zg6k-fGhgXra",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b1817bb0-f678-4b32-e442-7353ad89bcaf"
      },
      "source": [
        "# This query aligns with a repeated key (third and fourth), \n",
        "# so all associated values get averaged.\n",
        "temp_q = tf.constant([[0, 0, 10]], dtype=tf.float32)  # (1, 3)\n",
        "print_out(temp_q, temp_k, temp_v)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Attention weights are:\n",
            "tf.Tensor([[0.  0.  0.5 0.5]], shape=(1, 4), dtype=float32)\n",
            "Output is:\n",
            "tf.Tensor([[550.    5.5]], shape=(1, 2), dtype=float32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UAq3YOzUgXhb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "db5b9532-45b7-4e3d-cc1f-6d3b1d316ef4"
      },
      "source": [
        "# This query aligns equally with the first and second key, \n",
        "# so their values get averaged.\n",
        "temp_q = tf.constant([[10, 10, 0]], dtype=tf.float32)  # (1, 3)\n",
        "print_out(temp_q, temp_k, temp_v)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Attention weights are:\n",
            "tf.Tensor([[0.5 0.5 0.  0. ]], shape=(1, 4), dtype=float32)\n",
            "Output is:\n",
            "tf.Tensor([[5.5 0. ]], shape=(1, 2), dtype=float32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aOz-4_XIhaTP"
      },
      "source": [
        "Pass all the queries together."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6dlU8Tm-hYrF",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6f423a9d-ba5b-41ba-f440-62e97b3a095a"
      },
      "source": [
        "temp_q = tf.constant([[0, 0, 10], [0, 10, 0], [10, 10, 0]], dtype=tf.float32)  # (3, 3)\n",
        "print_out(temp_q, temp_k, temp_v)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Attention weights are:\n",
            "tf.Tensor(\n",
            "[[0.  0.  0.5 0.5]\n",
            " [0.  1.  0.  0. ]\n",
            " [0.5 0.5 0.  0. ]], shape=(3, 4), dtype=float32)\n",
            "Output is:\n",
            "tf.Tensor(\n",
            "[[550.    5.5]\n",
            " [ 10.    0. ]\n",
            " [  5.5   0. ]], shape=(3, 2), dtype=float32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kmzGPEy64qmA"
      },
      "source": [
        "## Multi-head attention"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fz5BMC8Kaoqo"
      },
      "source": [
        "<img src=\"https://www.tensorflow.org/images/tutorials/transformer/multi_head_attention.png\" width=\"500\" alt=\"multi-head attention\">\n",
        "\n",
        "\n",
        "Multi-head attention consists of four parts:\n",
        "*    Linear layers and split into heads.\n",
        "*    Scaled dot-product attention.\n",
        "*    Concatenation of heads.\n",
        "*    Final linear layer."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JPmbr6F1C-v_"
      },
      "source": [
        "Each multi-head attention block gets three inputs; Q (query), K (key), V (value). These are put through linear (Dense) layers and split up into multiple heads. \n",
        "\n",
        "The `scaled_dot_product_attention` defined above is applied to each head (broadcasted for efficiency). An appropriate mask must be used in the attention step.  The attention output for each head is then concatenated (using `tf.transpose`, and `tf.reshape`) and put through a final `Dense` layer.\n",
        "\n",
        "Instead of one single attention head, Q, K, and V are split into multiple heads because it allows the model to jointly attend to information at different positions from different representational spaces. After the split each head has a reduced dimensionality, so the total computation cost is the same as a single head attention with full dimensionality."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BSV3PPKsYecw"
      },
      "source": [
        "class MultiHeadAttention(tf.keras.layers.Layer):\n",
        "  def __init__(self, d_model, num_heads):\n",
        "    super(MultiHeadAttention, self).__init__()\n",
        "    self.num_heads = num_heads\n",
        "    self.d_model = d_model\n",
        "    \n",
        "    assert d_model % self.num_heads == 0\n",
        "    \n",
        "    self.depth = d_model // self.num_heads\n",
        "    \n",
        "    self.wq = tf.keras.layers.Dense(d_model)\n",
        "    self.wk = tf.keras.layers.Dense(d_model)\n",
        "    self.wv = tf.keras.layers.Dense(d_model)\n",
        "    \n",
        "    self.dense = tf.keras.layers.Dense(d_model)\n",
        "        \n",
        "  def split_heads(self, x, batch_size):\n",
        "    \"\"\"Split the last dimension into (num_heads, depth).\n",
        "    Transpose the result such that the shape is (batch_size, num_heads, seq_len, depth)\n",
        "    \"\"\"\n",
        "    x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n",
        "    return tf.transpose(x, perm=[0, 2, 1, 3])\n",
        "    \n",
        "  def call(self, v, k, q, mask):\n",
        "    batch_size = tf.shape(q)[0]\n",
        "    \n",
        "    q = self.wq(q)  # (batch_size, seq_len, d_model)\n",
        "    k = self.wk(k)  # (batch_size, seq_len, d_model)\n",
        "    v = self.wv(v)  # (batch_size, seq_len, d_model)\n",
        "    \n",
        "    q = self.split_heads(q, batch_size)  # (batch_size, num_heads, seq_len_q, depth)\n",
        "    k = self.split_heads(k, batch_size)  # (batch_size, num_heads, seq_len_k, depth)\n",
        "    v = self.split_heads(v, batch_size)  # (batch_size, num_heads, seq_len_v, depth)\n",
        "    \n",
        "    # scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)\n",
        "    # attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)\n",
        "    scaled_attention, attention_weights = scaled_dot_product_attention(\n",
        "        q, k, v, mask)\n",
        "    \n",
        "    scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])  # (batch_size, seq_len_q, num_heads, depth)\n",
        "\n",
        "    concat_attention = tf.reshape(scaled_attention, \n",
        "                                  (batch_size, -1, self.d_model))  # (batch_size, seq_len_q, d_model)\n",
        "\n",
        "    output = self.dense(concat_attention)  # (batch_size, seq_len_q, d_model)\n",
        "        \n",
        "    return output, attention_weights"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0D8FJue5lDyZ"
      },
      "source": [
        "Create a `MultiHeadAttention` layer to try out. At each location in the sequence, `y`, the `MultiHeadAttention` runs all 8 attention heads across all other locations in the sequence, returning a new vector of the same length at each location."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hu94p-_-2_BX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7aa62eb7-e0d3-4a08-8904-c6205a2043aa"
      },
      "source": [
        "temp_mha = MultiHeadAttention(d_model=512, num_heads=8)\n",
        "y = tf.random.uniform((1, 60, 512))  # (batch_size, encoder_sequence, d_model)\n",
        "out, attn = temp_mha(y, k=y, q=y, mask=None)\n",
        "out.shape, attn.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(TensorShape([1, 60, 512]), TensorShape([1, 8, 60, 60]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RdDqGayx67vv"
      },
      "source": [
        "## Point wise feed forward network"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gBqzJXGfHK3X"
      },
      "source": [
        "Point wise feed forward network consists of two fully-connected layers with a ReLU activation in between."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ET7xLt0yCT6Z"
      },
      "source": [
        "def point_wise_feed_forward_network(d_model, dff):\n",
        "  return tf.keras.Sequential([\n",
        "      tf.keras.layers.Dense(dff, activation='relu'),  # (batch_size, seq_len, dff)\n",
        "      tf.keras.layers.Dense(d_model)  # (batch_size, seq_len, d_model)\n",
        "  ])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mytb1lPyOHLB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e639a060-bcaf-4993-95af-95a0593cd395"
      },
      "source": [
        "sample_ffn = point_wise_feed_forward_network(512, 2048)\n",
        "sample_ffn(tf.random.uniform((64, 50, 512))).shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([64, 50, 512])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7e7hKcxn6-zd"
      },
      "source": [
        "## Encoder and decoder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yScbC0MUH8dS"
      },
      "source": [
        "<img src=\"https://www.tensorflow.org/images/tutorials/transformer/transformer.png\" width=\"600\" alt=\"transformer\">"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MfYJG-Kvgwy2"
      },
      "source": [
        "The transformer model follows the same general pattern as a standard [sequence to sequence with attention model](nmt_with_attention.ipynb). \n",
        "\n",
        "* The input sentence is passed through `N` encoder layers that generates an output for each word/token in the sequence.\n",
        "* The decoder attends on the encoder's output and its own input (self-attention) to predict the next word. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QFv-FNYUmvpn"
      },
      "source": [
        "### Encoder layer\n",
        "\n",
        "Each encoder layer consists of sublayers:\n",
        "\n",
        "1.   Multi-head attention (with padding mask) \n",
        "2.    Point wise feed forward networks. \n",
        "\n",
        "Each of these sublayers has a residual connection around it followed by a layer normalization. Residual connections help in avoiding the vanishing gradient problem in deep networks.\n",
        "\n",
        "The output of each sublayer is `LayerNorm(x + Sublayer(x))`. The normalization is done on the `d_model` (last) axis. There are N encoder layers in the transformer."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ncyS-Ms3i2x_"
      },
      "source": [
        "class EncoderLayer(tf.keras.layers.Layer):\n",
        "  def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
        "    super(EncoderLayer, self).__init__()\n",
        "\n",
        "    self.mha = MultiHeadAttention(d_model, num_heads)\n",
        "    self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
        "\n",
        "    self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    \n",
        "    self.dropout1 = tf.keras.layers.Dropout(rate)\n",
        "    self.dropout2 = tf.keras.layers.Dropout(rate)\n",
        "    \n",
        "  def call(self, x, training, mask):\n",
        "\n",
        "    attn_output, _ = self.mha(x, x, x, mask)  # (batch_size, input_seq_len, d_model)\n",
        "    attn_output = self.dropout1(attn_output, training=training)\n",
        "    out1 = self.layernorm1(x + attn_output)  # (batch_size, input_seq_len, d_model)\n",
        "    \n",
        "    ffn_output = self.ffn(out1)  # (batch_size, input_seq_len, d_model)\n",
        "    ffn_output = self.dropout2(ffn_output, training=training)\n",
        "    out2 = self.layernorm2(out1 + ffn_output)  # (batch_size, input_seq_len, d_model)\n",
        "    \n",
        "    return out2"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AzZRXdO0mI48",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2e94c0d8-7407-4439-ac1b-1ee30c36a975"
      },
      "source": [
        "sample_encoder_layer = EncoderLayer(512, 8, 2048)\n",
        "\n",
        "sample_encoder_layer_output = sample_encoder_layer(\n",
        "    tf.random.uniform((64, 43, 512)), False, None)\n",
        "\n",
        "sample_encoder_layer_output.shape  # (batch_size, input_seq_len, d_model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([64, 43, 512])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6LO_48Owmx_o"
      },
      "source": [
        "### Decoder layer\n",
        "\n",
        "Each decoder layer consists of sublayers:\n",
        "\n",
        "1.   Masked multi-head attention (with look ahead mask and padding mask)\n",
        "2.   Multi-head attention (with padding mask). V (value) and K (key) receive the *encoder output* as inputs. Q (query) receives the *output from the masked multi-head attention sublayer.*\n",
        "3.   Point wise feed forward networks\n",
        "\n",
        "Each of these sublayers has a residual connection around it followed by a layer normalization. The output of each sublayer is `LayerNorm(x + Sublayer(x))`. The normalization is done on the `d_model` (last) axis.\n",
        "\n",
        "There are N decoder layers in the transformer.\n",
        "\n",
        "As Q receives the output from decoder's first attention block, and K receives the encoder output, the attention weights represent the importance given to the decoder's input based on the encoder's output. In other words, the decoder predicts the next word by looking at the encoder output and self-attending to its own output. See the demonstration above in the scaled dot product attention section."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9SoX0-vd1hue"
      },
      "source": [
        "class DecoderLayer(tf.keras.layers.Layer):\n",
        "  def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
        "    super(DecoderLayer, self).__init__()\n",
        "\n",
        "    self.mha1 = MultiHeadAttention(d_model, num_heads)\n",
        "    self.mha2 = MultiHeadAttention(d_model, num_heads)\n",
        "\n",
        "    self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
        " \n",
        "    self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    \n",
        "    self.dropout1 = tf.keras.layers.Dropout(rate)\n",
        "    self.dropout2 = tf.keras.layers.Dropout(rate)\n",
        "    self.dropout3 = tf.keras.layers.Dropout(rate)\n",
        "    \n",
        "    \n",
        "  def call(self, x, enc_output, training, \n",
        "           look_ahead_mask, padding_mask):\n",
        "    # enc_output.shape == (batch_size, input_seq_len, d_model)\n",
        "\n",
        "    attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)  # (batch_size, target_seq_len, d_model)\n",
        "    attn1 = self.dropout1(attn1, training=training)\n",
        "    out1 = self.layernorm1(attn1 + x)\n",
        "    \n",
        "    attn2, attn_weights_block2 = self.mha2(\n",
        "        enc_output, enc_output, out1, padding_mask)  # (batch_size, target_seq_len, d_model)\n",
        "    attn2 = self.dropout2(attn2, training=training)\n",
        "    out2 = self.layernorm2(attn2 + out1)  # (batch_size, target_seq_len, d_model)\n",
        "    \n",
        "    ffn_output = self.ffn(out2)  # (batch_size, target_seq_len, d_model)\n",
        "    ffn_output = self.dropout3(ffn_output, training=training)\n",
        "    out3 = self.layernorm3(ffn_output + out2)  # (batch_size, target_seq_len, d_model)\n",
        "    \n",
        "    return out3, attn_weights_block1, attn_weights_block2"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ne2Bqx8k71l0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "17745e5a-632d-4dfb-8f72-ce8d0e83bfc9"
      },
      "source": [
        "sample_decoder_layer = DecoderLayer(512, 8, 2048)\n",
        "\n",
        "sample_decoder_layer_output, _, _ = sample_decoder_layer(\n",
        "    tf.random.uniform((64, 50, 512)), sample_encoder_layer_output, \n",
        "    False, None, None)\n",
        "\n",
        "sample_decoder_layer_output.shape  # (batch_size, target_seq_len, d_model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([64, 50, 512])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SE1H51Ajm0q1"
      },
      "source": [
        "### Encoder\n",
        "\n",
        "The `Encoder` consists of:\n",
        "1.   Input Embedding\n",
        "2.   Positional Encoding\n",
        "3.   N encoder layers\n",
        "\n",
        "The input is put through an embedding which is summed with the positional encoding. The output of this summation is the input to the encoder layers. The output of the encoder is the input to the decoder."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jpEox7gJ8FCI"
      },
      "source": [
        "class Encoder(tf.keras.layers.Layer):\n",
        "  def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,\n",
        "               maximum_position_encoding, rate=0.1):\n",
        "    super(Encoder, self).__init__()\n",
        "\n",
        "    self.d_model = d_model\n",
        "    self.num_layers = num_layers\n",
        "    \n",
        "    self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n",
        "    self.pos_encoding = positional_encoding(maximum_position_encoding, \n",
        "                                            self.d_model)\n",
        "    \n",
        "    \n",
        "    self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate) \n",
        "                       for _ in range(num_layers)]\n",
        "  \n",
        "    self.dropout = tf.keras.layers.Dropout(rate)\n",
        "        \n",
        "  def call(self, x, training, mask):\n",
        "\n",
        "    seq_len = tf.shape(x)[1]\n",
        "    \n",
        "    # adding embedding and position encoding.\n",
        "    x = self.embedding(x)  # (batch_size, input_seq_len, d_model)\n",
        "    x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
        "    x += self.pos_encoding[:, :seq_len, :]\n",
        "\n",
        "    x = self.dropout(x, training=training)\n",
        "    \n",
        "    for i in range(self.num_layers):\n",
        "      x = self.enc_layers[i](x, training, mask)\n",
        "    \n",
        "    return x  # (batch_size, input_seq_len, d_model)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8QG9nueFQKXx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a8675fe2-11c4-4c64-cbac-d99ece6788ba"
      },
      "source": [
        "sample_encoder = Encoder(num_layers=2, d_model=512, num_heads=8, \n",
        "                         dff=2048, input_vocab_size=8500,\n",
        "                         maximum_position_encoding=10000)\n",
        "temp_input = tf.random.uniform((64, 62), dtype=tf.int64, minval=0, maxval=200)\n",
        "\n",
        "sample_encoder_output = sample_encoder(temp_input, training=False, mask=None)\n",
        "\n",
        "print (sample_encoder_output.shape)  # (batch_size, input_seq_len, d_model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(64, 62, 512)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p-uO6ls8m2O5"
      },
      "source": [
        "### Decoder"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZtT7PKzrXkNr"
      },
      "source": [
        " The `Decoder` consists of:\n",
        "1.   Output Embedding\n",
        "2.   Positional Encoding\n",
        "3.   N decoder layers\n",
        "\n",
        "The target is put through an embedding which is summed with the positional encoding. The output of this summation is the input to the decoder layers. The output of the decoder is the input to the final linear layer."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d5_d5-PLQXwY"
      },
      "source": [
        "class Decoder(tf.keras.layers.Layer):\n",
        "  def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size,\n",
        "               maximum_position_encoding, rate=0.1):\n",
        "    super(Decoder, self).__init__()\n",
        "\n",
        "    self.d_model = d_model\n",
        "    self.num_layers = num_layers\n",
        "    \n",
        "    self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n",
        "    self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n",
        "    \n",
        "    self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate) \n",
        "                       for _ in range(num_layers)]\n",
        "    self.dropout = tf.keras.layers.Dropout(rate)\n",
        "    \n",
        "  def call(self, x, enc_output, training, \n",
        "           look_ahead_mask, padding_mask):\n",
        "\n",
        "    seq_len = tf.shape(x)[1]\n",
        "    attention_weights = {}\n",
        "    \n",
        "    x = self.embedding(x)  # (batch_size, target_seq_len, d_model)\n",
        "    x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
        "    x += self.pos_encoding[:, :seq_len, :]\n",
        "    \n",
        "    x = self.dropout(x, training=training)\n",
        "\n",
        "    for i in range(self.num_layers):\n",
        "      x, block1, block2 = self.dec_layers[i](x, enc_output, training,\n",
        "                                             look_ahead_mask, padding_mask)\n",
        "      \n",
        "      attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n",
        "      attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n",
        "    \n",
        "    # x.shape == (batch_size, target_seq_len, d_model)\n",
        "    return x, attention_weights"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a1jXoAMRZyvu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e870b5c9-ecb4-469f-8f35-73110aa7ef39"
      },
      "source": [
        "sample_decoder = Decoder(num_layers=2, d_model=512, num_heads=8, \n",
        "                         dff=2048, target_vocab_size=8000,\n",
        "                         maximum_position_encoding=5000)\n",
        "temp_input = tf.random.uniform((64, 26), dtype=tf.int64, minval=0, maxval=200)\n",
        "\n",
        "output, attn = sample_decoder(temp_input, \n",
        "                              enc_output=sample_encoder_output, \n",
        "                              training=False,\n",
        "                              look_ahead_mask=None, \n",
        "                              padding_mask=None)\n",
        "\n",
        "output.shape, attn['decoder_layer2_block2'].shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(TensorShape([64, 26, 512]), TensorShape([64, 8, 26, 62]))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y54xnJnuYgJ7"
      },
      "source": [
        "## Create the Transformer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uERO1y54cOKq"
      },
      "source": [
        "Transformer consists of the encoder, decoder and a final linear layer. The output of the decoder is the input to the linear layer and its output is returned."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PED3bIpOYkBu"
      },
      "source": [
        "class Transformer(tf.keras.Model):\n",
        "  def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, \n",
        "               target_vocab_size, pe_input, pe_target, rate=0.1):\n",
        "    super(Transformer, self).__init__()\n",
        "\n",
        "    self.encoder = Encoder(num_layers, d_model, num_heads, dff, \n",
        "                           input_vocab_size, pe_input, rate)\n",
        "\n",
        "    self.decoder = Decoder(num_layers, d_model, num_heads, dff, \n",
        "                           target_vocab_size, pe_target, rate)\n",
        "\n",
        "    self.final_layer = tf.keras.layers.Dense(target_vocab_size)\n",
        "    \n",
        "  def call(self, inp, tar, training, enc_padding_mask, \n",
        "           look_ahead_mask, dec_padding_mask):\n",
        "\n",
        "    enc_output = self.encoder(inp, training, enc_padding_mask)  # (batch_size, inp_seq_len, d_model)\n",
        "    \n",
        "    # dec_output.shape == (batch_size, tar_seq_len, d_model)\n",
        "    dec_output, attention_weights = self.decoder(\n",
        "        tar, enc_output, training, look_ahead_mask, dec_padding_mask)\n",
        "    \n",
        "    final_output = self.final_layer(dec_output)  # (batch_size, tar_seq_len, target_vocab_size)\n",
        "    \n",
        "    return final_output, attention_weights"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tJ4fbQcIkHW1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a3eb1c02-9497-41aa-df35-696d6f21932c"
      },
      "source": [
        "sample_transformer = Transformer(\n",
        "    num_layers=2, d_model=512, num_heads=8, dff=2048, \n",
        "    input_vocab_size=8500, target_vocab_size=8000, \n",
        "    pe_input=10000, pe_target=6000)\n",
        "\n",
        "temp_input = tf.random.uniform((64, 38), dtype=tf.int64, minval=0, maxval=200)\n",
        "temp_target = tf.random.uniform((64, 36), dtype=tf.int64, minval=0, maxval=200)\n",
        "\n",
        "fn_out, _ = sample_transformer(temp_input, temp_target, training=False, \n",
        "                               enc_padding_mask=None, \n",
        "                               look_ahead_mask=None,\n",
        "                               dec_padding_mask=None)\n",
        "\n",
        "fn_out.shape  # (batch_size, tar_seq_len, target_vocab_size)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TensorShape([64, 36, 8000])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wsINyf1VEQLC"
      },
      "source": [
        "## Set hyperparameters"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zVjWCxFNcgbt"
      },
      "source": [
        "To keep this example small and relatively fast, the values for *num_layers, d_model, and dff* have been reduced. \n",
        "\n",
        "The values used in the base model of transformer were; *num_layers=6*, *d_model = 512*, *dff = 2048*. See the [paper](https://arxiv.org/abs/1706.03762) for all the other versions of the transformer.\n",
        "\n",
        "Note: By changing the values below, you can get the model that achieved state of the art on many tasks."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lnJn5SLA2ahP"
      },
      "source": [
        "num_layers = 4\n",
        "d_model = 128\n",
        "dff = 512\n",
        "num_heads = 8\n",
        "\n",
        "input_vocab_size = tokenizer_pt.vocab_size + 2\n",
        "target_vocab_size = tokenizer_en.vocab_size + 2\n",
        "dropout_rate = 0.1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xYEGhEOtzn5W"
      },
      "source": [
        "## Optimizer"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GOmWW--yP3zx"
      },
      "source": [
        "Use the Adam optimizer with a custom learning rate scheduler according to the formula in the [paper](https://arxiv.org/abs/1706.03762).\n",
        "\n",
        "$$\\Large{lrate = d_{model}^{-0.5} * min(step{\\_}num^{-0.5}, step{\\_}num * warmup{\\_}steps^{-1.5})}$$\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iYQdOO1axwEI"
      },
      "source": [
        "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
        "  def __init__(self, d_model, warmup_steps=4000):\n",
        "    super(CustomSchedule, self).__init__()\n",
        "    \n",
        "    self.d_model = d_model\n",
        "    self.d_model = tf.cast(self.d_model, tf.float32)\n",
        "\n",
        "    self.warmup_steps = warmup_steps\n",
        "    \n",
        "  def __call__(self, step):\n",
        "    arg1 = tf.math.rsqrt(step)\n",
        "    arg2 = step * (self.warmup_steps ** -1.5)\n",
        "    \n",
        "    return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7r4scdulztRx"
      },
      "source": [
        "learning_rate = CustomSchedule(d_model)\n",
        "\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, \n",
        "                                     epsilon=1e-9)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f33ZCgvHpPdG",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        },
        "outputId": "e4ff0ce2-6933-4a07-c3ba-8bf39a2302c9"
      },
      "source": [
        "temp_learning_rate_schedule = CustomSchedule(d_model)\n",
        "\n",
        "plt.plot(temp_learning_rate_schedule(tf.range(40000, dtype=tf.float32)))\n",
        "plt.ylabel(\"Learning Rate\")\n",
        "plt.xlabel(\"Train Step\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0.5, 0, 'Train Step')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 63
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEGCAYAAACtqQjWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxV9Z34/9c7OwlkIQlhCRAIYQmKqBH3peKC2sq0xRHqd2qro9NWu3esfjvjOP7q/GrbqdZW67jgNipQaiu27nXfgLiggCC5Nwhhy02ASMISkry/f5xP4BJvkpvk3tyb3Pfz8cgj537OOZ/zvjeQd875fM77iKpijDHGREJSrAMwxhgzeFhSMcYYEzGWVIwxxkSMJRVjjDERY0nFGGNMxKTEOoBYKigo0JKSkliHYYwxA8q7775bp6qFodYldFIpKSmhsrIy1mEYY8yAIiKfdrbOLn8ZY4yJGEsqxhhjIsaSijHGmIixpGKMMSZiLKkYY4yJmKgmFRGZIyLrRaRKRK4PsT5dRBa79ctFpCRo3Q2ufb2InB/UvlBEakVkdSfH/LGIqIgUROM9GWOM6VzUkoqIJAN3AhcA5cACESnvsNmVwC5VnQTcBtzq9i0H5gPTgTnAXa4/gAddW6hjjgXOAzZF9M0YY4wJSzTPVGYBVarqV9VmYBEwt8M2c4GH3PJSYLaIiGtfpKoHVLUaqHL9oaqvATs7OeZtwHXAoKznr6osWbmZxgMtsQ7FGGNCimZSGQNsDnpd49pCbqOqLUADkB/mvkcQkbnAFlVd1c12V4tIpYhUBgKBcN5H3Phg826u+9OH/HTph7EOxRhjQhoUA/Uikgn8X+DG7rZV1XtUtUJVKwoLQ1YZiFubdu4F4IWPd8Q4EmOMCS2aSWULMDbodbFrC7mNiKQAOUB9mPsGKwUmAKtEZKPb/j0RGdmH+OOOL9AEQHNLG5tdgjHGmHgSzaSyEigTkQkikoY38L6swzbLgMvd8jzgJfWeb7wMmO9mh00AyoAVnR1IVT9S1RGqWqKqJXiXy45T1e2RfUux5Qs0IuItP7N6W2yDMcaYEKKWVNwYybXAc8DHwBJVXSMiN4vIxW6z+4F8EakCfgRc7/ZdAywB1gLPAteoaiuAiDwOvA1MEZEaEbkyWu8h3vgDTZw5uZDpo7N5ZvWgypfGmEEiqlWKVfVp4OkObTcGLe8HLulk31uAW0K0LwjjuCU9jTXetbUp1XWNnFKazwklw/nVc+vZ1rCPUTlDYh2aMcYcMigG6hPB1oZ97D/YxsTCLOYc5Q0VPWtnK8aYOGNJZYDwu0H60sKhlBYOZerIYTy1amuMozLGmCNZUhkgfIFGACYWZgEwd+YY3tu0m0/rm2IZljHGHMGSygDhDzQxLCOFwqHpAMydORoR+Mv7drZijIkfllQGCF+gkYmFQxE3p3h07hBOmpDPn9+vwZuFbYwxsWdJZYDwB5ooLcg6ou3Lx41hY/1e3t+8O0ZRGWPMkSypDACNB1rY/tl+SkcMPaL9gqNGkp6SxF/e76rYgDHG9B9LKgNAtZv5NbHDmcqwjFTOLS/iqVVbOdDSGovQjDHmCJZUBgB/nTfzq+OZCsAlFWPZtfcgz6+xIpPGmNizpDIA+GobSRIYn5/5uXWnTyqgOG8Ijy2355IZY2LPksoA4Ktrojgvk/SU5M+tS0oSFswax9v+evzuXhZjjIkVSyoDgK+2kdLCrE7XX1JRTEqSsGjl5k63McaY/mBJJc61tSkb65uYWPj58ZR2I4ZlcM60Ipa+W2MD9saYmLKkEufaC0mWdpFUAL524jh2NjVbkUljTExZUolz7U97nNjF5S+A0yYVMKEgi4VvbrQ77I0xMWNJJc61D753d6aSlCR889QSVm3ezXubdvVHaMYY8zmWVOKcL9DIsIwUCoamdbvtvOOLyRmSyn2vV/dDZMYY83mWVOKcP9B0RCHJrmSmpbBg1jieW7OdzTv39kN0xhhzJEsqcc4faOpyOnFHl58yniQRHnxrY/SCMsaYTkQ1qYjIHBFZLyJVInJ9iPXpIrLYrV8uIiVB625w7etF5Pyg9oUiUisiqzv09SsRWSciH4rIn0UkN5rvrT8cKiTZzXhKsFE5Q7jw6FEsXrmZhr0HoxidMcZ8XtSSiogkA3cCFwDlwAIRKe+w2ZXALlWdBNwG3Or2LQfmA9OBOcBdrj+AB11bRy8AR6nqDOAT4IaIvqEYqD70COHwz1QAvn1WKY0HWnjgLRtbMcb0r2ieqcwCqlTVr6rNwCJgbodt5gIPueWlwGzxBg/mAotU9YCqVgNVrj9U9TVgZ8eDqerzqtriXr4DFEf6DfW3w48QDv9MBWDaqGzOmVbEA29uZM9+O1sxxvSfaCaVMUBw3ZAa1xZyG5cQGoD8MPftyhXAM6FWiMjVIlIpIpWBQKAHXfY/f6DzQpLd+d7sSTTsO8gj73wahciMMSa0QTdQLyI/A1qAR0OtV9V7VLVCVSsKCwv7N7ge8gWaGDs8dCHJ7swozuXMyYXc93o1e5tbut/BGGMiIJpJZQswNuh1sWsLuY2IpAA5QH2Y+36OiHwD+CJwmQ6C28p9gcbPPZirJ7579iR2NjXz6DtWFt8Y0z+imVRWAmUiMkFE0vAG3pd12GYZcLlbnge85JLBMmC+mx02ASgDVnR1MBGZA1wHXKyqA/4mjbY2pbquqUczvzqqKBnOaZMK+MOrPhtbMcb0i6glFTdGci3wHPAxsERV14jIzSJysdvsfiBfRKqAHwHXu33XAEuAtcCzwDWq2gogIo8DbwNTRKRGRK50ff0eGAa8ICIfiMjd0Xpv/WHL7n0caGnr8SB9Rz+dM5WdTc3c+5o/QpEZY0znUqLZuao+DTzdoe3GoOX9wCWd7HsLcEuI9gWdbD+pT8HGGX9d76YTd3R0cQ4XzRjFfW9U808nl1A4LD0S4RljTEiDbqB+sPDV9m46cSg/OW8KzS1t/O6lDX3uyxhjumJJJU7568IvJNmdCQVZXHrCWB5bvomN7gzIGGOiwZJKnPJqfoVXSDIc359dRnpKEj//28cR6c8YY0KxpBKnfIHGbh/M1RMjsjP47uwyXvx4B6+sr41Yv8YYE8ySShxqPNDCjs8O9Gk6cSjfPLWECQVZ3PzUWppb2iLatzHGgCWVuHT4aY+RO1MBSE9J5sYvleOva+JBKzZpjIkCSypxyH/oufSRPVMB+MKUEcyeOoLfvriB7Q37I96/MSaxWVKJQ74+FJIMx41fKqdVlX9/cjWDoJqNMSaOWFKJQ/4+FJIMx/j8LH54zmReWLuDZ1Zvj8oxjDGJyZJKHPIFGiM+SN/RladN4Kgx2dz45Bp7QqQxJmIsqcSZ9kKSfalOHI6U5CRu/eoMdu1t5pan10b1WMaYxGFJJc60F5IsHRHdMxWA6aNzuPqMiSyprOFlu3fFGBMBllTizKFHCEf5TKXd92eXMaVoGNct/ZD6xgP9ckxjzOBlSSXORHM6cSgZqcncPn8mDXsPcsMTH9lsMGNMn1hSiTP+ukayI1RIMlzTRmVz3ZwpPL92B0sqN/fbcY0xg48llTjjq21iYgQLSYbrilMncEppPv/51NpDd/QbY0xPWVKJM/666E8nDiUpSfjvfzyG9JQkvvPoe+xrbu33GIwxA58llTiyZ/9Bdnx2IKLViXtiVM4Qbrt0Jut37OHf/mJ32xtjes6SShypjtAjhPvirCkj+O7ZZfzpvRoWr7TxFWNMz0Q1qYjIHBFZLyJVInJ9iPXpIrLYrV8uIiVB625w7etF5Pyg9oUiUisiqzv0NVxEXhCRDe57XjTfWzT4DlUn7v/LX8G+P7uM08sKuHHZGlZvaYhpLMaYgSVqSUVEkoE7gQuAcmCBiJR32OxKYJeqTgJuA251+5YD84HpwBzgLtcfwIOuraPrgb+rahnwd/d6QPEHmkgSGBelQpLhSk4Sbr90JgVZaVz1cCW1e6yasTEmPNE8U5kFVKmqX1WbgUXA3A7bzAUecstLgdniTXuaCyxS1QOqWg1Uuf5Q1deAnSGOF9zXQ8A/RPLN9Ad/oIlxUSwk2RP5Q9O59/IKdu89yNUPv8v+gzZwb4zpXjSTyhgg+KJ8jWsLuY2qtgANQH6Y+3ZUpKrb3PJ2oCjURiJytYhUikhlIBAI5330G+8RwrG99BVs+ugcbp8/kw827+a6pR/awL0xpluDcqBevd9+IX8Dquo9qlqhqhWFhYX9HFnnWl0hyVgO0ody/vSRXDdnCstWbeV3L1XFOhxjTJyLZlLZAowNel3s2kJuIyIpQA5QH+a+He0QkVGur1HAgKqQuNUVkoynM5V23z6zlK8cN4bfvPAJS2xGmDGmC9FMKiuBMhGZICJpeAPvyzpsswy43C3PA15yZxnLgPludtgEoAxY0c3xgvu6HHgyAu+h3/R3IcmeEBF+8ZUZnDG5kOuf+JAX1u6IdUjGmDgVtaTixkiuBZ4DPgaWqOoaEblZRC52m90P5ItIFfAj3IwtVV0DLAHWAs8C16hqK4CIPA68DUwRkRoRudL19QvgXBHZAJzjXg8Y7YUk+6PkfW+kpSTxh8uO4+jiXK597D1WVIeaK2GMSXSSyIOvFRUVWllZGeswAPjZnz/iqVVbWfUf5/V73a+e2NnUzLy73yKw5wCLrz6Z8tHZsQ7JGNPPRORdVa0ItW5QDtQPRP5AE6Uj+r+QZE8Nz0rj4StmMTQ9hcvue4ePt30W65CMMXHEkkqc8AUamVgQn5e+OirOy+Txq04iPSWZy+5bzvrte2IdkjEmTlhSiQN79h+kdk/sCkn2RklBFo9ffRKpycLX7n2HDTsssRhjLKnEhUOD9HE4nbgrEwqyeOyqk0hOEhbca5fCjDGWVOKCv669kOTAOVNpV1o4lMevPomUpCQu/Z+3efdTmxVmTCLrNqmIyGQR+Xt7VWARmSEi/xb90BKHP9BEcpLEvJBkb5UWDmXpt08mf2g6l923nFfWD6j7To0xERTOmcq9wA3AQQBV/RDvRkYTIb5AI2PzhsRFIcneKs7LZMm/nMzEgqFc9XAlT63aGuuQjDExEE5SyVTVjnezt0QjmETlDzQNuPGUUAqHpbPoX07i2LF5fG/R+9zzms+KUBqTYMJJKnUiUoor0Cgi84BtXe9iwtXapvjrmgbUzK+uZGek8vCVs7jwqFH819Pr+L9/Xs3B1rZYh2WM6ScpYWxzDXAPMFVEtgDVwGVRjSqBbN29j+Y4LSTZWxmpyfxuwbGMz8/krld81Ozay52XHUd2RmqsQzPGRFk4ZyqqqucAhcBUVT0tzP1MGOLlEcKRlpQkXDdnKr+cN4O3ffV89a632FjXFOuwjDFRFk5y+BOAqjapavsdbkujF1Ji8bl7VAbL5a+O/rFiLA9fOYtA4wG+9Ps3+PvHVuHYmMGs06QiIlNF5KtAjoh8JejrG0BGv0U4yPkDjeQMSSU/Ky3WoUTNKaUFPHXtaYzPz+TKhyr5zfPraW2zAXxjBqOuxlSmAF8EcoEvBbXvAa6KZlCJxHuEcFbcF5Lsq7HDM1n6rVP497+s5o6XqlhV08Dtl84kbxAnU2MSUadJRVWfBJ4UkZNV9e1+jCmh+ANNnF4WP481jqaM1GR+OW8GM8flctOyNVx4x+vcdulMTpqYH+vQjDEREs6Yyvsico2I3CUiC9u/oh5ZAmgvJFk6YnCOp4QiIlx24nie+PapZKQms+Ded/jN8+tpsWnHxgwK4SSVR4CRwPnAq3jPi7eStBHQXkhyoJS8j6Sji3P463dP46vHFXPHS1Vces87bN65N9ZhGWP6KJykMklV/x1oUtWHgIuAE6MbVmJoLyQ5KYHOVIJlpafw60uO4Y4Fx/LJ9j1c+NvXWVK52e7CN2YACyepHHTfd4vIUUAOMCJ6ISUOX60rJDk8MZNKu4uPGc3T3z+daaOzuW7ph3zjgZVs3b0v1mEZY3ohnKRyj4jkAf8GLAPWArdGNaoE4a9rZNzwTNJS7F7SscMzWXTVSfznxdNZUb2T8297jcUrN9lZizEDTLe/zVT1PlXdpaqvqepEVR0BPBNO5yIyR0TWi0iViFwfYn26iCx265eLSEnQuhtc+3oROb+7PkVktoi8JyIfiMgbIjIpnBhjyVfbxMSCxD5LCZaUJFx+SgnP/eAMykdn89M/fcTXF66wO/GNGUC6TCoicrKIzBOREe71DBF5DHizu45FJBm4E7gAKAcWiEh5h82uBHap6iTgNtwZkNtuPjAdmAPcJSLJ3fT5B+AyVZ0JPIZ3ZhW3WtuU6vrBU0gyksblZ/L4VSdx89zpvL9pN+fd/hq/fXEDB1paYx2aMaYbXd1R/ytgIfBV4G8i8nPgeWA5UBZG37OAKlX1q2ozsAiY22GbucBDbnkpMFu8uwDnAotU9YCqVgNVrr+u+lQg2y3nAHH9QI/2QpKDreZXpCQlCV8/uYS///hMzisv4rYXP2HO7a/zxoa6WIdmjOlCV3fUXwQcq6r73ZjKZuAoVd0YZt9j3D7tavj8rLFD26hqi4g0APmu/Z0O+45xy531+c/A0yKyD/gMOClUUCJyNXA1wLhx48J8K5FX5QpJDqbqxNFQlJ3B7792HP9YEeDGJ1fzf+5fzhdnjOKGC6cxJndIrMMzxnTQ1eWv/aq6H0BVdwEbepBQYuGHwIWqWgw8APwm1Eaqeo+qVqhqRWFh7O5kb79HZSA+lz4WzphcyLM/OIMfnFPGC2t3cPavX+FXz62j8YA9L86YeNLVmcpEEVkW9HpC8GtVvbibvrcAY4NeF7u2UNvUiEgK3mWr+m72/Vy7iBQCx6jqcte+GHi2m/hiyucKSQ632ldhy0hN5gfnTOaSirH86tl13PmyjyWVNfzkvMnMO34syUmDu36aMQNBV0ml4/jHf/ew75VAmYhMwEsI84GvddhmGXA58DYwD3hJVdUlr8dE5DfAaLwxnBWAdNLnLrxqypNV9RPgXODjHsbbr/wJUkgyGsbkDuH2+cdy+Skl/PxvH/PTP33EA29u5PoLpnLm5EL7TI2Joa4KSr7al47dGMm1wHNAMrBQVdeIyM1ApaouA+4HHhGRKmAnXpLAbbcE756YFuAaVW0FCNWna78K+JOItOElmSv6En+0+QJNnDk5MQpJRsux4/JY+q2Tefqj7fzi2Y/5xgMrOaEkj5+cN4UTrUilMTEhiXxzWUVFhVZWVvb7cffsP8jRNz3PdXOm8J2z4v52mgGhuaWNxZWb+f1LG9jx2QFOLyvgJ+dN4ZixubEOzZhBR0TeVdWKUOvsVu4YODxIbzO/IiUtJYl/Omk8r/7rF/jZhdNYs/Uz5t75Jlc9XMmHNbtjHZ4xCaOrMRUTJYefS28zvyItIzWZq86YyIITx7HwjWrufd3PC2t3cHpZAdd8YRInThhuYy7GRFG3SUVEnsK7sTBYA1AJ/E/7tGMTPn/ACklG29D0FL43u4xvnlrC/76zifvf8DP/nnc4fnwe13yhlC9MGWHJxZgoCOfylx9oBO51X5/hPU9lsnttesgXsEKS/WVYRirfPquUN356NjfPnc72hv1c8WAlF/z2df78fg3NLfZwMGMiKZzLX6eo6glBr58SkZWqeoKIrIlWYIOZP2CFJPtbRmoyXz+5hAWzxvHkB1v5wytV/HDxKv7r6XV8/aTxfO3EceQPTY91mMYMeOH8qTxURA7VM3HL7SPMzVGJahBrLyRZOsIG6WMhNTmJeccX88IPz+TBb57AtFHZ/PcLn3DyL17ip0s/ZN32z2IdojEDWjhnKj8G3hARH97NhxOA74hIFoeLQZowbdnlFZK0M5XYSkoSzpoygrOmjGDDjj088NZGnnivhsWVmzmlNJ//c9J4zi0vIjXZLlEa0xPdJhVVfVpEyoCprml90OD87VGLbJDyuUcI25lK/CgrGsZ/fflo/vW8KTy+chP/+/anfOfR9ygYms4/VhSzYNY4xg7PjHWYxgwI4U4pPh4ocdsfIyKo6sNRi2oQ89W66sR2phJ38rLS+M5Zk/iXM0p59ZNaHlu+ibtf9fGHV32cXlbI12aNY/a0EXb2YkwXwplS/AhQCnwAtD8lSQFLKr3gr2siN9MKScaz5CTh7KlFnD21iK2797F45WYWr9zMt/73XQqHpfMPM0fzleOKmTYqu/vOjEkw4ZypVADlmsj1XCLIV9vIxAIrJDlQjM4dwg/Pncx3z57Ey+sD/LFyMw++tZF7X6+mfFQ2XzluDHNnjqFwmM0cMwbCSyqrgZHAtijHkhD8dVZIciBKSU7i3PIizi0vYmdTM0+t2soT79Xw8799zP//zDrOnFzIV44bwznTishITY51uMbETDhJpQBYKyIrgAPtjWE8T8V08Nn+gwT2HLCaXwPc8Kw0Lj+lhMtPKWHDjj088f4W/vzeFl5aV0tWWjLnlBdx0dGjOHNKIekplmBMYgknqdwU7SASRXshyYlW82vQKCsaxk/nTOUn503hHX89f/1wK8+s3s6TH2xlWHoK504v4oszRnHapEKroGASQjhTivv0XBVzmP9QIUk7UxlskpOEUycVcOqkAm6eexRv+er566qtPLdmO0+8t4XsjBTOnz6SC44eySmlBXaJzAxanSYVEXlDVU8TkT0cWVBSAFVVm/rSQ75Aoyskafc8DGapyUmcObmQMycXcsuXj+aNqgB/XbWNZ1Zv54/v1pCZlsyZkws5t7yIs6eOIDfTZgKawaOrJz+e5r4P679wBjd/oMkKSSaYtJSkQ9OTD7S08ravnufX7uDFtTt4ZvV2kpOEWSXDD00CsJsszUAX1pMfRSQZKCIoCanqpijG1S/6+8mP5932KuOGZ3Lf5Sd0v7EZ1NralA+3NPDC2u08v2YHG9xNsVNHDnPlYwo5fnye3Whp4lJXT34M5+bH7wL/AewA2uuEKzAjYhEmgNY2ZWP9Xs6aMiLWoZg4kJQkzByby8yxufzr+VPZWNfEC2t38OLHO7jvdT93v+pjaHoKp07K56wpIzhzciGjc4fEOmxjuhXO7K/vA1NUtb6nnYvIHOC3QDJwn6r+osP6dLw7848H6oFLVXWjW3cDcCXeXfzfU9XnuupTvLsJfw5c4vb5g6re0dOYo6W9kKQ97dGEUlKQxVVnTOSqMyayZ/9B3qyq59VPAry6vpbn1uwAYHLRUM6aMoIzygqpKMmzwX4Tl8JJKpvxnvTYI+6S2Z3AuUANsFJElqnq2qDNrgR2qeokEZkP3ApcKiLlwHxgOjAaeFFEJrt9OuvzG8BYYKqqtolIXJ0StD9CeKLN/DLdGJaRypyjRjLnqJGoKhtqG3llfS2vfhLggTeruec1P2kpSVSMz+OU0nxOmVTAjDE5pNilMhMHwkkqfuAVEfkbR978+Jtu9psFVKmqH0BEFgFzgeCkMpfD98EsBX7vzjjmAotU9QBQLSJVrj+66PPbwNdUtc3FVxvGe+s3PptObHpBRJhcNIzJRcO4+oxSmg608I6/nrd83tevn/8Env+EoekpnDhhOCeX5nPqpAKmFA0jKclKAZn+F05S2eS+0txXuMbgneW0qwFO7GwbVW0RkQYg37W/02HfMW65sz5L8c5yvgwE8C6ZbegYlIhcDVwNMG7cuI6ro8YXsEKSpu+y0lOYPa2I2dOKANjZ1Mzbvnre8tXxlq+ev6/z/pbKz0rjxInDOaHE+5o2KptkSzKmH3SZVNwlrMmqelk/xdMX6cB+Va0Qka8AC4HTO26kqvcA94A3+6u/gvMHGq3cvYm44VlpXDRjFBfNGAXA1t37eNtXz5u+Opb7d/L0R9sBGJqewnHj85hVkscJJcM5ZmyujcmYqOgyqahqq4iMF5E0Ve3po4O34I1xtCt2baG2qRGRFCAHb8C+q307a68BnnDLfwYe6GG8UeWva+IsKyRpomx07hC+enwxXz2+GPCSzMqNO72v6l3e5TIgLTmJGcU5VJQMZ9aEPGaOzbOzaBMR4Y6pvCkiy4Cm9sYwxlRWAmUiMgHvF/984GsdtlkGXA68DcwDXlJVdcd6TER+gzdQXwaswLubv7M+/wJ8AagGzgQ+CeO99Yv2QpI2SG/62+jcIcyd6ZXnB9i9t5nKjbtYuXEnKzbudNOXvRP2kvxMZo7N5dhxecwcm8u0Udl2o67psXCSis99JQFh313vxkiuBZ7Dm/67UFXXiMjNQKWqLgPuBx5xA/E78ZIEbrsleAPwLcA1qtoKEKpPd8hfAI+KyA+BRuCfw4012toLSdp0YhNruZlpnFNexDnl3pjMvuZWVtXs5oPNu/lg027e8tXzlw+2Al41gKPH5LhE491TMyZ3iD0LyHQprDvqB6v+uqP+T+/W8OM/ruLFH53JJHs2vYljqsq2hv18sHk372/axfubdvPRlgYOtHj3PRcOS2fGmByOcl9Hj8mhKDvdEk2C6esd9YXAdXj3jGS0t6vq2RGLcJDz11khSTMwiAijc4cwOncIFx7tDf4fbG1j3bY9vL95Fx+4JPPy+lra3N+jBUPTOWpMNkcHJZpRORmWaBJUOJe/HgUWA18EvoU3BhKIZlCDja+2ifFWSNIMUKnJSRxdnMPRxTl8/WSvbW9zC2u3fsbqLQ18tMX7/tongUOJJj8rjeljcjh6TDbTRmUzdWQ2JfmZdoNmAggnqeSr6v0i8n33bJVXRWRltAMbTPx1jfZgLjOoZKalUFEynIqS4Yfa9jW38vF2l2hqGvhoSwN3V9XR6jJNekoSk4uGMXXkMC/RjBrGtJHZ5Nmss0ElnKRy0H3fJiIXAVuB4V1sb4K0tikb6/byBSskaQa5IWnJHDcuj+PG5R1q23+wlaraRtZt38O6bZ+xbvseXlpXyx/frTm0TVF2OlNHHk4yU0cNo7RwqFVoHqDCSSo/F5Ec4MfA74Bs4IdRjWoQqdm1l+bWNjtTMQkpIzX50KB+sMCeA6zb/hnrtu3hY/f9bV89za3ehIDUZGFCQRZlI4ZROmIoZSOGUlY0lAkFWaSn2E2b8Sycxwn/1S024N0HYnrg8HRim/VlTLvCYekUDivk9LLDNwQfbG2juq6Jj90ZzYYdjazd9hnPrN52aKwmSWB8fhaTghLNpMJhlI7IIjMtnL+RTcx/dBMAABPjSURBVLSFM/trMvAHoEhVjxKRGcDFqvrzqEc3CFh1YmPCk5qcdKh45tyg9v0HW6mua2JDbSNVO/awobaRDbWNvLyulpa2w7dEFOcNoWzEUEoLhzKhMIsJBVlMLBhqU577WTip/V7gX4H/AVDVD0XkMbxnl5huWCFJY/omIzWZaaO8WWTBDra28Wl9Ext2NB5KNBt27OEtX/2h+2oAMtOSKcnPYkJhFhMLvGTTnnByMlP7++0MeuEklUxVXdEh07dEKZ5Bxx9otEtfxkRBanISk0YMY9KIYVwQ1N7Wpmz7bD/VgSaq6xrx1zVRXdfE6i0NPPPR4Utp4BXknBCUaCYUZDFueCbj8jPJzrCE0xvhJJU6ESnFe4QwIjIP2BbVqAYRX6CJL0yxQpLG9JekJGFM7hDG5A7htLKCI9Y1t7Sxaedequu8hFPtEs7rGwIsDZqRBpCbmcr44ZmMHZ7J+PxMxh1azmJkdoY9SqAT4SSVa/BKxU8VkS14BRsHQin8mGvYd5C6xgOUWmkWY+JCWkoSk0YMdeWSio5Y13ighU31e9m0s4lNO/fyaf1eNu3cy0dbGnh29fYjxm/SkpMozhtyRMJpP8MZkzuEYQl8lhPO7C8/cI6IZAFJqrpHRH4A3B716AY4f/sgvT1HxZi4NzQ9hfLR2ZSPzv7cupbWNrY17D8i2bQnn/c27WLP/iNHBLIzUijOy2RMnnfGVJznfY3J9dryMlMH7eSBsOfgqWpT0MsfYUmlW+3TiW3mlzEDW0pyEmPd5a9TJx25TlVp2HfwULLZsnsfW3btY8vufWyq38tbVXU0NbcesU9mWrJ3ia5DsinOG0Jx7hAKhqYP2MdB93Zi98B8t/3MF2gkJUkYn2+FJI0ZrESE3Mw0cjPTOGZs7ufWtyedml37qHHJxks6e6nZtY8PNu9m996DR+yTlpzEyJwMRuZkMDong5E5Qxh16PUQRuZkkJ+VFpeJp7dJJXHr5feAP9DEuOGZVm7CmAQWnHQ6VhZo13igha2791Gzay9bdu2jZvc+tjfsZ9vu/by7aRfbG7ZxsPXIX7upyUJR9uEk0550RrkENConIyZnPJ0mFRHZQ+jkIcCQqEU0iHiFJO3SlzGma0PTUw7d+BlKW5tS39TsJZqGfWz/bD9bd+9ne8O+Q8+/eXb1/kNlbtqlJHmJZ2ROBkXZ6d5ydgZF2RmcUprPiOyMkMfri06TiqqG/ZRH83lWSNIYEylJSeJK26RzdHHosx1VZWdTM9sa9rOt4XDC8Zb3s277Hl5dHzg0vvPwFbP6N6mYvmkvJGk3Phpj+oOIkD80nfyh6Z1eZgPYs/8gOz47wKicyCcUsKQSNYdrftl0YmNM/BiWkRrV+2iiOoIsInNEZL2IVInI9SHWp4vIYrd+uYiUBK27wbWvF5Hze9DnHSLSGK33FC6bTmyMSURRSyoikgzcCVwAlAMLRKS8w2ZXArtUdRJwG3Cr27ccmA9MB+YAd4lIcnd9ikgFkEcc8AWayLNCksaYBBPNM5VZQJWq+lW1GVgER1S0xr1+yC0vBWaLd5vpXGCRqh5Q1WqgyvXXaZ8u4fwKuC6K7ylsvoDN/DLGJJ5oJpUxwOag1zWuLeQ2qtqC9yCw/C727arPa4FlqtplsUsRuVpEKkWkMhAI9OgN9YQ/0ESpjacYYxLMoLgrT0RGA5fgPe64S6p6j6pWqGpFYWF0qge3F5K0MxVjTKKJZlLZAowNel3s2kJuIyIpQA5Q38W+nbUfC0wCqkRkI5ApIlWReiM9ZYUkjTGJKppJZSVQJiITRCQNb+B9WYdtlgGXu+V5wEuqqq59vpsdNgEoA1Z01qeq/k1VR6pqiaqWAHvd4H9M+NqfS28l740xCSZq96moaouIXAs8ByQDC1V1jYjcDFSq6jLgfuARd1axEy9J4LZbAqzFe8rkNaraChCqz2i9h97yu0KS44ZbIUljTGKJ6s2Pqvo08HSHthuDlvfjjYWE2vcW4JZw+gyxTUxPEfyBJsblWyFJY0zisd96UeALNDKxwC59GWMSjyWVCGtpbePT+r2UjrBBemNM4rGkEmE1u/Z5hSTtTMUYk4AsqUSYv84KSRpjEpcllQhrLyRpJe+NMYnIkkqE+QKN5GWmkmeFJI0xCciSSoT5Ak12lmKMSViWVCLMH2i08RRjTMKypBJBDXsPUtfYbIUkjTEJy5JKBPnczC+7/GWMSVSWVCLo8COE7fKXMSYxWVKJICskaYxJdJZUIsgXaLRCksaYhGa//SLIb9OJjTEJzpJKhLS0trGxvsnGU4wxCc2SSoTU7NrHwVa1QpLGmIRmSSVC2gtJWsl7Y0wis6QSIb5aN53YzlSMMQnMkkqE+OsaGZ6VZoUkjTEJLapJRUTmiMh6EakSketDrE8XkcVu/XIRKQlad4NrXy8i53fXp4g86tpXi8hCEUmN5nvryFfbxMQCu/RljElsUUsqIpIM3AlcAJQDC0SkvMNmVwK7VHUScBtwq9u3HJgPTAfmAHeJSHI3fT4KTAWOBoYA/xyt9xaKv84KSRpjTDTPVGYBVarqV9VmYBEwt8M2c4GH3PJSYLaIiGtfpKoHVLUaqHL9ddqnqj6tDrACKI7ieztCeyFJu0fFGJPooplUxgCbg17XuLaQ26hqC9AA5Hexb7d9uste/wQ82+d3ECbfoUcIW1IxxiS2wThQfxfwmqq+HmqliFwtIpUiUhkIBCJywMOPELbLX8aYxBbNpLIFGBv0uti1hdxGRFKAHKC+i3277FNE/gMoBH7UWVCqeo+qVqhqRWFhYQ/fUmg+V0hyrBWSNMYkuGgmlZVAmYhMEJE0vIH3ZR22WQZc7pbnAS+5MZFlwHw3O2wCUIY3TtJpnyLyz8D5wAJVbYvi+/ocf6CR8VZI0hhjSIlWx6raIiLXAs8BycBCVV0jIjcDlaq6DLgfeEREqoCdeEkCt90SYC3QAlyjqq0Aofp0h7wb+BR42xvr5wlVvTla7y+YL9Bk4ynGGEMUkwp4M7KApzu03Ri0vB+4pJN9bwFuCadP1x7V99KZltY2Pq1vYva0EbE4vDHGxBW7XtNHhwpJ2pmKMcZYUukrX6D9ufQ288sYYyyp9NGh59JbIUljjLGk0le+gBWSNMaYdpZU+sgfsEKSxhjTzpJKH/kCjTZIb4wxjiWVPmjYe5D6pmarTmyMMY4llT5oLyRpZyrGGOOxpNIHvtr26sR2pmKMMWBJpU/8dU2kJlshSWOMaWdJpQ98tY2MG26FJI0xpp39NuwDf50VkjTGmGCWVHqpvZCkDdIbY8xhllR6abMrJGmD9MYYc5gllV7yB2w6sTHGdGRJpZesOrExxnyeJZVe8geayM9KIzfTCkkaY0w7Syq95As02niKMcZ0YEmll7zqxDaeYowxwSyp9MLuvc3UNzVTOsLOVIwxJlhUk4qIzBGR9SJSJSLXh1ifLiKL3frlIlIStO4G175eRM7vrk8RmeD6qHJ9Rm2ww2dPezTGmJCillREJBm4E7gAKAcWiEh5h82uBHap6iTgNuBWt285MB+YDswB7hKR5G76vBW4zfW1y/UdFYemE4+wpGKMMcGieaYyC6hSVb+qNgOLgLkdtpkLPOSWlwKzRURc+yJVPaCq1UCV6y9kn26fs10fuD7/IVpvzBdwhSTzhkTrEMYYMyBFM6mMATYHva5xbSG3UdUWoAHI72Lfztrzgd2uj86OBYCIXC0ilSJSGQgEevG2oCQ/ky8fO4YUKyRpjDFHSLjfiqp6j6pWqGpFYWFhr/qYP2scv5x3TIQjM8aYgS+aSWULMDbodbFrC7mNiKQAOUB9F/t21l4P5Lo+OjuWMcaYKItmUlkJlLlZWWl4A+/LOmyzDLjcLc8DXlJVde3z3eywCUAZsKKzPt0+L7s+cH0+GcX3ZowxJoSU7jfpHVVtEZFrgeeAZGChqq4RkZuBSlVdBtwPPCIiVcBOvCSB224JsBZoAa5R1VaAUH26Q/4UWCQiPwfed30bY4zpR+L9kZ+YKioqtLKyMtZhGGPMgCIi76pqRah1CTdQb4wxJnosqRhjjIkYSyrGGGMixpKKMcaYiEnogXoRCQCf9nL3AqAuguFEisXVMxZXz1hcPROvcUHfYhuvqiHvHk/opNIXIlLZ2eyHWLK4esbi6hmLq2fiNS6IXmx2+csYY0zEWFIxxhgTMZZUeu+eWAfQCYurZyyunrG4eiZe44IoxWZjKsYYYyLGzlSMMcZEjCUVY4wxEWNJpRdEZI6IrBeRKhG5vh+Ot1FEPhKRD0Sk0rUNF5EXRGSD+57n2kVE7nCxfSgixwX1c7nbfoOIXN7Z8bqJZaGI1IrI6qC2iMUiIse791rl9pU+xHWTiGxxn9sHInJh0Lob3DHWi8j5Qe0hf7bucQvLXfti9+iF7mIaKyIvi8haEVkjIt+Ph8+ri7hi+nm5/TJEZIWIrHKx/WdX/Yn3eIzFrn25iJT0NuZexvWgiFQHfWYzXXt//ttPFpH3ReSv8fBZoar21YMvvJL7PmAikAasAsqjfMyNQEGHtl8C17vl64Fb3fKFwDOAACcBy137cMDvvue55bxexHIGcBywOhqx4D035yS3zzPABX2I6ybgJyG2LXc/t3Rggvt5Jnf1swWWAPPd8t3At8OIaRRwnFseBnzijh3Tz6uLuGL6ebltBRjqllOB5e79hewP+A5wt1ueDyzubcy9jOtBYF6I7fvz3/6PgMeAv3b12ffXZ2VnKj03C6hSVb+qNgOLgLkxiGMu8JBbfgj4h6D2h9XzDt4TMUcB5wMvqOpOVd0FvADM6elBVfU1vGffRDwWty5bVd9R71/7w0F99SauzswFFqnqAVWtBqrwfq4hf7buL8azgaUh3mNXMW1T1ffc8h7gY2AMMf68uoirM/3yebl4VFUb3ctU96Vd9Bf8WS4FZrvj9yjmPsTVmX75WYpIMXARcJ973dVn3y+flSWVnhsDbA56XUPX/yEjQYHnReRdEbnatRWp6ja3vB0o6ia+aMYdqVjGuOVIxnitu/ywUNxlpl7ElQ/sVtWW3sblLjUci/cXbtx8Xh3igjj4vNzlnA+AWrxfur4u+jsUg1vf4I4f8f8HHeNS1fbP7Bb3md0mIukd4wrz+L39Wd4OXAe0udddffb98llZUhkYTlPV44ALgGtE5Izgle4vm7iYGx5PsQB/AEqBmcA24L9jEYSIDAX+BPxAVT8LXhfLzytEXHHxealqq6rOBIrx/lqeGos4OuoYl4gcBdyAF98JeJe0ftpf8YjIF4FaVX23v44ZDksqPbcFGBv0uti1RY2qbnHfa4E/4/1H2+FOmXHfa7uJL5pxRyqWLW45IjGq6g73i6ANuBfvc+tNXPV4ly9SOrR3S0RS8X5xP6qqT7jmmH9eoeKKh88rmKruBl4GTu6iv0MxuPU57vhR+38QFNccdylRVfUA8AC9/8x687M8FbhYRDbiXZo6G/gtsf6suht0sa/PDYql4A2uTeDw4NX0KB4vCxgWtPwW3ljIrzhysPeXbvkijhwgXOHahwPVeIODeW55eC9jKuHIAfGIxcLnBysv7ENco4KWf4h33RhgOkcOTPrxBiU7/dkCf+TIwc/vhBGP4F0bv71De0w/ry7iiunn5bYtBHLd8hDgdeCLnfUHXMORg89LehtzL+MaFfSZ3g78Ikb/9s/i8EB9bD+r3vxSSfQvvJkdn+Bd6/1ZlI810f0wVwFr2o+Hdy3078AG4MWgf5gC3Oli+wioCOrrCrxBuCrgm72M53G8SyMH8a6xXhnJWIAKYLXb5/e4qg+9jOsRd9wPgWUc+UvzZ+4Y6wmaZdPZz9b9HFa4eP8IpIcR02l4l7Y+BD5wXxfG+vPqIq6Yfl5uvxnA+y6G1cCNXfUHZLjXVW79xN7G3Mu4XnKf2Wrgfzk8Q6zf/u27fc/icFKJ6WdlZVqMMcZEjI2pGGOMiRhLKsYYYyLGkooxxpiIsaRijDEmYiypGGOMiRhLKsb0kIjkB1Wl3S5HVvbtshqviFSIyB09PN4VrnrthyKyWkTmuvZviMjovrwXYyLNphQb0wcichPQqKq/DmpL0cO1l/rafzHwKl5V4QZXWqVQVatF5BW8qsKVkTiWMZFgZyrGRIB7rsbdIrIc+KWIzBKRt91zLt4SkSluu7OCnntxkyvc+IqI+EXkeyG6HgHsARoBVLXRJZR5eDfLPerOkIa453G86gqPPhdUCuYVEfmt2261iMwKcRxjIsKSijGRUwycoqo/AtYBp6vqscCNwH91ss9UvHLos4D/cDW5gq0CdgDVIvKAiHwJQFWXApXAZeoVOWwBfof3bI/jgYXALUH9ZLrtvuPWGRMVKd1vYowJ0x9VtdUt5wAPiUgZXkmUjsmi3d/UK0Z4QERq8crgHyqBrqqtIjIHrwrubOA2ETleVW/q0M8U4CjgBe8RGSTjla1p97jr7zURyRaRXPUKIxoTUZZUjImcpqDl/w94WVW/7J5Z8kon+xwIWm4lxP9J9QY+VwArROQFvGq4N3XYTIA1qnpyJ8fpOHhqg6kmKuzylzHRkcPhMuHf6G0nIjJagp5vjvesk0/d8h68xwGDVwiwUEROdvulisj0oP0ude2nAQ2q2tDbmIzpip2pGBMdv8S7/PVvwN/60E8q8Gs3dXg/EAC+5dY9CNwtIvvwnjkyD7hDRHLw/m/fjlfZGmC/iLzv+ruiD/EY0yWbUmzMIGdTj01/sstfxhhjIsbOVIwxxkSMnakYY4yJGEsqxhhjIsaSijHGmIixpGKMMSZiLKkYY4yJmP8HLnCrOoiHd3sAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YgkDE7hzo8r5"
      },
      "source": [
        "## Loss and metrics"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oxGJtoDuYIHL"
      },
      "source": [
        "Since the target sequences are padded, it is important to apply a padding mask when calculating the loss."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MlhsJMm0TW_B"
      },
      "source": [
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n",
        "    from_logits=True, reduction='none')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "67oqVHiT0Eiu"
      },
      "source": [
        "def loss_function(real, pred):\n",
        "  mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
        "  loss_ = loss_object(real, pred)\n",
        "\n",
        "  mask = tf.cast(mask, dtype=loss_.dtype)\n",
        "  loss_ *= mask\n",
        "  \n",
        "  return tf.reduce_sum(loss_)/tf.reduce_sum(mask)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "phlyxMnm-Tpx"
      },
      "source": [
        "train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
        "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n",
        "    name='train_accuracy')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aeHumfr7zmMa"
      },
      "source": [
        "## Training and checkpointing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UiysUa--4tOU"
      },
      "source": [
        "transformer = Transformer(num_layers, d_model, num_heads, dff,\n",
        "                          input_vocab_size, target_vocab_size, \n",
        "                          pe_input=input_vocab_size, \n",
        "                          pe_target=target_vocab_size,\n",
        "                          rate=dropout_rate)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZOJUSB1T8GjM"
      },
      "source": [
        "def create_masks(inp, tar):\n",
        "  # Encoder padding mask\n",
        "  enc_padding_mask = create_padding_mask(inp)\n",
        "  \n",
        "  # Used in the 2nd attention block in the decoder.\n",
        "  # This padding mask is used to mask the encoder outputs.\n",
        "  dec_padding_mask = create_padding_mask(inp)\n",
        "  \n",
        "  # Used in the 1st attention block in the decoder.\n",
        "  # It is used to pad and mask future tokens in the input received by \n",
        "  # the decoder.\n",
        "  look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n",
        "  dec_target_padding_mask = create_padding_mask(tar)\n",
        "  combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)\n",
        "  \n",
        "  return enc_padding_mask, combined_mask, dec_padding_mask"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fzuf06YZp66w"
      },
      "source": [
        "Create the checkpoint path and the checkpoint manager. This will be used to save checkpoints every `n` epochs."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hNhuYfllndLZ"
      },
      "source": [
        "checkpoint_path = \"./checkpoints/train\"\n",
        "\n",
        "ckpt = tf.train.Checkpoint(transformer=transformer,\n",
        "                           optimizer=optimizer)\n",
        "\n",
        "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n",
        "\n",
        "# if a checkpoint exists, restore the latest checkpoint.\n",
        "if ckpt_manager.latest_checkpoint:\n",
        "  ckpt.restore(ckpt_manager.latest_checkpoint)\n",
        "  print ('Latest checkpoint restored!!')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0Di_Yaa1gf9r"
      },
      "source": [
        "The target is divided into tar_inp and tar_real. tar_inp is passed as an input to the decoder. `tar_real` is that same input shifted by 1: At each location in `tar_input`, `tar_real` contains the  next token that should be predicted.\n",
        "\n",
        "For example, `sentence` = \"SOS A lion in the jungle is sleeping EOS\"\n",
        "\n",
        "`tar_inp` =  \"SOS A lion in the jungle is sleeping\"\n",
        "\n",
        "`tar_real` = \"A lion in the jungle is sleeping EOS\"\n",
        "\n",
        "The transformer is an auto-regressive model: it makes predictions one part at a time, and uses its output so far to decide what to do next. \n",
        "\n",
        "During training this example uses teacher-forcing (like in the [text generation tutorial](./text_generation.ipynb)). Teacher forcing is passing the true output to the next time step regardless of what the model predicts at the current time step.\n",
        "\n",
        "As the transformer predicts each word, *self-attention* allows it to look at the previous words in the input sequence to better predict the next word.\n",
        "\n",
        "To prevent the model from peeking at the expected output the model uses a look-ahead mask."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LKpoA6q1sJFj"
      },
      "source": [
        "EPOCHS = 30"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F7d7KGpsW6qK"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iJwmp9OE29oj"
      },
      "source": [
        "# The @tf.function trace-compiles train_step into a TF graph for faster\n",
        "# execution. The function specializes to the precise shape of the argument\n",
        "# tensors. To avoid re-tracing due to the variable sequence lengths or variable\n",
        "# batch sizes (the last batch is smaller), use input_signature to specify\n",
        "# more generic shapes.\n",
        "\n",
        "train_step_signature = [\n",
        "    tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n",
        "    tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n",
        "]\n",
        "\n",
        "@tf.function(input_signature=train_step_signature)\n",
        "def train_step(inp, tar):\n",
        "  tar_inp = tar[:, :-1]\n",
        "  tar_real = tar[:, 1:]\n",
        "  \n",
        "  enc_padding_mask, combined_mask, dec_padding_mask = create_masks(inp, tar_inp)\n",
        "  \n",
        "  with tf.GradientTape() as tape:\n",
        "    predictions, _ = transformer(inp, tar_inp, \n",
        "                                 True, \n",
        "                                 enc_padding_mask, \n",
        "                                 combined_mask, \n",
        "                                 dec_padding_mask)\n",
        "    loss = loss_function(tar_real, predictions)\n",
        "\n",
        "  gradients = tape.gradient(loss, transformer.trainable_variables)    \n",
        "  optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n",
        "  \n",
        "  train_loss(loss)\n",
        "  train_accuracy(tar_real, predictions)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qM2PDWGDJ_8V"
      },
      "source": [
        "Portuguese is used as the input language and English is the target language."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bbvmaKNiznHZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "58232a37-2b84-40b4-d62a-0e6769dcb7d2"
      },
      "source": [
        "for epoch in range(EPOCHS):\n",
        "  start = time.time()\n",
        "  \n",
        "  train_loss.reset_states()\n",
        "  train_accuracy.reset_states()\n",
        "  \n",
        "  # inp -> portuguese, tar -> english\n",
        "  for (batch, (inp, tar)) in enumerate(train_dataset):\n",
        "    train_step(inp, tar)\n",
        "    \n",
        "    if batch % 50 == 0:\n",
        "      print ('Epoch {} Batch {} Loss {:.4f} Accuracy {:.4f}'.format(\n",
        "          epoch + 1, batch, train_loss.result(), train_accuracy.result()))\n",
        "      \n",
        "  if (epoch + 1) % 5 == 0:\n",
        "    ckpt_save_path = ckpt_manager.save()\n",
        "    print ('Saving checkpoint for epoch {} at {}'.format(epoch+1,\n",
        "                                                         ckpt_save_path))\n",
        "    \n",
        "  print ('Epoch {} Loss {:.4f} Accuracy {:.4f}'.format(epoch + 1, \n",
        "                                                train_loss.result(), \n",
        "                                                train_accuracy.result()))\n",
        "\n",
        "  print ('Time taken for 1 epoch: {} secs\\n'.format(time.time() - start))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1 Batch 0 Loss 9.0547 Accuracy 0.0000\n",
            "Epoch 1 Batch 50 Loss 8.9988 Accuracy 0.0003\n",
            "Epoch 1 Batch 100 Loss 8.9019 Accuracy 0.0132\n",
            "Epoch 1 Batch 150 Loss 8.7945 Accuracy 0.0182\n",
            "Epoch 1 Batch 200 Loss 8.6645 Accuracy 0.0208\n",
            "Epoch 1 Loss 8.6176 Accuracy 0.0213\n",
            "Time taken for 1 epoch: 40.939945459365845 secs\n",
            "\n",
            "Epoch 2 Batch 0 Loss 7.9727 Accuracy 0.0345\n",
            "Epoch 2 Batch 50 Loss 7.7590 Accuracy 0.0283\n",
            "Epoch 2 Batch 100 Loss 7.5607 Accuracy 0.0283\n",
            "Epoch 2 Batch 150 Loss 7.3728 Accuracy 0.0284\n",
            "Epoch 2 Batch 200 Loss 7.2046 Accuracy 0.0285\n",
            "Epoch 2 Loss 7.1625 Accuracy 0.0284\n",
            "Time taken for 1 epoch: 14.97199010848999 secs\n",
            "\n",
            "Epoch 3 Batch 0 Loss 6.3379 Accuracy 0.0278\n",
            "Epoch 3 Batch 50 Loss 6.5057 Accuracy 0.0288\n",
            "Epoch 3 Batch 100 Loss 6.4311 Accuracy 0.0310\n",
            "Epoch 3 Batch 150 Loss 6.3467 Accuracy 0.0336\n",
            "Epoch 3 Batch 200 Loss 6.2554 Accuracy 0.0373\n",
            "Epoch 3 Loss 6.2181 Accuracy 0.0384\n",
            "Time taken for 1 epoch: 15.204065799713135 secs\n",
            "\n",
            "Epoch 4 Batch 0 Loss 5.8486 Accuracy 0.0451\n",
            "Epoch 4 Batch 50 Loss 5.5897 Accuracy 0.0572\n",
            "Epoch 4 Batch 100 Loss 5.4568 Accuracy 0.0626\n",
            "Epoch 4 Batch 150 Loss 5.3750 Accuracy 0.0654\n",
            "Epoch 4 Batch 200 Loss 5.2699 Accuracy 0.0684\n",
            "Epoch 4 Loss 5.2413 Accuracy 0.0689\n",
            "Time taken for 1 epoch: 15.06148362159729 secs\n",
            "\n",
            "Epoch 5 Batch 0 Loss 5.0130 Accuracy 0.0657\n",
            "Epoch 5 Batch 50 Loss 4.7602 Accuracy 0.0819\n",
            "Epoch 5 Batch 100 Loss 4.6586 Accuracy 0.0850\n",
            "Epoch 5 Batch 150 Loss 4.6076 Accuracy 0.0858\n",
            "Epoch 5 Batch 200 Loss 4.5612 Accuracy 0.0873\n",
            "Saving checkpoint for epoch 5 at ./checkpoints/train/ckpt-1\n",
            "Epoch 5 Loss 4.5500 Accuracy 0.0876\n",
            "Time taken for 1 epoch: 15.329580545425415 secs\n",
            "\n",
            "Epoch 6 Batch 0 Loss 4.1429 Accuracy 0.0852\n",
            "Epoch 6 Batch 50 Loss 4.1829 Accuracy 0.0987\n",
            "Epoch 6 Batch 100 Loss 4.1735 Accuracy 0.0991\n",
            "Epoch 6 Batch 150 Loss 4.1734 Accuracy 0.0981\n",
            "Epoch 6 Batch 200 Loss 4.1420 Accuracy 0.0999\n",
            "Epoch 6 Loss 4.1320 Accuracy 0.1001\n",
            "Time taken for 1 epoch: 14.963034868240356 secs\n",
            "\n",
            "Epoch 7 Batch 0 Loss 4.3846 Accuracy 0.0830\n",
            "Epoch 7 Batch 50 Loss 3.8786 Accuracy 0.1065\n",
            "Epoch 7 Batch 100 Loss 3.8432 Accuracy 0.1093\n",
            "Epoch 7 Batch 150 Loss 3.8310 Accuracy 0.1094\n",
            "Epoch 7 Batch 200 Loss 3.8030 Accuracy 0.1109\n",
            "Epoch 7 Loss 3.7971 Accuracy 0.1110\n",
            "Time taken for 1 epoch: 15.155653476715088 secs\n",
            "\n",
            "Epoch 8 Batch 0 Loss 3.2210 Accuracy 0.1345\n",
            "Epoch 8 Batch 50 Loss 3.5555 Accuracy 0.1190\n",
            "Epoch 8 Batch 100 Loss 3.5172 Accuracy 0.1191\n",
            "Epoch 8 Batch 150 Loss 3.5118 Accuracy 0.1199\n",
            "Epoch 8 Batch 200 Loss 3.5243 Accuracy 0.1194\n",
            "Epoch 8 Loss 3.5185 Accuracy 0.1200\n",
            "Time taken for 1 epoch: 15.133555889129639 secs\n",
            "\n",
            "Epoch 9 Batch 0 Loss 3.3740 Accuracy 0.1304\n",
            "Epoch 9 Batch 50 Loss 3.2767 Accuracy 0.1283\n",
            "Epoch 9 Batch 100 Loss 3.2673 Accuracy 0.1302\n",
            "Epoch 9 Batch 150 Loss 3.2543 Accuracy 0.1304\n",
            "Epoch 9 Batch 200 Loss 3.2498 Accuracy 0.1301\n",
            "Epoch 9 Loss 3.2463 Accuracy 0.1307\n",
            "Time taken for 1 epoch: 15.182158470153809 secs\n",
            "\n",
            "Epoch 10 Batch 0 Loss 2.6878 Accuracy 0.1542\n",
            "Epoch 10 Batch 50 Loss 2.9486 Accuracy 0.1397\n",
            "Epoch 10 Batch 100 Loss 2.9733 Accuracy 0.1411\n",
            "Epoch 10 Batch 150 Loss 2.9993 Accuracy 0.1402\n",
            "Epoch 10 Batch 200 Loss 2.9974 Accuracy 0.1409\n",
            "Saving checkpoint for epoch 10 at ./checkpoints/train/ckpt-2\n",
            "Epoch 10 Loss 2.9936 Accuracy 0.1412\n",
            "Time taken for 1 epoch: 15.643861770629883 secs\n",
            "\n",
            "Epoch 11 Batch 0 Loss 3.0798 Accuracy 0.1573\n",
            "Epoch 11 Batch 50 Loss 2.6814 Accuracy 0.1552\n",
            "Epoch 11 Batch 100 Loss 2.7270 Accuracy 0.1538\n",
            "Epoch 11 Batch 150 Loss 2.7515 Accuracy 0.1518\n",
            "Epoch 11 Batch 200 Loss 2.7511 Accuracy 0.1527\n",
            "Epoch 11 Loss 2.7582 Accuracy 0.1523\n",
            "Time taken for 1 epoch: 15.200133562088013 secs\n",
            "\n",
            "Epoch 12 Batch 0 Loss 2.4830 Accuracy 0.1463\n",
            "Epoch 12 Batch 50 Loss 2.4407 Accuracy 0.1643\n",
            "Epoch 12 Batch 100 Loss 2.4719 Accuracy 0.1631\n",
            "Epoch 12 Batch 150 Loss 2.5116 Accuracy 0.1605\n",
            "Epoch 12 Batch 200 Loss 2.5353 Accuracy 0.1597\n",
            "Epoch 12 Loss 2.5393 Accuracy 0.1595\n",
            "Time taken for 1 epoch: 15.055858135223389 secs\n",
            "\n",
            "Epoch 13 Batch 0 Loss 2.2881 Accuracy 0.1522\n",
            "Epoch 13 Batch 50 Loss 2.2586 Accuracy 0.1732\n",
            "Epoch 13 Batch 100 Loss 2.3105 Accuracy 0.1687\n",
            "Epoch 13 Batch 150 Loss 2.3322 Accuracy 0.1688\n",
            "Epoch 13 Batch 200 Loss 2.3484 Accuracy 0.1682\n",
            "Epoch 13 Loss 2.3413 Accuracy 0.1683\n",
            "Time taken for 1 epoch: 15.1887788772583 secs\n",
            "\n",
            "Epoch 14 Batch 0 Loss 2.0629 Accuracy 0.1711\n",
            "Epoch 14 Batch 50 Loss 2.1152 Accuracy 0.1796\n",
            "Epoch 14 Batch 100 Loss 2.1292 Accuracy 0.1781\n",
            "Epoch 14 Batch 150 Loss 2.1479 Accuracy 0.1763\n",
            "Epoch 14 Batch 200 Loss 2.1597 Accuracy 0.1755\n",
            "Epoch 14 Loss 2.1720 Accuracy 0.1751\n",
            "Time taken for 1 epoch: 15.23840594291687 secs\n",
            "\n",
            "Epoch 15 Batch 0 Loss 2.0375 Accuracy 0.1793\n",
            "Epoch 15 Batch 50 Loss 1.9421 Accuracy 0.1845\n",
            "Epoch 15 Batch 100 Loss 1.9519 Accuracy 0.1858\n",
            "Epoch 15 Batch 150 Loss 1.9684 Accuracy 0.1849\n",
            "Epoch 15 Batch 200 Loss 2.0020 Accuracy 0.1841\n",
            "Saving checkpoint for epoch 15 at ./checkpoints/train/ckpt-3\n",
            "Epoch 15 Loss 2.0074 Accuracy 0.1835\n",
            "Time taken for 1 epoch: 15.2719087600708 secs\n",
            "\n",
            "Epoch 16 Batch 0 Loss 1.5695 Accuracy 0.1823\n",
            "Epoch 16 Batch 50 Loss 1.7390 Accuracy 0.1930\n",
            "Epoch 16 Batch 100 Loss 1.7655 Accuracy 0.1940\n",
            "Epoch 16 Batch 150 Loss 1.8119 Accuracy 0.1912\n",
            "Epoch 16 Batch 200 Loss 1.8467 Accuracy 0.1896\n",
            "Epoch 16 Loss 1.8544 Accuracy 0.1895\n",
            "Time taken for 1 epoch: 15.063377380371094 secs\n",
            "\n",
            "Epoch 17 Batch 0 Loss 1.4791 Accuracy 0.1883\n",
            "Epoch 17 Batch 50 Loss 1.6052 Accuracy 0.1962\n",
            "Epoch 17 Batch 100 Loss 1.6241 Accuracy 0.1981\n",
            "Epoch 17 Batch 150 Loss 1.6756 Accuracy 0.1963\n",
            "Epoch 17 Batch 200 Loss 1.7195 Accuracy 0.1942\n",
            "Epoch 17 Loss 1.7291 Accuracy 0.1937\n",
            "Time taken for 1 epoch: 14.974653720855713 secs\n",
            "\n",
            "Epoch 18 Batch 0 Loss 1.7515 Accuracy 0.2302\n",
            "Epoch 18 Batch 50 Loss 1.4557 Accuracy 0.2099\n",
            "Epoch 18 Batch 100 Loss 1.5113 Accuracy 0.2059\n",
            "Epoch 18 Batch 150 Loss 1.5613 Accuracy 0.2019\n",
            "Epoch 18 Batch 200 Loss 1.6082 Accuracy 0.2000\n",
            "Epoch 18 Loss 1.6159 Accuracy 0.2000\n",
            "Time taken for 1 epoch: 14.94517731666565 secs\n",
            "\n",
            "Epoch 19 Batch 0 Loss 1.2732 Accuracy 0.1853\n",
            "Epoch 19 Batch 50 Loss 1.3745 Accuracy 0.2073\n",
            "Epoch 19 Batch 100 Loss 1.4083 Accuracy 0.2098\n",
            "Epoch 19 Batch 150 Loss 1.4551 Accuracy 0.2075\n",
            "Epoch 19 Batch 200 Loss 1.4940 Accuracy 0.2055\n",
            "Epoch 19 Loss 1.5053 Accuracy 0.2047\n",
            "Time taken for 1 epoch: 15.142241716384888 secs\n",
            "\n",
            "Epoch 20 Batch 0 Loss 0.9777 Accuracy 0.2444\n",
            "Epoch 20 Batch 50 Loss 1.2507 Accuracy 0.2244\n",
            "Epoch 20 Batch 100 Loss 1.2828 Accuracy 0.2216\n",
            "Epoch 20 Batch 150 Loss 1.3355 Accuracy 0.2179\n",
            "Epoch 20 Batch 200 Loss 1.3664 Accuracy 0.2154\n",
            "Saving checkpoint for epoch 20 at ./checkpoints/train/ckpt-4\n",
            "Epoch 20 Loss 1.3751 Accuracy 0.2144\n",
            "Time taken for 1 epoch: 15.120635271072388 secs\n",
            "\n",
            "Epoch 21 Batch 0 Loss 1.0962 Accuracy 0.2610\n",
            "Epoch 21 Batch 50 Loss 1.1179 Accuracy 0.2331\n",
            "Epoch 21 Batch 100 Loss 1.1592 Accuracy 0.2293\n",
            "Epoch 21 Batch 150 Loss 1.2079 Accuracy 0.2239\n",
            "Epoch 21 Batch 200 Loss 1.2497 Accuracy 0.2218\n",
            "Epoch 21 Loss 1.2555 Accuracy 0.2210\n",
            "Time taken for 1 epoch: 15.163581848144531 secs\n",
            "\n",
            "Epoch 22 Batch 0 Loss 0.9802 Accuracy 0.2621\n",
            "Epoch 22 Batch 50 Loss 0.9931 Accuracy 0.2442\n",
            "Epoch 22 Batch 100 Loss 1.0493 Accuracy 0.2371\n",
            "Epoch 22 Batch 150 Loss 1.0909 Accuracy 0.2317\n",
            "Epoch 22 Batch 200 Loss 1.1243 Accuracy 0.2290\n",
            "Epoch 22 Loss 1.1398 Accuracy 0.2282\n",
            "Time taken for 1 epoch: 15.603339672088623 secs\n",
            "\n",
            "Epoch 23 Batch 0 Loss 0.9965 Accuracy 0.2368\n",
            "Epoch 23 Batch 50 Loss 0.9296 Accuracy 0.2448\n",
            "Epoch 23 Batch 100 Loss 0.9730 Accuracy 0.2396\n",
            "Epoch 23 Batch 150 Loss 1.0042 Accuracy 0.2391\n",
            "Epoch 23 Batch 200 Loss 1.0381 Accuracy 0.2365\n",
            "Epoch 23 Loss 1.0487 Accuracy 0.2357\n",
            "Time taken for 1 epoch: 15.077216386795044 secs\n",
            "\n",
            "Epoch 24 Batch 0 Loss 0.8681 Accuracy 0.2469\n",
            "Epoch 24 Batch 50 Loss 0.8500 Accuracy 0.2530\n",
            "Epoch 24 Batch 100 Loss 0.8810 Accuracy 0.2492\n",
            "Epoch 24 Batch 150 Loss 0.9274 Accuracy 0.2436\n",
            "Epoch 24 Batch 200 Loss 0.9591 Accuracy 0.2431\n",
            "Epoch 24 Loss 0.9694 Accuracy 0.2419\n",
            "Time taken for 1 epoch: 14.98857593536377 secs\n",
            "\n",
            "Epoch 25 Batch 0 Loss 0.6845 Accuracy 0.2947\n",
            "Epoch 25 Batch 50 Loss 0.7870 Accuracy 0.2597\n",
            "Epoch 25 Batch 100 Loss 0.8244 Accuracy 0.2537\n",
            "Epoch 25 Batch 150 Loss 0.8570 Accuracy 0.2489\n",
            "Epoch 25 Batch 200 Loss 0.8852 Accuracy 0.2474\n",
            "Saving checkpoint for epoch 25 at ./checkpoints/train/ckpt-5\n",
            "Epoch 25 Loss 0.8933 Accuracy 0.2469\n",
            "Time taken for 1 epoch: 15.170691967010498 secs\n",
            "\n",
            "Epoch 26 Batch 0 Loss 0.6153 Accuracy 0.2452\n",
            "Epoch 26 Batch 50 Loss 0.7425 Accuracy 0.2590\n",
            "Epoch 26 Batch 100 Loss 0.7715 Accuracy 0.2535\n",
            "Epoch 26 Batch 150 Loss 0.8057 Accuracy 0.2518\n",
            "Epoch 26 Batch 200 Loss 0.8274 Accuracy 0.2496\n",
            "Epoch 26 Loss 0.8375 Accuracy 0.2492\n",
            "Time taken for 1 epoch: 14.773926496505737 secs\n",
            "\n",
            "Epoch 27 Batch 0 Loss 0.7529 Accuracy 0.2411\n",
            "Epoch 27 Batch 50 Loss 0.6758 Accuracy 0.2662\n",
            "Epoch 27 Batch 100 Loss 0.7078 Accuracy 0.2619\n",
            "Epoch 27 Batch 150 Loss 0.7399 Accuracy 0.2593\n",
            "Epoch 27 Batch 200 Loss 0.7649 Accuracy 0.2567\n",
            "Epoch 27 Loss 0.7754 Accuracy 0.2562\n",
            "Time taken for 1 epoch: 14.951397180557251 secs\n",
            "\n",
            "Epoch 28 Batch 0 Loss 0.6948 Accuracy 0.2769\n",
            "Epoch 28 Batch 50 Loss 0.6351 Accuracy 0.2739\n",
            "Epoch 28 Batch 100 Loss 0.6699 Accuracy 0.2686\n",
            "Epoch 28 Batch 150 Loss 0.6911 Accuracy 0.2648\n",
            "Epoch 28 Batch 200 Loss 0.7132 Accuracy 0.2607\n",
            "Epoch 28 Loss 0.7231 Accuracy 0.2595\n",
            "Time taken for 1 epoch: 14.849536657333374 secs\n",
            "\n",
            "Epoch 29 Batch 0 Loss 0.4810 Accuracy 0.3009\n",
            "Epoch 29 Batch 50 Loss 0.6079 Accuracy 0.2667\n",
            "Epoch 29 Batch 100 Loss 0.6259 Accuracy 0.2660\n",
            "Epoch 29 Batch 150 Loss 0.6558 Accuracy 0.2635\n",
            "Epoch 29 Batch 200 Loss 0.6783 Accuracy 0.2622\n",
            "Epoch 29 Loss 0.6826 Accuracy 0.2615\n",
            "Time taken for 1 epoch: 14.876307725906372 secs\n",
            "\n",
            "Epoch 30 Batch 0 Loss 0.5379 Accuracy 0.2896\n",
            "Epoch 30 Batch 50 Loss 0.5638 Accuracy 0.2682\n",
            "Epoch 30 Batch 100 Loss 0.5799 Accuracy 0.2695\n",
            "Epoch 30 Batch 150 Loss 0.6089 Accuracy 0.2666\n",
            "Epoch 30 Batch 200 Loss 0.6304 Accuracy 0.2643\n",
            "Saving checkpoint for epoch 30 at ./checkpoints/train/ckpt-6\n",
            "Epoch 30 Loss 0.6400 Accuracy 0.2646\n",
            "Time taken for 1 epoch: 15.400777101516724 secs\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QfcsSWswSdGV"
      },
      "source": [
        "## Evaluate"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5buvMlnvyrFm"
      },
      "source": [
        "def evaluate(inp_sentence):\n",
        "  start_token = [tokenizer_pt.vocab_size]\n",
        "  end_token = [tokenizer_pt.vocab_size + 1]\n",
        "  \n",
        "  # inp sentence is portuguese, hence adding the start and end token\n",
        "  inp_sentence = start_token + tokenizer_pt.encode(inp_sentence) + end_token\n",
        "  encoder_input = tf.expand_dims(inp_sentence, 0)\n",
        "  \n",
        "  # as the target is english, the first word to the transformer should be the\n",
        "  # english start token.\n",
        "  decoder_input = [tokenizer_en.vocab_size]\n",
        "  output = tf.expand_dims(decoder_input, 0)\n",
        "    \n",
        "  for i in range(MAX_LENGTH):\n",
        "    enc_padding_mask, combined_mask, dec_padding_mask = create_masks(\n",
        "        encoder_input, output)\n",
        "  \n",
        "    # predictions.shape == (batch_size, seq_len, vocab_size)\n",
        "    predictions, attention_weights = transformer(encoder_input, \n",
        "                                                 output,\n",
        "                                                 False,\n",
        "                                                 enc_padding_mask,\n",
        "                                                 combined_mask,\n",
        "                                                 dec_padding_mask)\n",
        "    \n",
        "    # select the last word from the seq_len dimension\n",
        "    predictions = predictions[: ,-1:, :]  # (batch_size, 1, vocab_size)\n",
        "\n",
        "    predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n",
        "    \n",
        "    # return the result if the predicted_id is equal to the end token\n",
        "    if predicted_id == tokenizer_en.vocab_size+1:\n",
        "      return tf.squeeze(output, axis=0), attention_weights\n",
        "    \n",
        "    # concatentate the predicted_id to the output which is given to the decoder\n",
        "    # as its input.\n",
        "    output = tf.concat([output, predicted_id], axis=-1)\n",
        "\n",
        "  return tf.squeeze(output, axis=0), attention_weights"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CN-BV43FMBej"
      },
      "source": [
        "def plot_attention_weights(attention, sentence, result, layer):\n",
        "  fig = plt.figure(figsize=(16, 8))\n",
        "  \n",
        "  sentence = tokenizer_pt.encode(sentence)\n",
        "  \n",
        "  attention = tf.squeeze(attention[layer], axis=0)\n",
        "  \n",
        "  for head in range(attention.shape[0]):\n",
        "    ax = fig.add_subplot(2, 4, head+1)\n",
        "    \n",
        "    # plot the attention weights\n",
        "    ax.matshow(attention[head][:-1, :], cmap='viridis')\n",
        "\n",
        "    fontdict = {'fontsize': 10}\n",
        "    \n",
        "    ax.set_xticks(range(len(sentence)+2))\n",
        "    ax.set_yticks(range(len(result)))\n",
        "    \n",
        "    ax.set_ylim(len(result)-1.5, -0.5)\n",
        "        \n",
        "    ax.set_xticklabels(\n",
        "        ['<start>']+[tokenizer_pt.decode([i]) for i in sentence]+['<end>'], \n",
        "        fontdict=fontdict, rotation=90)\n",
        "    \n",
        "    ax.set_yticklabels([tokenizer_en.decode([i]) for i in result \n",
        "                        if i < tokenizer_en.vocab_size], \n",
        "                       fontdict=fontdict)\n",
        "    \n",
        "    ax.set_xlabel('Head {}'.format(head+1))\n",
        "  \n",
        "  plt.tight_layout()\n",
        "  plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lU2_yG_vBGza"
      },
      "source": [
        "def translate(sentence, plot=''):\n",
        "  result, attention_weights = evaluate(sentence)\n",
        "  \n",
        "  predicted_sentence = tokenizer_en.decode([i for i in result \n",
        "                                            if i < tokenizer_en.vocab_size])  \n",
        "\n",
        "  print('Input: {}'.format(sentence))\n",
        "  print('Predicted translation: {}'.format(predicted_sentence))\n",
        "  \n",
        "  if plot:\n",
        "    plot_attention_weights(attention_weights, sentence, result, plot)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YsxrAlvFG8SZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "736b0812-259d-4bfe-fa32-ecf99b7fca9f"
      },
      "source": [
        "translate(\"paciente con disuria y fiebre\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: paciente con disuria y fiebre\n",
            "Predicted translation: ciprofloxacina 500 mg cada 12 hs por 5 dias \n",
            "mucho liquido en caso de dolor de cabeza\n",
            "dieta \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_1MxkSZvz0jX"
      },
      "source": [
        "You can pass different layers and attention blocks of the decoder to the `plot` parameter."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C95x16Nlemq2"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}