LSTM.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/InhuKim/AI_proj_Team2/blob/main/LSTM.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "77911691",
      "metadata": {
        "id": "77911691"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import os\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data import TensorDataset\n",
        "from torch.autograd import Variable\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from tqdm.auto import tqdm"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "daaef434",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "daaef434",
        "outputId": "6ae37429-dcfc-4424-cd0e-e9a25648d619"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using cuda device\n"
          ]
        }
      ],
      "source": [
        "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
        "print(f\"Using {device} device\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CLeT7S2gjhZv",
        "outputId": "6c110065-a24b-4dc2-8088-39ebd2b5ec90"
      },
      "id": "CLeT7S2gjhZv",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4b54d1a5",
      "metadata": {
        "id": "4b54d1a5"
      },
      "outputs": [],
      "source": [
        "time_series = pd.DataFrame()\n",
        "target_label = pd.DataFrame()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "81cf9c98",
      "metadata": {
        "id": "81cf9c98"
      },
      "outputs": [],
      "source": [
        "path = '/content/drive/MyDrive/Colab_Notebooks/data/meta_data/train'\n",
        "count = 0\n",
        "row = pd.DataFrame()\n",
        "\n",
        "for i in range(75):\n",
        "    if i + 1 < 10:\n",
        "        i_str = '0' + str(i +1)\n",
        "    elif i + 1 > 10:\n",
        "        i_str = str(i +1)\n",
        "    elif i + 1 == 10:\n",
        "        i_str = str(i + 1)\n",
        "\n",
        "    file_list = os.listdir(path + '/CASE' + i_str)\n",
        "    file_list_py = [file for file in file_list if file.endswith('.csv')]\n",
        "\n",
        "    for l in file_list_py:\n",
        "        data = pd.read_csv(path + '/CASE' + i_str + '/' + l)\n",
        "        time_series = pd.concat([time_series, data])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "166a74ae",
      "metadata": {
        "id": "166a74ae"
      },
      "outputs": [],
      "source": [
        "path = '/content/drive/MyDrive/Colab_Notebooks/data/meta_data/target'\n",
        "count2 = 0\n",
        "\n",
        "for j in range(75):\n",
        "    if j + 1 < 10:\n",
        "        j_str = '0' + str(j +1)\n",
        "    elif j + 1 > 10:\n",
        "        j_str = str(j +1)\n",
        "    elif j + 1 == 10:\n",
        "        j_str = str(j + 1)\n",
        "\n",
        "    file_list = os.listdir(path + '/CASE' + j_str)\n",
        "    file_list_py = [file for file in file_list if file.endswith('.csv')]\n",
        "\n",
        "    for k in file_list_py:\n",
        "        target = pd.read_csv(path + '/CASE' + j_str + '/' + k)\n",
        "        target_label = pd.concat([target_label, target])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6812fff8",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6812fff8",
        "outputId": "a0e41c2c-eb4b-40a0-9ac0-a236a8c46550"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1284\n",
            "1292.0\n"
          ]
        }
      ],
      "source": [
        "print(len(target_label))\n",
        "print(len(time_series) / 1440)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "2262f94c",
      "metadata": {
        "id": "2262f94c"
      },
      "outputs": [],
      "source": [
        "# time_series = time_series[['내부온도관측치', '내부습도관측치', 'CO2관측치', '최근분무량','냉방온도', '난방온도']]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a70eefac",
      "metadata": {
        "id": "a70eefac"
      },
      "outputs": [],
      "source": [
        "target_dataset = target_label.to_numpy()\n",
        "target_dataset = target_dataset[:, 1]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ecd9c9f0",
      "metadata": {
        "id": "ecd9c9f0"
      },
      "outputs": [],
      "source": [
        "time_dataset = time_series.to_numpy()\n",
        "time_dataset = time_dataset[:, 1:]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "time_dataset.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BPxmfQmNezZZ",
        "outputId": "642e0c36-ddf6-45a9-e027-d45e47b57e4e"
      },
      "id": "BPxmfQmNezZZ",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1860480, 18)"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "7ccaf8bb",
      "metadata": {
        "id": "7ccaf8bb"
      },
      "outputs": [],
      "source": [
        "def sliding_windows(data, target):\n",
        "    x = []\n",
        "    y = []\n",
        "\n",
        "    for i in range(len(target)):\n",
        "        _x = data[(i * 1440): ((i + 1) * 1440)]\n",
        "        _y = target[i]\n",
        "        x.append(_x)\n",
        "        y.append(_y)\n",
        "\n",
        "    return np.array(x),np.array(y)\n",
        "\n",
        "sc_df = MinMaxScaler()\n",
        "# sc_t = MinMaxScaler()\n",
        "training_data = sc_df.fit_transform(time_dataset)\n",
        "# target_data = sc_t.fit_transform(target_dataset.reshape(-1, 1))\n",
        "\n",
        "\n",
        "x, y = sliding_windows(training_data, target_dataset)\n",
        "\n",
        "train_size = int(len(y) * 0.7)\n",
        "test_size = len(y) - train_size\n",
        "\n",
        "dataX = Variable(torch.Tensor(np.array(x)))\n",
        "dataY = Variable(torch.Tensor(np.array(y)))\n",
        "\n",
        "trainX = Variable(torch.Tensor(np.array(x))).to(device)\n",
        "trainY = Variable(torch.Tensor(np.array(y))).to(device)\n",
        "\n",
        "testX = Variable(torch.Tensor(np.array(x[1020:1275]))).to(device)\n",
        "testY = Variable(torch.Tensor(np.array(y[1020:1275]))).to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3e41e0e9",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3e41e0e9",
        "outputId": "a11a1603-bee4-40db-da11-9a9a0ea5baff"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([1284, 1440, 18])\n",
            "torch.Size([1284])\n",
            "torch.Size([255, 1440, 18])\n"
          ]
        }
      ],
      "source": [
        "print(dataX.shape)\n",
        "print(trainY.shape)\n",
        "print(testX.shape)\n",
        "\n",
        "train_dataset = TensorDataset(trainX, trainY)\n",
        "test_dataset = TensorDataset(testX, testY)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d80727ae",
      "metadata": {
        "id": "d80727ae"
      },
      "outputs": [],
      "source": [
        "train_dataloader = DataLoader(train_dataset, batch_size=20, shuffle=True)\n",
        "test_dataloader = DataLoader(test_dataset, batch_size=5, shuffle=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "c2c206b2",
      "metadata": {
        "id": "c2c206b2"
      },
      "source": [
        "# 일반 LSTM"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "25ab09dd",
      "metadata": {
        "id": "25ab09dd"
      },
      "outputs": [],
      "source": [
        "class LSTM(nn.Module):\n",
        "\n",
        "    def __init__(self, input_size, hidden_size, num_layers, bidirectional):\n",
        "        super(LSTM, self).__init__()\n",
        "        \n",
        "        self.num_layers = num_layers\n",
        "        self.input_size = input_size\n",
        "        self.hidden_size = hidden_size\n",
        "        self.bidirectional = bidirectional\n",
        "        \n",
        "        self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size,\n",
        "                            num_layers=num_layers, batch_first=True, bidirectional=bidirectional).to(device)\n",
        "        \n",
        "        self.fc = nn.Linear(hidden_size *2, 256).to(device)\n",
        "        self.fc2 = nn.Linear(256, 1).to(device)\n",
        "        self.relu = nn.ReLU().to(device)\n",
        "\n",
        "    def forward(self, x):\n",
        "        h_0 = Variable(torch.zeros(\n",
        "            self.num_layers *2, x.size(0), self.hidden_size)).to(device)\n",
        "        \n",
        "        c_0 = Variable(torch.zeros(\n",
        "            self.num_layers *2, x.size(0), self.hidden_size)).to(device)\n",
        "        \n",
        "        # Propagate input through LSTM\n",
        "        ula, (h_out, _) = self.lstm(x, (h_0, c_0))\n",
        "                \n",
        "        out = self.fc(h_out.view(-1, self.hidden_size*2))\n",
        "        out = self.fc2(out)\n",
        "        \n",
        "        return out"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class RNN(nn.Module):\n",
        "    \n",
        "    def __init__(self, input_size, hidden_size, num_layers, bidirectional):\n",
        "        super(RNN, self).__init__()\n",
        "        \n",
        "        self.RNN = nn.RNN(input_size = input_size, hidden_size=hidden_size, num_layers=num_layers, bidirectional=bidirectional)\n",
        "        self.fc = nn.Linear(hidden_size *2, 1)\n",
        "        self.fc2 = nn.Linear(1440, 1)\n",
        "        \n",
        "        \n",
        "    def forward(self, x):\n",
        "        x, _ = self.RNN(x)\n",
        "        x = self.fc(x)\n",
        "        x = torch.flatten(x, start_dim=1)\n",
        "        out = self.fc2(x)\n",
        "        \n",
        "        return out"
      ],
      "metadata": {
        "id": "EWaiWunEt5z1"
      },
      "id": "EWaiWunEt5z1",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "a = torch.tensor(np.arange(8*1440*18).reshape(8, 1440, 18), dtype=torch.float32).to(device)\n",
        "print(a.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5yN1gzcWu497",
        "outputId": "25b6acce-5f5d-4662-9ec4-15d708447677"
      },
      "id": "5yN1gzcWu497",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([8, 1440, 18])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model = LSTM(18, 512, 1, True)\n",
        "out = model(a)\n",
        "print(out.squeeze(1).shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J2gv3NftuyOi",
        "outputId": "1c0ac90c-b32d-4ead-e9ef-a580e6bba177"
      },
      "id": "J2gv3NftuyOi",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([8])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for i in train_dataloader:\n",
        "    x, y = i\n",
        "    print(x.shape)\n",
        "    print(y.shape)\n",
        "    out = model(x)\n",
        "    out2 = torch.squeeze(out)\n",
        "    print(out2.shape)\n",
        "    print(y.shape)\n",
        "    break"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0quHcnHrvuyA",
        "outputId": "c4a0e61c-a606-4e3e-b240-febda58d8221"
      },
      "id": "0quHcnHrvuyA",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([20, 1440, 18])\n",
            "torch.Size([20])\n",
            "torch.Size([20])\n",
            "torch.Size([20])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def validation(model, vali_loader, criterion, device):\n",
        "    model.eval()\n",
        "    vali_loss = []\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for idx, sample in enumerate(vali_loader):\n",
        "            x_vali, y_vali = sample\n",
        "\n",
        "            out = model(x_vali)\n",
        "            loss = criterion(out, y_vali)\n",
        "            vali_loss.append(loss.item())\n",
        "\n",
        "    vali_mae_loss = np.mean(vali_loss)\n",
        "    return vali_mae_loss"
      ],
      "metadata": {
        "id": "UBUBjU5D6wxv"
      },
      "id": "UBUBjU5D6wxv",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9648f012",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9648f012",
        "outputId": "bd1378fc-3593-4eb7-a8b6-f954d3471abb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torch/nn/modules/loss.py:96: UserWarning: Using a target size (torch.Size([5])) that is different to the input size (torch.Size([5, 1])). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.\n",
            "  return F.l1_loss(input, target, reduction=self.reduction)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch [0] Train MAE : [81.42246] Validation MAE : [85.77097]\n",
            "\n",
            "model saved.\n",
            "Epoch [1] Train MAE : [80.61663] Validation MAE : [85.42286]\n",
            "\n",
            "model saved.\n",
            "Epoch [2] Train MAE : [80.36885] Validation MAE : [85.02014]\n",
            "\n",
            "model saved.\n",
            "Epoch [3] Train MAE : [79.94500] Validation MAE : [84.53270]\n",
            "\n",
            "model saved.\n",
            "Epoch [4] Train MAE : [79.90925] Validation MAE : [83.93952]\n",
            "\n",
            "model saved.\n",
            "Epoch [5] Train MAE : [78.42036] Validation MAE : [83.15953]\n",
            "\n",
            "model saved.\n",
            "Epoch [6] Train MAE : [78.00048] Validation MAE : [82.12045]\n",
            "\n",
            "model saved.\n",
            "Epoch [7] Train MAE : [77.46098] Validation MAE : [80.68768]\n",
            "\n",
            "model saved.\n",
            "Epoch [8] Train MAE : [76.14638] Validation MAE : [78.95782]\n",
            "\n",
            "model saved.\n",
            "Epoch [9] Train MAE : [75.33761] Validation MAE : [76.93920]\n",
            "\n",
            "model saved.\n",
            "Epoch [10] Train MAE : [72.46159] Validation MAE : [75.59797]\n",
            "\n",
            "model saved.\n",
            "Epoch [11] Train MAE : [71.10032] Validation MAE : [75.27508]\n",
            "\n",
            "model saved.\n",
            "Epoch [12] Train MAE : [72.52404] Validation MAE : [75.22346]\n",
            "\n",
            "model saved.\n",
            "Epoch [13] Train MAE : [71.05122] Validation MAE : [75.31047]\n",
            "\n",
            "Epoch [14] Train MAE : [72.01637] Validation MAE : [75.08566]\n",
            "\n",
            "model saved.\n",
            "Epoch [15] Train MAE : [71.50325] Validation MAE : [75.13663]\n",
            "\n",
            "Epoch [16] Train MAE : [71.23852] Validation MAE : [75.17185]\n",
            "\n",
            "Epoch [17] Train MAE : [71.39598] Validation MAE : [75.18590]\n",
            "\n",
            "Epoch [18] Train MAE : [72.29671] Validation MAE : [75.09385]\n",
            "\n",
            "Epoch [19] Train MAE : [71.20316] Validation MAE : [75.03215]\n",
            "\n",
            "model saved.\n",
            "Epoch [20] Train MAE : [71.04268] Validation MAE : [75.17725]\n",
            "\n",
            "Epoch [21] Train MAE : [71.97992] Validation MAE : [75.05888]\n",
            "\n",
            "Epoch [22] Train MAE : [71.23452] Validation MAE : [75.09006]\n",
            "\n",
            "Epoch [23] Train MAE : [72.24784] Validation MAE : [75.04657]\n",
            "\n",
            "Epoch [24] Train MAE : [70.97485] Validation MAE : [75.04767]\n",
            "\n",
            "Epoch [25] Train MAE : [71.42384] Validation MAE : [75.09709]\n",
            "\n",
            "Epoch [26] Train MAE : [71.25604] Validation MAE : [75.07873]\n",
            "\n",
            "Epoch [27] Train MAE : [71.08742] Validation MAE : [75.16806]\n",
            "\n",
            "Epoch [28] Train MAE : [71.69282] Validation MAE : [75.08829]\n",
            "\n",
            "Epoch [29] Train MAE : [72.67697] Validation MAE : [74.95391]\n",
            "\n",
            "model saved.\n",
            "Epoch [30] Train MAE : [73.34902] Validation MAE : [75.04091]\n",
            "\n",
            "Epoch [31] Train MAE : [71.93681] Validation MAE : [74.98900]\n",
            "\n",
            "Epoch [32] Train MAE : [70.86532] Validation MAE : [75.06724]\n",
            "\n",
            "Epoch [33] Train MAE : [71.81530] Validation MAE : [74.92284]\n",
            "\n",
            "model saved.\n",
            "Epoch [34] Train MAE : [72.14021] Validation MAE : [75.05297]\n",
            "\n",
            "Epoch [35] Train MAE : [71.35157] Validation MAE : [75.15288]\n",
            "\n",
            "Epoch [36] Train MAE : [71.07000] Validation MAE : [74.98715]\n",
            "\n",
            "Epoch [37] Train MAE : [72.01708] Validation MAE : [74.94766]\n",
            "\n",
            "Epoch [38] Train MAE : [71.56673] Validation MAE : [75.00493]\n",
            "\n",
            "Epoch [39] Train MAE : [71.27945] Validation MAE : [75.05645]\n",
            "\n",
            "Epoch [40] Train MAE : [70.97995] Validation MAE : [74.95428]\n",
            "\n",
            "Epoch [41] Train MAE : [71.05990] Validation MAE : [75.03561]\n",
            "\n",
            "Epoch [42] Train MAE : [71.01763] Validation MAE : [75.03551]\n",
            "\n",
            "Epoch [43] Train MAE : [70.91758] Validation MAE : [75.03450]\n",
            "\n",
            "Epoch [44] Train MAE : [72.28494] Validation MAE : [74.82314]\n",
            "\n",
            "model saved.\n",
            "Epoch [45] Train MAE : [71.09279] Validation MAE : [74.98296]\n",
            "\n",
            "Epoch [46] Train MAE : [70.88101] Validation MAE : [74.97059]\n",
            "\n",
            "Epoch [47] Train MAE : [71.01489] Validation MAE : [75.09479]\n",
            "\n",
            "Epoch [48] Train MAE : [70.95032] Validation MAE : [75.03655]\n",
            "\n",
            "Epoch [49] Train MAE : [71.72843] Validation MAE : [74.85683]\n",
            "\n"
          ]
        }
      ],
      "source": [
        "num_epochs = 50\n",
        "learning_rate = 0.003\n",
        "\n",
        "input_size = 18\n",
        "hidden_size = 512\n",
        "num_layers = 1\n",
        "\n",
        "best_mae = 9999\n",
        "\n",
        "model = LSTM(input_size, hidden_size, num_layers, bidirectional=True).to(device)\n",
        "\n",
        "criterion = torch.nn.L1Loss().to(device)    # mean-squared error for regression\n",
        "#optimizer = torch.optim.Adam(lstm.parameters(), lr=learning_rate)\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n",
        "\n",
        "train_loss_plot = []\n",
        "vali_loss_plot = []\n",
        "\n",
        "# Train the model\n",
        "for epoch in range(num_epochs):\n",
        "    model.train()\n",
        "    train_loss = []\n",
        "    for batch_idx, samples in enumerate(train_dataloader):\n",
        "        x_train, y_train = samples\n",
        "        \n",
        "        optimizer.zero_grad()\n",
        "        outputs = model(x_train)\n",
        "        out = torch.squeeze(outputs)\n",
        "    \n",
        "        # obtain the loss function\n",
        "        loss = criterion(out, y_train)\n",
        "    \n",
        "        loss.backward()\n",
        "    \n",
        "        optimizer.step()\n",
        "\n",
        "        train_loss.append(loss.item())\n",
        "\n",
        "        # validation\n",
        "    vali_mae = validation(model, test_dataloader, criterion, device)\n",
        "    vali_loss_plot.append(vali_mae)\n",
        "\n",
        "    print(f'Epoch [{epoch}] Train MAE : [{np.mean(train_loss):.5f}] Validation MAE : [{vali_mae:.5f}]\\n')\n",
        "    vali_loss_plot.append(vali_mae)\n",
        "    train_loss_plot.append(np.mean(train_loss))\n",
        "\n",
        "    if best_mae > vali_mae:\n",
        "        best_mae = vali_mae\n",
        "        torch.save(model.state_dict(), '/content/drive/MyDrive/Colab_Notebooks/competition/meta_LSTM_full_feature.pth')\n",
        "        print('model saved.')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "valid_loss = pd.DataFrame(vali_loss_plot, index=None)\n",
        "valid_loss = valid_loss.drop_duplicates()"
      ],
      "metadata": {
        "id": "-hwTZnMx_n50"
      },
      "id": "-hwTZnMx_n50",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "valid_loss_plot = valid_loss[0].to_list()\n",
        "valid_loss_plot"
      ],
      "metadata": {
        "id": "4C0paCojWwzc"
      },
      "id": "4C0paCojWwzc",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "Eg02pMjal1Fs"
      },
      "id": "Eg02pMjal1Fs",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "figure = plt.figure(figsize=(20,5))\n",
        "ax1 = figure.add_subplot(1,1,1)\n",
        "ax1.plot(train_loss_plot, label='Traing Loss', color='blue')\n",
        "ax1.plot(valid_loss_plot, label='Validation Loss', color='orange')\n",
        "ax1.legend(loc='upper right')\n",
        "ax1.set_title('LSTM Loss')\n",
        "\n",
        "plt.savefig('/content/drive/MyDrive/Colab_Notebooks/competition/LSTM_loss.png')\n",
        "plt.show"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 354
        },
        "id": "1Dum3mxEz9cm",
        "outputId": "fa496813-087a-47e5-93b6-5a7cbfd7735c"
      },
      "id": "1Dum3mxEz9cm",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<function matplotlib.pyplot.show>"
            ]
          },
          "metadata": {},
          "execution_count": 89
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1440x360 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABH4AAAE/CAYAAAAwiQR3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3xV9f3H8dc3gwQCIiKigoo4UBBBCA5UhLpFQaVWqVbAKmrrntVWbeuorVqt2tbaZYeKC/25tcWFo8oQFBXcIENlyF4Z398f5wZCDJBAkpOE1/PxOI+z7rn3c29uLtx3viPEGJEkSZIkSVLjk5V2AZIkSZIkSaodBj+SJEmSJEmNlMGPJEmSJElSI2XwI0mSJEmS1EgZ/EiSJEmSJDVSBj+SJEmSJEmNlMGPJEmSJElSI2XwI0mS6rUQwuchhEPWcu7KEMJnIYTFIYTpIYQHMsffyxxbHEIoCSEsL7d/ZQhhaAghhhBurXB/AzPH71nL4/UNIUyv8ScpSZJUSwx+JElSgxRCGAL8ADgkxtgcKARGAcQYu8QYm2eOjwbOKduPMd6QuYtPgO+FEHLK3e0Q4MO6exaSJEm1y+BHkiQ1VL2A52KMnwDEGL+MMd5djeu/BN4FDgcIIWwB9AYe35BiQgi7hxBeCiHMz7Q4GlDu3FEhhPdDCItCCDNCCJdkjm8ZQngyc828EMLoEIL/P5MkSTXG/1hIkqSG6n/AqSGES0MIhSGE7A24j38Cp2a2TwL+D1hR3TsJIeQCTwDPA1sB5wL3hhA6ZW7yV+DMGGMLYA/ghczxi4HpQBugLXAlEDfgeUiSJFXK4EeSJDVIMcZ/kwQshwMvA1+HEC6v5t08CvQNIbQkCYD+uYHl7As0B26MMa6MMb4APAkMzpwvAjqHEDaLMX4TYxxf7vg2wA4xxqIY4+gYo8GPJEmqMQY/kiSpwYox3htjPATYHDgLuDaEcHg1rl8GPAX8DGgdY3xtA0vZFvgixlha7thUoF1mexBwFDA1hPByCGG/zPGbgI+B50MIn4YQfrKBjy9JklQpgx9JktTgZVrLPAS8Q9KVqjr+SdLl6t8bUcJMYLsK4/NsD8zI1DcmxjiQpBvYY8CDmeOLYowXxxg7AgOAi0IIB29EHZIkSWsw+JEkSQ1Bbgghv9ySk5mSvX8IoUUIISuEcCTQBXizmvf9MnAocEdVL6hQSz7wFrAUuCyEkBtC6AscA4wIITQJIZwcQmgZYywCFgKlmfs5OoSwcwghAAuAkrJzkiRJNSFn/TeRJElK3dMV9q8HxpMMhvxvIJuka9XZMcZXq3PHmTF1RlXjknbAsgrHdiEJev4AXEHS0ufUGOPkEEITkmnn78wMQD0FOLncdXeSDO78DfCHGOOL1alfkiRpXYLjB0qSJEmSJDVOdvWSJEmSJElqpAx+JEmSJEmSGimDH0mSJEmSpEbK4EeSJEmSJKmRMviRJEmSJElqpOp0Ovctt9wydujQoS4fUpIkSZIkqVEbN27cnBhjm8rO1Wnw06FDB8aOHVuXDylJkiRJktSohRCmru2cXb0kSZIkSZIaKYMfSZIkSZKkRsrgR5IkSZIkqZGq0hg/IYQLgdOBCLwLDANWANcBJwAlwB9jjLfXUp2SJEmSJKmGFBUVMX36dJYvX552KaqG/Px82rdvT25ubpWvWW/wE0JoB5wHdI4xLgshPAicBARgO2C3GGNpCGGrDaxbkiRJkiTVoenTp9OiRQs6dOhACCHtclQFMUbmzp3L9OnT2XHHHat8XVW7euUATUMIOUAzYCZwNvDLGGNppoCvq1mzJEmSJElKwfLly2ndurWhTwMSQqB169bVbqW13uAnxjgDuBmYBswCFsQYnwd2Ak4MIYwNITwTQthlA+qWJEmSJEkpMPRpeDbkZ7be4CeE0AoYCOwIbAsUhBBOAfKA5THGQuDPwN/Wcv3wTDg0dvbs2dUuUJIkSZIkNS5z586le/fudO/ena233pp27dqt2l+5cuU6rx07diznnXdejdTx85//nJtvvrlG7qu+qsrgzocAn8UYZwOEEEYCvYHpwMjMbR4F/l7ZxTHGu4G7AQoLC+PGFixJkiRJkhq21q1bM2HCBCAJX5o3b84ll1yy6nxxcTE5OZVHFoWFhRQWFtZJnY1BVcb4mQbsG0JoFpI2RQcDHwCPAf0ytzkI+LB2SqxnYoQpt8O8cZAMbyRJkiRJkjbS0KFDOeuss9hnn3247LLLeOutt9hvv/3Ya6+96N27N1OmTAHgpZde4uijjwaS0Oi0006jb9++dOzYkdtvXz3Z+LXXXkunTp044IADGDx4cJVb9sQYufTSS9ljjz3o2rUrDzzwAACzZs2iT58+dO/enT322IPRo0dTUlLC0KFDV9321ltvreFXZeOtt8VPjPHNEMLDwHigGHibpAVPU+DezFTvi0mme2/8lk6Dcecn23ltYJvDYJvDYevDoGnbdGuTJEmSJKkBmz59Oq+//jrZ2dksXLiQ0aNHk5OTw3//+1+uvPJKHnnkkW9dM3nyZF588UUWLVpEp06dOPvss5kwYQKPPPIIEydOpKioiB49etCzZ88q1TBy5EgmTJjAxIkTmTNnDr169aJPnz7cd999HH744fz0pz+lpKSEpUuXMmHCBGbMmMGkSZMAmD9/fo2+HjWhKl29iDFeA1xT4fAKoH+NV1TfFewAx30JXz4Ps56DWc/D5/cm51rtlYRA2xwBW+4H2U3SrVWSJEmSpPW44ALI9LqqMd27w223Vf+6E044gezsbAAWLFjAkCFD+OijjwghUFRUVOk1/fv3Jy8vj7y8PLbaaiu++uorXnvtNQYOHEh+fj75+fkcc8wxVa7h1VdfZfDgwWRnZ9O2bVsOOuggxowZQ69evTjttNMoKiri2GOPpXv37nTs2JFPP/2Uc889l/79+3PYYYdV/0nXsqpO567ymraFHX8Avf8Nx38JR4yFbtdDbgv44GYY1RceaQ0vD4SP/giLP027YkmSJEmS6r2CgoJV21dddRX9+vVj0qRJPPHEE2udxjwvL2/VdnZ2NsXFxbVSW58+fXjllVdo164dQ4cO5Z///CetWrVi4sSJ9O3bl7vuuovTT69/naGq1OJH6xCyYIueydLlSihaCF++kGkN9CzMeDy5XfOdYdsjkhZBW/WF3Oapli1JkiRJEmxYy5y6sGDBAtq1awfAPffcU61r999/f84880yuuOIKiouLefLJJxk+fHiVrj3wwAP505/+xJAhQ5g3bx6vvPIKN910E1OnTqV9+/acccYZrFixgvHjx3PUUUfRpEkTBg0aRKdOnTjllFOq+zRrncFPTcvdDLY7NllihEUfrQ6BPvkbfHgnZDWBNges7ha2eVcIIe3KJUmSJEmqNy677DKGDBnCddddR//+1RtpplevXgwYMIA999yTtm3b0rVrV1q2bFnpba+77jpuK5d+ffHFF7zxxht069aNEAK/+c1v2HrrrfnHP/7BTTfdRG5uLs2bN+ef//wnM2bMYNiwYZSWJpM//epXv9rwJ1xLQox1N8N6YWFhHDt2bJ09Xr1Tshxmv5oJgp6D+e8mx5tuk4RA7Y+DrQ+FnKbp1ilJkiRJatQ++OADdt9997TLqFWLFy+mefPmLF26lD59+nD33XfTo0ePtMvaaJX97EII42KMlc5xb4ufupSdD1sfkix73QRLZySDQ896Dr54DD69B3IKYNv+sN3xsO1RybhBkiRJkiSpWoYPH87777/P8uXLGTJkSKMIfTaEwU+amrWDnYYlS2kRfPUifDESpj8K0x6ErLykJdB2x0P7AdCkVdoVS5IkSZLUINx3331pl1AvGPzUF1m5sM1hyVL4e5jzOnzxSBIEzXgcQg607QfbDYL2A6Hp1mlXLEmSJEmS6jmnc6+PsrJhqwOh520wcCoc/hbsfjEs/gzGnAWPbgv/6QOTb4Ml09KuVpIkSZIk1VO2+KnvQoDWvZKl269gwaSkFdAXj8D4C5Nli8KkJdB2x8Nmu6ZdsSRJkiRJqicMfhqSEJKp3zfvCl2vgYUfwfSRMO0RmHhFsrTcY3UI5DTxkiRJkiRt0uzq1ZBttgt0vhyOeCvpEtbjNsjbAib9Ep7pBk91hvdvgmVfpV2pJEmSJEmr9OvXj+eee26NY7fddhtnn332Wq/p27cvY8eOBeCoo45i/vz537rNz3/+c26++eZ1PvZjjz3G+++/v2r/6quv5r///W91yq/USy+9xNFHH73R91PTDH4ai4LtYbfz4ZCX4bhZ0OsuyGsNEy6Dx9rD6EEw8xkoLUm7UkmSJEnSJm7w4MGMGDFijWMjRoxg8ODBVbr+6aefZvPNN9+gx64Y/Pzyl7/kkEMO2aD7aggMfhqjpm1hlzPh0Feh//vQ6Xz4+hV46Sh4vAO8cw0s/jztKiVJkiRJm6jvfve7PPXUU6xcuRKAzz//nJkzZ3LggQdy9tlnU1hYSJcuXbjmmmsqvb5Dhw7MmTMHgOuvv55dd92VAw44gClTpqy6zZ///Gd69epFt27dGDRoEEuXLuX111/n8ccf59JLL6V79+588sknDB06lIcffhiAUaNGsddee9G1a1dOO+00VqxYserxrrnmGnr06EHXrl2ZPHlylZ/r/fffT9euXdljjz24/PLLASgpKWHo0KHssccedO3alVtvvRWA22+/nc6dO7Pnnnty0kknVfNVrZzBT2PXcnfocTMcOwMOeAhadoFJ18LjHeGFw2HaQ1CyIu0qJUmSJEmbkC222IK9996bZ555Bkha+3zve98jhMD111/P2LFjeeedd3j55Zd555131no/48aNY8SIEUyYMIGnn36aMWPGrDp3/PHHM2bMGCZOnMjuu+/OX//6V3r37s2AAQO46aabmDBhAjvttNOq2y9fvpyhQ4fywAMP8O6771JcXMwf//jHVee33HJLxo8fz9lnn73e7mRlZs6cyeWXX84LL7zAhAkTGDNmDI899hgTJkxgxowZTJo0iXfffZdhw4YBcOONN/L222/zzjvvcNddd1XrNV0bB3feVGQ3ge2/myxLpsInf4dP/wavfg/ytoQdT4WdfggtO6ddqSRJkiSpLo27AL6ZULP32ao79LxtnTcp6+41cOBARowYwV//+lcAHnzwQe6++26Ki4uZNWsW77//PnvuuWel9zF69GiOO+44mjVrBsCAAQNWnZs0aRI/+9nPmD9/PosXL+bwww9fZz1Tpkxhxx13ZNddk9myhwwZwu9//3suuOACIAmSAHr27MnIkSOr8CLAmDFj6Nu3L23atAHg5JNP5pVXXuGqq67i008/5dxzz6V///4cdthhAOy5556cfPLJHHvssRx77LFVeoz1scXPpqhgB9jz5zDgM+j7LGzVFz68A57qAs/3hk/+BkWL065SkiRJktSIDRw4kFGjRjF+/HiWLl1Kz549+eyzz7j55psZNWoU77zzDv3792f58uUbdP9Dhw7lzjvv5N133+Waa67Z4Pspk5eXB0B2djbFxcUbdV+tWrVi4sSJ9O3bl7vuuovTTz8dgKeeeoof//jHjB8/nl69em3044AtfjZtWdmw7eHJsvxr+Oxf8Mlf4M0fwrjzYYfBsNPp0LqX08JLkiRJUmO1npY5taV58+b069eP0047bdWgzgsXLqSgoICWLVvy1Vdf8cwzz9C3b9+13kefPn0YOnQoV1xxBcXFxTzxxBOceeaZACxatIhtttmGoqIi7r33Xtq1awdAixYtWLRo0bfuq1OnTnz++ed8/PHH7LzzzvzrX//ioIMO2qjnuPfee3PeeecxZ84cWrVqxf3338+5557LnDlzaNKkCYMGDaJTp06ccsoplJaW8sUXX9CvXz8OOOAARowYweLFizd4EOsyBj9K5G8Fu18Mu10Ec15PAqDP74VP/gybd00CoA6nJNPFS5IkSZJUAwYPHsxxxx23aoavbt26sddee7Hbbrux3Xbbsf/++6/z+h49enDiiSfSrVs3ttpqK3r16rXq3LXXXss+++xDmzZt2GeffVaFPSeddBJnnHEGt99++6pBnQHy8/P5+9//zgknnEBxcTG9evXirLPOqtbzGTVqFO3bt1+1/9BDD3HjjTfSr18/Yoz079+fgQMHMnHiRIYNG0ZpaSkAv/rVrygpKeGUU05hwYIFxBg577zzNjr0AQgxxo2+k6oqLCyMY8eOrbPH00ZauQCmjoBP/grzxkBWHnT4PnS+HDbrlHZ1kiRJkqQN9MEHH7D77runXYY2QGU/uxDCuBhjYWW3d4wfrV2Tlsm08Ee8BUdOSAZ/nno/PLk7jD4B5o1Pu0JJkiRJkrQOBj+qmlbdoNfvYeBU6PwT+PJ5eLYnvHgEfP0K1GHLMUmSJEmSVDUGP6qe/K2g+w0wcBp0uyFp9fPfg+C/B8KMpwyAJEmSJEmqRwx+tGGatIQuV8DAz6HnHbDkC3j5aHhmL/h8BJSWpF2hJEmSJGkd6nLMX9WMDfmZGfxo4+Q0g07nwICPYd97oHQFvD4YntwNPv4LlKxIu0JJkiRJUgX5+fnMnTvX8KcBiTEyd+5c8vPzq3VdlWb1CiFcCJwOROBdYFiMcXnm3O3AaTHG5uu7H2f12gTEUpj+GLx3A8wbB023hd0vgZ3OgNz1vkUkSZIkSXWgqKiI6dOns3z58rRLUTXk5+fTvn17cnNz1zi+rlm91hv8hBDaAa8CnWOMy0IIDwJPxxjvCSEUAucDxxn8aA0xwpf/gfd+BV+/BE22gE7nw67nQN4WaVcnSZIkSVKjURPTuecATUMIOUAzYGYIIRu4CbisZspUoxICbHMYHPIiHPo6tNkf3r0G/m8HePtSWDYr7QolSZIkSWr01hv8xBhnADcD04BZwIIY4/PAOcDjMUa/wWvd2uwHBz0OR06EdsfA5N/C/3WAt86CxZ+mXZ0kSZIkSY3WeoOfEEIrYCCwI7AtUBBCOBU4AbijCtcPDyGMDSGMnT179sbWq4as1Z6w/31w9BToOBQ+/Ts8sSuMvwSKl6RdnSRJkiRJjU5VunodAnwWY5wdYywCRgK/AHYGPg4hfA40CyF8XNnFMca7Y4yFMcbCNm3a1FTdasha7Ax7/wkGfAYdh8HkW+DJzjDjybQrkyRJkiSpUalK8DMN2DeE0CyEEICDgd/GGLeOMXaIMXYAlsYYd67NQtUINdsW9vkzHDI6mfHr5WNg9Hdh6Yy0K5MkSZIkqVGoyhg/bwIPA+NJpnLPAu6u5bq0KdnqADjibeh2A8x8Cp7cHabcCaUlaVcmSZIkSVKDtt7p3GuS07lrvRZ9AmN+BF8+D1v0gn3uhlbd065KkiRJkqR6qyamc5fqRoudoN+z0Ps+WDoVni1MBn8uWpx2ZZIkSZIkNTgGP6p/QoAOg+HoybDTD5PBn5/qDNOfSLsySZIkSZIaFIMf1V9NWiWzfx36KuRuBq8MgNGDHPxZkiRJkqQqMvhR/ddmfzhiPHT7Fcx8OjP48x0O/ixJkiRJ0noY/KhhyG4CXX4C/d+DNr1h3Hnw/L4w7+20K5MkSZIkqd4y+FHD0rwj9H0Get8PS7+A5wph/MUO/ixJkiRJUiUMftTwhAAdToKjP4CdzoDJv3XwZ0mSJEmSKmHwo4arSSvY+y449DXIbZkM/vzK8bB0etqVSZIkSZJULxj8qOFr0xuOHA/db4RZz8JTe8Cs59OuSpIkSZKk1Bn8qHHIyoXOl8NR70LB9vDSUTDlTogx7cokSZIkSUqNwY8alxY7JV2/tj0Kxp0LY34EpUVpVyVJkiRJUioMftT45LaAAx9NWgB9fBe8eDismJt2VZIkSZIk1TmDHzVOWdnJmD/7/gNmvwbP7QMLJqddlSRJkiRJdcrgR41bx1Ph4BeheBE8vy/MfC7tiiRJkiRJqjMGP2r82vSGw9+Cgh3g5aNg8u8c9FmSJEmStEkw+NGmoWCHZNDndgNg/AUw5iwHfZYkSZIkNXoGP9p05DaHAx+BzlfAx3fDC4c56LMkSZIkqVEz+NGmJWRB9xtgv3/BnDfgub1hwftpVyVJkiRJUq0w+NGmacdT4JCXoHgJPL8fzHwm7YokSZIkSapxBj/adG25bzLoc/OO8PLRMPk2B32WJEmSJDUqBj/atBVsD4e+Cu0GwvgL4a3hULIy7aokSZIkSaoRBj9STgEc+DB0+Sl88hd48VBYPiftqiRJkiRJ2mgGPxIkgz53uw563wtz3oTn94H576VdlSRJkiRJG8XgRyqvw/fhkJeheGky6POMp9KuSJIkSZKkDVal4CeEcGEI4b0QwqQQwv0hhPwQwr0hhCmZY38LIeTWdrFSndhyHzhiDLTYBV4+Bj74rYM+S5IkSZIapPUGPyGEdsB5QGGMcQ8gGzgJuBfYDegKNAVOr8U6pbrVrD0c+gpsNwjevjgZ+NnwR5IkSZLUwFS1q1cO0DSEkAM0A2bGGJ+OGcBbQPvaKlJKRU4BHPAAdDofpvwOJt+SdkWSJEmSJFXLeoOfGOMM4GZgGjALWBBjfL7sfKaL1w+AZ2urSCk1IQt6/Ba2/x68fSlMeyjtiiRJkiRJqrKqdPVqBQwEdgS2BQpCCKeUu8kfgFdijKPXcv3wEMLYEMLY2bNn10TNUt0KWbDfP6DN/vD6D2D2a2lXJEmSJElSlVSlq9chwGcxxtkxxiJgJNAbIIRwDdAGuGhtF8cY744xFsYYC9u0aVMTNUt1Lzsf+vwfFGwPrwyEhR+lXZEkSZIkSetVleBnGrBvCKFZCCEABwMfhBBOBw4HBscYS2uzSKleyGsNfZ8BsuClI2G5LdgkSZIkSfVbVcb4eRN4GBgPvJu55m7gLqAt8EYIYUII4eraLFSqF1rsBAc9DstmwMsDoHhZ2hVJkiRJkrRWOVW5UYzxGuCaDblWanS23Bd63wejB8Ebp8D+D0JWdtpVSZIkSZL0LVWdzl1Sedsdl8z29cXIZLYvSZIkSZLqIVvtSBtqtwtg8Wcw5VZoviN0OjftiiRJkiRJWoPBj7QxevwWlk6DcecnM361H5h2RZIkSZIkrWJXrw1w//2weHHaVaheyMqG3vdC617w2mCY81baFUmSJEmStIrBTzV99BGccgp06QLPPpt2NaoXcprBQU9A/tbwyjFJ9y9JkiRJkuoBg59q2mUXGD0amjWDI4+EU0+FuXPTrkqpy98K+j0DpcXw0pGwYl7aFUmSJEmSZPCzIXr3hgkT4Kqrkm5fu+8OI0ZAjGlXplRt1gn6PJa0+HnlWChZnnZFkiRJkqRNnMHPBsrLg1/+EsaNgw4dYPBgGDAApk9PuzKlaqsDYb9/wuzR8L9hEEvTrkiSJEmStAkz+NlIe+4Jb7wBt9wCo0ZB587wxz9Cqd/3N107nAjdb4SpI2DiT9OuRpIkSZK0CTP4qQHZ2XDRRTBpEuy9N/zoR9C3L3z4YdqVKTW7XwY7nwnv3wgf3512NZIkSZKkTZTBTw3q2BH+8x/429/g3XeT1kC/+hUUFaVdmepcCFB4J2x7FIz5Ecx8Ju2KJEmSJEmbIIOfGhYCDBsGH3wARx8NV16ZtAIaNy7tylTnsnJg/wdg827w6gkw7+20K5IkSZIkbWIMfmrJ1lvDww/DyJHw5Zewzz5w2WWwdGnalalO5TaHvk9Ck9bwcn9YMi3tiiRJkiRJmxCDn1p23HFJ659hw+Cmm6BbN3jppbSrUp1qug30fRqKl8JL/WHlgrQrkiRJkiRtIgx+6sDmm8Of/5zM+lVaCv36wfDhMH9+2pWpzmzeBQ4cCYumwOhBULIy7YokSZIkSZsAg5869J3vJIM+X3IJ/PWvydTvjz2WdlWqM1t/B/b+C3w1Ct46A2JMuyJJkiRJUiNn8FPHmjVLuny9+Sa0aZN0BTvhhGQcIG0COp4KXX8Bn/0TJv827WokSZIkSY2cwU9KCgth7Fi4/np4/HHYaSfo2xcuvhjuuw+mTEm6hakR2uMqaH8cTLwC5jndmyRJkiSp9oRYh91NCgsL49ixY+vs8RqKyZPhjjuSIGjiRFixIjnevDnstRf07JksPXpAp06QnZ1uvaoBK+bC090gpxkcMT6Z/UuSJEmSpA0QQhgXYyys9JzBT/1SVATvvw/jx8O4cckyYQIsX56cLyiA7t3XDIN22w1yctKtWxvgq5dg1Heg41DY929pVyNJkiRJaqAMfhq44uJkSviKYdDSpcn5pk1Xh0E9eiTrzp0NgxqEiT+D966H/UfADiemXY0kSZIkqQEy+GmESkqSLmLlw6C334YlS5Lz+fmw337JwNGDBsFWW6Vbr9aitAj+0wcWfgBHToDmHdKuSJIkSZLUwBj8bCJKSuCjj1YHQc88k4RDWVnJVPInnpjMIta6ddqVag2LP4NnukPLPeCQlyHLplqSJEmSpKpbV/DjrF6NSHZ2Mt7PySfDb3+bjBX0zjtwxRXw+edwxhmw9dZw5JFwzz0wf37aFQuA5jtCr7tgzusw6dq0q5EkSZIkNSJVCn5CCBeGEN4LIUwKIdwfQsgPIewYQngzhPBxCOGBEEKT2i5W1RMCdO0K110HH36YtAK66KKkFdCwYdC2LQwYAPfeC4sWpV3tJq7DYNhxCLx3HXw9Ou1qJEmSJEmNxHqDnxBCO+A8oDDGuAeQDZwE/Bq4Nca4M/AN8MPaLFQbJ4Rk4Odf/xo+/RT+9z/48Y+TMYJOOSUZA2jQIHjggdXjBKmOFd4BBR3h9ZNh5TdpVyNJkiRJagSq2tUrB2gaQsgBmgGzgO8AD2fO/wM4tubLU20IAfbZJ+kONm0avPpq0g3s9dfhpJOSEOjEE2HkSFi2LO1qNyG5LWD/+2DZLHjzDKjD8bckSZIkSY3TeoOfGOMM4GZgGkngswAYB8yPMRZnbjYdaFdbRar2ZGXB/vvD7bfD9Onw4oswZEiyLpsN7JRT4PHHYcWKtKvdBLTuBd2uhy8egU/+mnY1kiRJkqQGbr2zeoUQWgGPACcC84GHSFr6/DzTzYsQwnbAM5muYBWvHw4MB9h+++17Tp06tUafgGpHcTG89FLS9WvkSJg3D1q2hI4dk6ni8/KSZWO327RJxiFSObEUXjwcZr8OR4yDlrulXZEkSZIkqVC6aUYAACAASURBVB7bqOncQwgnAEfEGH+Y2T8V2A84Adg6xlgcQtiPJAg6fF335XTuDVNREfz3v0kA9OWXsHx50vpnxYp1b5eWVu3+b7oJLrmkdp9Dg7NsFjy9JzRrD4f9D7Lz0q5IkiRJklRPrSv4yanC9dOAfUMIzYBlwMHAWOBF4LvACGAI8H81U67qm9zcZAr4I4+s3nXFxesPie64Ay67DHbdNZlhTBlNt4F9/w4vHwMTfgI9b027IkmSJElSA7Te4CfG+GYI4WFgPFAMvA3cDTwFjAghXJc55oAkWkNODjRvnixr06sXTJ0K3/8+vPYadOtWd/XVe+2Ohl3PgSm3wTaHwbbVTN4kSZIkSZu89Xb1qkl29VJlZs1KAqCsLHjrLdh667QrqkdKlsNze8OyL+God6CpL44kSZIkaU3r6upV1encpVqzzTbwxBMwdy4cd1zSDUwZ2fmw/wgoXgRvDEkGfpYkSZIkqYoMflQv7LUX/Pvf8L//wWmnQR02RKv/WnaGHrfCl8/D5NvSrkaSJEmS1IAY/KjeOO44uOEGuP9+uO66tKupZ3Y+E9ofCxN/AvPGp12NJEmSJKmBMPhRvfKTn8Cpp8LVV8NDD6VdTT0SAuzzF8jbCl4bDEWL065IkiRJktQAGPyoXgkB7r4b9t8fhgwBxwIvJ6819P43LPoIxp2fdjWSJEmSpAbA4Ef1Tl4ePPootG0LAwbA9OlpV1SPtO0LXa6AT/8GUx9MuxpJkiRJUj1n8KN6qU2bZKavxYuT8GfJkrQrqke6/hxa7wNvDYfFn6ddjSRJkiSpHjP4Ub21xx4wYgRMnJiM+1PqTOaJrFzY/75kavc3ToHS4rQrkiRJkiTVUwY/qteOOgpuuQVGjoSrrkq7mnqkeUfY+y6Y/RpMcgo0SZIkSVLlDH5U751/Pgwfnkz1/q9/pV1NPdLh+7DjqfDetfD16LSrkSRJkiTVQwY/qvdCgDvvhH794PTT4bXX0q6oHim8Ewo6wusnw8pv0q5GkiRJklTPGPyoQcjNhYcfhh12gOOOg88/T7uieiK3RTLez7JZ8OZwiDHtiiRJkiRJ9YjBjxqMLbaAJ5+EoiI45hhYuDDtiuqJ1r1gz2vhi4fhi5FpVyNJkiRJqkcMftSg7Lpr0vLngw/g+9+HkpK0K6ondr8EWnWHcedB0aK0q5EkSZIk1RMGP2pwDj44GfPnqafg0kvTrqaeyMqBXnclXb7ecfozSZIkSVLC4EcN0llnwXnnwa23wp//nHY19cSW+8AuZ8GHd8C88WlXI0mSJEmqBwx+1GDdcgsccQT86Efw4otpV1NPdLsB8trAW2dCqf3gJEmSJGlTZ/CjBisnB0aMSMb9GTQIPvoo7YrqgSabQ49bYd5Y+PiutKuRJEmSJKXM4EcNWsuW8MQTkJ0NRx8N33yTdkX1wA4nwdaHwsQrkzF/JEmSJEmbLIMfNXgdO8LIkfDZZ3DCCcl075u0EKDw91CyAsZdmHY1kiRJkqQUGfyoUTjwwGSQ51GjYOhQmDIl7YpSttku0OVKmPYAzHwu7WokSZIkSSkx+FGjMWQIXH013Hcf7LYbdO8ON9ywCY/90/lyaLErjP0RFC9LuxpJkiRJUgoMftSo/OIXMH063HYbFBTAT3+aDP7cowfceCN88knaFdah7DzY+y5Y/Cm8d0Pa1UiSJEmSUmDwo0anXTs4/3x47TWYNg1++1vIy4MrroCdd4bCQvjNb5IxgRq9tv2gww/gg1/DgslpVyNJkiRJqmPrDX5CCJ1CCBPKLQtDCBeEELqHEP6XOTY2hLB3XRQsVcd228GFF8Ibb8Dnn8PNNyczgF1+eTIo9N57J8emTk270lrU42bILoAxZ0GMaVcjSZIkSapD6w1+YoxTYozdY4zdgZ7AUuBR4DfALzLHr87sS/XWDjvAxRfDm2/Cp58mrX5ihEsvhQ4dYN99k9ZBX3yRdqU1LH8r2OvX8PXL8Nm/0q5GkiRJklSHqtvV62DgkxjjVCACm2WOtwRm1mRhUm3accck8BkzJhn358Ybk2ngL74Ytt8eevdOxgmaMSPtSmvITqfDlvvB2xfDirlpVyNJkiRJqiMhVqPrRwjhb8D4GOOdIYTdgeeAQBIg9c4EQmtVWFgYx44duzH1SrXqo4/goYfgwQdh4sTk2P77JzOGnXwyNGuWbn0b5Zt34Nke0HEY7PPntKuRJEmSJNWQEMK4GGNhZeeq3OInhNAEGAA8lDl0NnBhjHE74ELgr2u5bnhmDKCxs2fPrl7lUh3bZRe48kqYMAEmT4Zrr4X582H4cGjfPhkbaNq0tKvcQK32hN0uhE/+ArNfS7saSZIkSVIdqHKLnxDCQODHMcbDMvsLgM1jjDGEEIAFMcbN1nUftvhRQxQjvPoq/O538OijEAIcdxycdx4ccECy32AUL4EnO0PuZnDkeMjKTbsiSZIkSdJGqpEWP8Bg4P5y+zOBgzLb3wE+2rDypPotBDjwQHj44WRQ6IsvhlGjoE8f6NkT7rkHli9Pu8oqyimAwjtgwSSYfGva1UiSJEmSalmVgp8QQgFwKDCy3OEzgFtCCBOBG4DhNV+eVL/ssAP8+tfJzF9/+hOsWAHDhiUDQl99NcxsCEOctx8A7Y+Fd38Oiz9PuxpJkiRJUi2qUvATY1wSY2wdY1xQ7tirMcaeMcZuMcZ9Yozjaq9MqX4pKEjG/Zk0Cf7zn2Qq+OuuS4Khk09Opoyv13r+DkIWjD0n6csmSZIkSWqUqjudu6RyQoBDDoHHH4cPP4RzzoEnn0yCoH33hfvvh5Ur066yEgXbQ9dfwMynYPpjaVcjSZIkSaolBj9SDdl5Z7j1Vpg+He64A+bNg+9/Hzp0SFoDff112hVW0Ol82LwbjD0XihalXY0kSZIkqRYY/Eg1rEWLpOXP5Mnw1FOw555w1VXJOEDDhiVTxdcLWTmw912wbCa8c03a1UiSJEmSaoHBj1RLsrLgqKPg2Wfh/ffhhz+Ehx6CvfZKZgSbMiXtCoEt94Wdz4QPfwfz3k67GkmSJElSDTP4kerA7rvD73+fdAO75Rb44AM4+GD47LO0KwO63wB5W8JbZ0JpSdrVSJIkSZJqkMGPVIc23xwuugheeAGWLk3CnxkzUi6qSSvY67cwbwx8cnfKxUiSJEmSapLBj5SCrl3huedgzpxkVrDZs1MuqMP3oe3BMOEKWPZlysVIkiRJkmqKwY+Ukl69kqnfp06Fww6D+fNTLCYE6PUHKFkG4y9KsRBJkiRJUk0y+JFS1KcPPPoovPdeMhD04sUpFrPZrtD5Cph6P8x6PsVCJEmSJEk1xeBHStnhh8OIEfDWWzBwICxfnmIxXX4CLXaBMT+C4mUpFiJJkiRJqgkGP1I9cPzx8Pe/J4M+n3ACFBWlVEh2ftLla/En8P6NKRUhSZIkSaopBj9SPfGDH8Af/5iM+/ODH0BJWjOrb30IdDg5CX4WTkmpCEmSJElSTTD4keqRs86Cm26CBx6AM86A0tKUCtnrFshuCm8MgeIlKRUhSZIkSdpYBj9SPXPJJXD11UnXrwsugBhTKKJpW9j3bzBvDIz+LpSsTKEISZIkSdLGMviR6qGf/xwuvBDuuAN+9rOUitjueNj7bpj1LLzxAyhNq++ZJEmSJGlD5aRdgKRvCwFuuQWWLIEbboAWLeAnP0mhkJ1+CCu/gbcvhSabQ6+7kuIkSZIkSQ2CwY9UT4UAf/gDLF4MV1wBzZvDOeekUMjul8CKefD+r6BJK+jubF+SJEmS1FAY/Ej1WHY23HNP0vLn3HOhoACGDUuhkG7XJy1/3v91Ev50vjyFIiRJkiRJ1WXwI9VzubnJLF/HHAOnn560/DnhhDouIgQovBOK5sOEnyThz87D67gISZIkSVJ1GfxIDUBeHjz6KBxxBHz/+9CsGfTvX8dFZGXDfv+EooXw1lmQ2xJ2OLGOi5AkSZIkVYezekkNREEBPPkkdOsGgwbBCy+kUERWLhzwELQ5IJnpa+azKRQhSZIkSaoqgx+pAWnZEp57DnbeGQYMgDfeSKGInGZw0BPQsguMPh5mv5ZCEZIkSZKkqjD4kRqY1q3hP/+BbbaBI4+ECRNSKKJJS+j3HDTbDl7qD99MTKEISZIkSdL6GPxIDdA228B//wubbQaHHQaTJ6dQRP5W8J3/QG4LePEwWPhRCkVIkiRJktbF4EdqoHbYIQl/srLg4IPh009TKKJge+j3H4il8OKhsHR6CkVIkiRJktZmvcFPCKFTCGFCuWVhCOGCzLlzQwiTQwjvhRB+U/vlSipv112Tbl/Ll8N3vgOvvppCES13g37Pwop58MJhsHxOCkVIkiRJkiqz3uAnxjglxtg9xtgd6AksBR4NIfQDBgLdYoxdgJtrt1RJlenaFZ5/HkKAAw+Ec86BRYvquIgteiYDPi/5DF46MpnyXZIkSZKUuup29ToY+CTGOBU4G7gxxrgCIMb4dU0XJ6lqevaEd9+FCy6AP/wBunSBp5+u4yLaHpRM9f7N2/DyQCheVscFSJIkSZIqqm7wcxJwf2Z7V+DAEMKbIYSXQwi9KrsghDA8hDA2hDB29uzZG1OrpHVo3hxuvRVeew1atID+/eEHP4A5ddnzqt3RsO8/4OuX4bUTobSoDh9ckiRJklRRlYOfEEITYADwUOZQDrAFsC9wKfBgCCFUvC7GeHeMsTDGWNimTZsaKFnSuuy3H4wfD1dfDSNGQOfO8MADEGMdFbDjyVB4B8x4Av73w2TgZ0mSJElSKqrT4udIYHyM8avM/nRgZEy8BZQCW9Z0gZKqLy8PfvGLJADq0AFOOgkGDoTpdTXp1q4/hj2vhc//BeMuqMPUSZIkSZJUXnWCn8Gs7uYF8BjQDyCEsCvQBHA6H6ke6doV3ngDbrklmfq9Sxe4+24orYtGOF1+CrtdBB/eAe/+og4eUJIkSZJUUZWCnxBCAXAoMLLc4b8BHUMIk4ARwJAY/bO+VN9kZ8NFFyWDP/fsCWeeCQcfDB9/XMsPHALsdTN0HAaTfgGTf1fLDyhJkiRJqqhKwU+McUmMsXWMcUG5YytjjKfEGPeIMfaIMb5Qe2VK2lg77QSjRsFf/gJvv520BrrpJigursUHDQH2vhu2Ox7GXwCf/qMWH0ySJEmSVFF1Z/WS1ICFAD/8Ibz/Phx+OFx2Gey7L0ycWIsPmpUDve+DtgfDm6fBW2fBtIdhuT1DJUmSJKm2GfxIm6Btt4VHH4UHH4QvvoDCQrjqKlixopYeMDsP+jwG238PPr8XXj0BRraBZ/aC8RfDjKegaGEtPbgkSZIkbbpCXQ7LU1hYGMeOHVtnjydp/ebOhYsvhn/8A3bbLekKtv/+tfiApUUwdyx8NQq+egFmvw6lKyBkwxa9YOuDoe13YMv9IKdpLRYiSZIkSY1DCGFcjLGw0nMGP5IAnnsOhg9PWgCdcw7ccAM0b14HD1y8DOa8kYRAX70Ac9+CWAJZedCmdxICtT0YWhdCVm4dFCRJkiRJDYvBj6QqWbQIfvpTuPNO2G47+Ne/oE+fOi6iaCF8PXp1EPTNhOR4TnPYqk8mCPoOtOoGwd6qkiRJkmTwI6laXn8dTj0VliyBqVOhSZMUi1k+B75+eXXXsIVTkuNNtoC2faHNgcl2TkFmaZassyuu85PRrTdGLIWiBbBiHqycByu/Kbc9L9kuquTYym8gvw1svueay2a72opJkiRJ0kYz+JFUbc8+C0cembT6OeWUtKspZ+kM+OrFJAT6chQsnVbFC8O3Q6E1tsvOFQAxCWtWBTeZ8GblN8m5tclpnoRQeVsk6yatMuvNYdksmP8OLPwgGecIIKsJtOz87UCoaduNfJEkSZIkbUoMfiRVW2kpdO4MLVrAW29tfGOZWhEjrJgNRYugeEmylCzNbC+Fksy6OudKlib3W1mAs2q/snCnFWRXoWlUyUpYNAW+eScJgsqWZTNX3yZ/q0wI1C1Zt9oTNts9mR2txl/DUihZlnlNlkLJimSw7ZLl5bZXQGmF/ZLl5c6t45rSouQxYkklS3WPlwARclpAXmto0jpZl99ussXqY2XHczerp2/gSqxckISZS6Zl1l8k66VfQFY+NN0Gmm4N+Zml/H5Dep6qWTEmn18r564Oq1fM/XbLw+y85LMqd/MkkG7SavW67FhuS8jKSfsZqTGKpcn7cPlsWDEn+fe7ZBk02x6ad0w+y+zCLUkbzOBH0gb5wx/gxz+G116D3r3TrqaRWz4b5r+7Zhi04L0kTIFk1rPNdlvdKqjl7snxVUFWuVBrVZBVIeBaI/gqO7Zs42sPWclg3Nn5mXVeuXUTICupPys7WZdf1neu4nFCMg7UirmZL7mZpWj+OurLyQR1rcutywdGWyRhUm6LpNVWbotkf9V2Qc18GSlZCcumrw5zKoY7S6ZB8aJv196sfbKUrIDlXyZLWaux8rKbZsKgTCC0Khwqt990a8hvu/4uhjEmj1G6AkpXJktJ2faKCvvltilN6lhjyV+9nZNZ1/cujjEmz3PV70ol6/K/R6UrM+/RnMx7NiezZCchSsjJrMudK7//re2c5He/0gBnLeFOZe+JMtnNklCndAWsnJ8JUdchp8WawVDuWkKiJltAs22hafukO6tf2jctxcuS8GbFnGQpH+iUX5cdXzk3CX/WJjsfCnZMQqBvLTsmn8WS6q/SkuT3ftnMpKX7slnJH6Rymq9ecpuvuZ9T4B8bapDBj6QNsnhxMsjzYYfBAw+kXc0mqLQYFn28Zhg0/x1YMnUdF4XVXdcqjnO0Rle3ysZCapq0KlkV3ORXCHEq7JcFPfXhH+zSkkx3vHJhUNl22RfkimHRyrmrg7X1ySlYMwwqC4kqC4xymychz6pwJxPsLPuSb3UVzGsDzbaDgu2Tv3oXbL/mfn7bJEgor+yv5ssyIdCyL2H5rMr3V86r/PnkbZkssXQt4c7Kav8IqiVkrxkO5VQMi8oFRhXfX9/6f0us5vnSpDXaukKdkqXr/oKaluxmq1sblrVwK7+9RsBZdrxV8lqWKWsdVDQ/8zszP7N8kzlWyfbK+atvX7Sw8tqycqHptklI2bTdmutmmXX+NlVrGVldpUXlxlz7pkJQNo81Phe/1c23ss/IptUPsWJMgvRVr9X8NbcrO1Z+OxZnXqvynwHlPguataubwDSWJkHNshnJl7elmfWyGbB0Jqz4enWYU7K08vsIWZnPmDarP2vKtvPbrLmfnZ98Ti75FBaXWxZ98u0gPL/tWkKhjsl7ryEGj6XF3/7DzYa2Xg6hwuvdJvN6l1vntamd38HyYkxqXPX58k25z5EFdffZWmkL2Coey9sSNu8CLTrV/uvVEJQWw/Kvk/9bLJ1Z+XrZTFj+1Yb9fLPz1wyC1hoSNU9aNzfvAC12geY7J/9/0CoGP5I22KWXwq23wmefJSGQ6oGV82Hhh8kX4opfWLLy7O5THcXLki+GRYugeHHyRaNsu2hRZn8tx1dtl1uXDxmym5b70lZJuNNsu9r/D0vJiuQ/a8tmrW4ttOzLZH/FnOQ9lNUks2RaaJW11Fpju7JjlZwPIXlNS5Zluv4t+/ZSXH6/Crep7D+R33qPr2e/4u2zm64Zgq5vvb7bZjfJBErFyRf4WJKsy++XbVf1NtlNKnRh3GLNACctpSVQvDATtMzNhAPTk4Bg6fSkVVvZdmXBQH7bcoFQJhRqWi4cym62ZnhT2SD6K8sFPCvmfTsgqAll75HKQqLs/OT3vWKws65WV2X3uaoFVbl1k82BkHkdM0HxirkVLg5Jy701wqAKAVFem3V//hctzgQ45cOc8gHPjOSzIRZX8thbJ+FK3lbfDm8q7jfZfONDmBiTn+/iTytflk5b87Mhq0nSKqigYzJOXnbZHzIqWdY43nQd5zJLyEl+P4uXZP6dWLx6u6jCfrXOL9mAkH0d4xUS12xptbYv4LmbfTsYWvVzrHCc0soDnLLtNQLkcsfW97vQUIRsaLErtOySLJvvkaxb7Fy3LVdjZuzJJZ/B4s9gyedJ4AKZPw5lWk+Htayrer5kaaalTrkWO8sygW9l76e8Npku59uuZb0NENZ8/xctXvN3oeL+t45V+L2rqFn7JAQqC4LKtlvsVD/+zaxjBj+SNtjUqdCxYxIA3Xhj2tVI9VjZeElFizLdy1obwmnTFWPy1/2yUKh8ILR0eiZ0mL72VmkVZeVWMsZaubHWKhuDLW+LZMwiWEuX2LW1pKiwXfH2JcvKdYerEOCU7xa3xvGW1RunrXhpEgKVtRos6xpa/ljFrrpZeWuGQbF0zYCnsoAsd7MkiGu6bSaEq7DdbNukm2h9aNlZXmlR8jp8KxT6JAk9SlckAXLZWHMbI2RVrxVDyP52y4Wy7dzy++tpebYxM5SuMZ5SZlleYV1xuzqvU8jOvM9blXvPl+sOWnG77Da5m9XNe6nS77eVHFvb7ZbNSrrbL3gPFkyC+e8l762y+8jKTbrflwVCLbtAyz2S1mcVW+lWVdHi1cFOWbhTPuip2NoypwDIynTdLRsbsXT9XXnXKyTBX9Ntk1aazSqsy0Kd/LZ13xqqbHbdxZ/Cwo9g8cew6KPVy4o5az6PZttlQqDygdAuyc+pNsbNrAcMfiRtlEGD4MUXYfp0aNYs7WokSY1G8bLVwURZK6HKBtPPbmaQWl6MSaugimFQ+YCIrEyAs20mxKkk4MltnvYzqX2lJeUmIFiemayg/LJszf1vnV+etCjKrRjmVAh0sguSdUNseRtjZvy8OWsGQiG78gAnp6DhPceNVbwUFk5eHQjNn5Ssl3y++jbZ+ZlAaI81WwkV7JC07loyNRPklAt0yvYrtvLLKUjGvCrokLRma75jZgysDsm6Scu11xpLy02UUVp5OFTZ+awmVRsHsL5aOb9cEFQhFCr/R4aQlbSaLB8I7XpOw33e5Rj8SNoor7wCBx0Ef/oTDB+edjWSJElSPVC0GBa8v7p1UFkwtHT66ttk5SXhY3lZTZJAqGDHcsFOh9X7eVtueuFabVoxb+2hUCyGExY2itfb4EfSRokRevaElSvh3XcbxeeiJEmSVDtWzl8dCC2cnHT3XNVqZ8eku1RDHJC8sSlr7bauFlQNyLqCn3rWYVdSfRQCnH8+DB0Ko0bBIYekXZH0/+3dd3iUVRYG8PeGFDokEKRKEaQuZGlKERCkCSIrIgooqAiiINhQbKvYBZEmKoKiCEoXRHFhJSDSNDSBKCAEqZLQQ5GUOfvHyewECMlMMjPfN5P39zzzTDKTzJwpX7nnnnsvERERkU2FlwSim+uF7MuYoEn65IRpRiJyy913A2XKAOPHWx0JERERERERuYuJHyJyS0QE8PDDwJIlwO7dVkdDRERERERE7mDih4jcNngwEBYGTJxodSRERERERETkDiZ+iMhtZcsCvXoBn34KnD5tdTRERERERESUEyZ+iMgjw4cDZ89q8oeIiIiIiIjsjYkfIvJIo0ZAixY63Cs93epoiIiIiIiIKDtM/BCRx4YNA/buBb791upIiIiIiIiIKDtM/BCRx/71L6BSJS7tTkREREREZHc5Jn6MMTWNMVsyXc4YY4Znuv9JY4wYY0r7NlQisovQUODRR4EVK4Bff7U6GiIiIiIiIrqaHBM/IrJTRGJEJAZAIwDnASwEAGNMJQAdAOz3aZREZDsPPQQUKgRMmGB1JERERERERHQ1ng71agdgj4j8mfH7ewBGABCvRkVEthcVBdx3HzBzJnDsmNXREBERERERUVY8TfzcDeBLADDG3A7gkIhsze4fjDEDjTFxxpi4pKSkXIZJRHb02GPA338DU6ZYHQkRERERERFlxYi4V6xjjAkHcBhAXQDJAGIBdBCR08aYfQAai0i2/f6NGzeWuLi4vEVMRLbSoQMQHw8kJABhYVZHQ0RERERElP8YYzaKSOOs7vOk4qczgE0ichTAdQCqAtiakfSpCGCTMaZsXoMlosAybBhw6BAwf77VkRAREREREdHlPEn83IOMYV4isk1EyohIFRGpAuAggIYi8pcPYiQiG+vcGahRAxg3zupIiIiIiIiI6HJuJX6MMUUAtAewwLfhEFGgCQnRuX42bNALERERERER2YdbiR8ROScipUTk9FXur5LT/D5EFLz69QOKFwfGj7c6EiIiIiIiIsrM01W9iIiuUKwY8OCDwNy5Ot8PERERERER2QMTP0TkFUOGAOnpwAcfWB0JEREREREROTHxQ0ReUa0a0K0b8NFHwIULVkdDREREREREABM/RORFw4YBx44Bs2ZZHQkREREREREBTPwQkRe1aQPUr6+TPItYHQ0REeVnqanaEbFzp9WREBERWYuJHyLyGmO06mfbNmDlSqujISKi/EgEmDcPqFsX6NMHaN0a2LfP6qiIiIisw8QPEXlV795A6dJc2p2IiPwvNha48UagZ08gPFznnUtJATp1Ao4ftzo6IiIiazDxQ0ReVbAgMGgQsHgxsHev1dEQEVF+sHUr0Lkz0LYtcOQI8OmnetvAgcCiRVrx060bFx8gIqL8iYkfIvK6Rx4BChQAJk2yOhIiIgpmCQlA377AP/8JbNgAjBkD7NoF9O+vxyEAuOkm4IsvgHXr9G/T0y0NmYiIyO+Y+CEirytfXsvsp00DkpOtjoaIiIJNUhIwfDhQsyYwfz7wzDNaZfrkk1p5erk77wTGjgUWLACeeIILEBARUf7CxA8R+cTw4cCZM8D06VZHQkREweLcOeC114DrrgMmTgT69QP++AN4802gZMns/3f4cE36TJigSSAiIqL8gokfIvKJpk11gs2JEwGHw+poiIgokKWmAh98oAmfF18EbrkF2L4d+PhjoEIF9x9n9GjgrruAp54CvvrKd/ESERHZCRM/ROQzw4YBu3cDS5daHQkREQUiEWDOHKBOHZ0/7vrrgbVrdchW7dqeP15ICPDZZ0CrVlottHKl10MmIiKyiZn77QAAIABJREFUHSZ+iMhnevTQnlgu7U5ERJ5asUKrR3v10nl7liwBVq0CmjXL2+MWLAh8/bVWD3XvDuzY4Z14iYiI7IqJHyLymbAw7aFdvhyIj7c6GiIiCgSbNwMdOwLt2gGJiVqhs2UL0KULYIx3niMyEvj+e6BwYaBTJ+DQIe88LhERkR0x8UNEPjVwoPauTphgdSRERGRn+/cDffoADRsCcXHAu+8CO3cC993nWprdm669FvjuO+D0aeDWW/WaiIgoGDHxQ0Q+Vbo00Lcv8PnnwIkTVkdDRER29J//ADExwMKFwMiRwJ49ugJXVkuze1NMjC4HHx+vw5NTUnz7fERERFZg4oeIfO6xx4ALF3T1FSIiIieHA3jjDaBzZ6BiRWDbNv09p6XZval9e+CTT4AffgAeeEAnlCYiIgomTPwQkc/94x9A27bA++8DaWlWR5O9Dz8EBg3SpYOJiMh3zpzRKpvnnwfuuQdYt04nXLbCvfcCr78OzJyp8RAREQWTUKsDIKL8Ydgw4PbbgRkzgPvvtzqarM2aBQwerD+HhACTJ3tvIlEiInKJjwf+9S8d0jVunFaGWr2/HTlS5xl6802gUiXX8YCIiCjQMfFDRH7RpQvQvLmeSFetCrRpY3VEl4qNBfr3B1q3Bho1AsaOBWrWBIYPtzoyIqLgMm+e7m+LFtUl21u1sjoiZQwwaRJw+DAwZAhQvrx2WBAREQU6DvUiIr8oUAD45hst4+/WTZfrtYsdO7TnuUYNnVh09Gj9/YkngCVLrI6OfI3zeRD5R1oaMGIE0LOnDgHeuNE+SR+n0FDgyy+Bxo11+Nn69VZHRERElHdM/BCR30RF6cotkZFAp07A7t1WR6Q9u507A4UK6bK+kZE6zGvGDF1S+O67ga1brY6SfOHCBeDRR4FSpYDFi62OhrJy9Chw7pzVUZA3JCUBHTtqYv2RR4BVq4AKFayOKmtFimjSv0IFoGtXYNcuqyMiokB19KhWvc+aZXUklN8x8UNEflWxIrBsma7k0qGDJl6skpysB+MTJ4BvvwUqV3bdV6SIJgMiI/XE/8gR6+Ik79u2TXv0J08GihfXCq/337c6Ksps4UKtEGzSBEhMtDoayotfftEhtGvWAJ9+qttaeLjVUWUvOhpYulQ7Ajp35neQiDx35ozuP777DujTRxcQIbJKjokfY0xNY8yWTJczxpjhxpjRxpjfjTG/GmMWGmP8uPAmEQWymjX1hPrYMe0BPnnS/zGkpupwg23bdL6Jhg2v/Jvy5XV42smTOjzt/Hn/x0neJaJzeDRpAhw/rhVoO3YAt96qc3qMGKFJSbKOwwG8/DJwxx06/HLfPuCWW3R/QYFn2jSgZUtNoKxdq3P7BIrq1bXy58gR7QDIr9VnIrqf/PxzaztriALJ338D3bvreebChboPGTwYePddqyOj/CrHxI+I7BSRGBGJAdAIwHkACwEsB1BPROoD2AVgpE8jJaKg0rgx8PXXWkJ/223+TaqIAA8/rI3+Dz/UYWdXExOj8z1s3Ajcdx+TAoEsKUkTeEOHAu3aAb/+qlVnRYroSdngwToMpXdvPWEj/0tO1uW9X3lFEwTr1mnydfduoH17rc6jwHDxIjBoEDBggE6av3Fj1gl2u2vaFJgzR+Pv1UvnKcoPEhI0ade7N1CuHFCvHtCvH1C7th43eSwkurr0dK3wiY0Fpk/XBNCCBcBddwFPPaXHOM4vSP7m6VCvdgD2iMifIrJMRJyHv/UAKno3NCIKdu3aATNnai9wz55aheMPr70GfPIJ8MIL2ijJyW23AWPGAPPn6/9YLTVVk1YceuC+//4XaNBAhxmOH6+9+GXKuO4PDdXhJ2+/DcyezSSDFfbsAZo100TP+PG6jRYsqPuJRYuA337TRN2pU1ZHSjk5cEAnbZ4yRZdIX7pU59IKVF276rDQb7/V+YmCscF29Kh2cgwYAFSrppcBA3TVtXbtgKlTgZ9+0k6bwYOBm27SKqBgJwKsXKkdBUTuENH9xIIFwLhxmgACgLAwneenf3+tan3mmeDcl5CNiYjbFwCfABiSxe3fAOib0/83atRIiIgu99FHIoBI374i6em+fa7p0/W57rtPxOFw//8cDpGBA/V/P/3UZ+Hl6NAhkZYtNY7wcJHevUVWr/bsteQnFy+KPP20vl+1a4ts2ZLz/3z5pb63NWuK7N3r+xitEBsrcuutIosXWx2JWrZMJDJSJCpK5L//zfpvvv1WP5emTUVOnfJvfOS+FStEoqNFihYVmT/f6mi86/nndV/y6qtWR5J3J0+KfP21yGOPidStq68LEClRQuT220UmTBDZsePKY4vDIfLZZyKlSomEhYm88ILIhQvWvAZfS07W8xLne1O/vsg774gcOGB1ZGRnL7yg35eRI7O+Pz1d5NFH9W8eecT3572UvwCIk6vlcq52xxV/CIQDOAbgmstufx469Mtc5f8GAogDEHfttdf671UTUUB57TXdIw0f7rskxvLlIqGhIu3aaULAUykpIrfcoie7K1d6P76crFwpcs01IoULi0yapCfsJUro+1avnsj774ucPu3/uOxq1y6RRo30/Rk0SOTcOff/d9UqkZIlRcqUEfn5Z9/F6G9padpoDQnR7zEg0qWLyB9/WBOPwyEydqzG849/iOzZk/3fL16s23CzZiJnzvgnRnKPwyEyZoxIgQIitWqJ/Pab1RF5n8OhnQaAyOuvB1bC/fx5PQY++6wmT0NC9HUUKiTSvr3IW2/pvi4tzb3HS0wUufdefYwaNTThF0x27NDOAmNE/v1vPb42a6av1xiRtm21E4jHXMpswgT9jjz4YPb7B4dDZMQI/dt+/URSU/0WIgU5byV+bgew7LLb+gNYB6CwO4/Bih8iuhqHQ2TYMNcJtbdt3SpSrJg2LvNSLXDypDZqoqI0seAPDofI229rg6pmTZHt2133nT0rMnWqSMOG+t4VLSry8MP6evMrh0NPyIsU0SqS3FYdxMeLVKmiiTa7VMbkxV9/aeIS0Eqx48e1oV60qEhEhMiLL3qWHMur8+ddDccePbR33R3z5+u20LKl+/9DvpWcLHLXXfpZ3nFHcCflLl7U7QcQuf/+3HUi+ENKisiaNZrobdNGq+UATZy2aKHb+8qVIn//nbfnWb5cpFo11/tx7Jh34rfS55/rfr9MmSsrEHfvFnn5ZZHq1fU1Fywo0quXyJIl+p5T/jVrln4nund3L5HjcIiMGqX/07OnffclFFi8lfj5CsD9mX7vBCAeQLS7j8HEDxFlJz1dpE8f3TNNmeK9xz1wQKRCBb14o0T7jz+0zL1GDW08+9KpU3oS4TwxuFqDyuEQ2bBBpH9/PREFRJo3F5kxI3jL8LNy8qTI3Xfr62/dOu+f95EjWjUUEiIyebJXQrREbKxI2bL63fj440t7Ig8dcjVkK1cWWbDA95UMBw+KNGmizzlqlOel7rNn62fSpo1/k1V0pV27dKhQSIhWjQRSFUxuORwiL72k39+2bUVOnLA6oktt3qzHJ+cQpZgYkSefFPnuO98k5c6d00qi0FCR0qVFvvgiML8H58+LPPSQvmetWokcPnz1v3U4RNat0yE7pUrp/5QuLTJkiMj69YH5+in3vv9ev/+tW3t+zvXuu67q2/x0vka+kefED4AiAI4DKJHptj8AHACwJePyYU6Pw8QPEeUkJUWkc2dtRHhjfohTp7TKp1gx71bBrF6tPaht2viul2brVu1VDA0Vee89908kjx/X4TPXX+86GR0xIudhNIFuzRpNXBQooFVj7g5ZyElyskjXrvpejhgRWOPxMw/tuv767LeBVat0WwFEOnYU2bnTNzGtWaNJqKJFdY6R3Jo5U19Xu3baYCP/W7xYpHhxbfguX251NP732Wc6ZLJWLXvsXx0OTexGRIiUL68VCElJ/nv+rVtFbrhB9yEdOtjjPXHXrl0iDRrI/+dm8WToTUqKyDffaNWbs+OlRg2RV16xbhgt+c/69Vph3KBB7ivKP/zQNYSQlaz+tXixzuEZLMlar1T8eOPCxA8RuePsWR1LHx4u8sMPuX+cixe1URgaqpPHetuMGboXfeAB7x8wpk/XuRfKldMkU244HFqm3qOHJkOMEenUSWTRIu8lRewgLU1PsAsUEKlaVXthvS01VYfQAVrWHwi9ckeP6twdgMg997jX05+aKjJunDbmw8K0F//sWe/FNHWqPm716pcOWcytzz7T73XHjoHxmQSLLVu0AhHQirh9+6yOyDorV+qQ0uhokbVrrYvj7FnX0MlbbtHt3wppaSITJ2pnS6FCOkzZ7kOg5szReKOidBL5vDh1SuSTT0Ruvln3TYCez0yeHBzD4OhS8fGa+K5WTSuE8+Lzz7Uzo3lzrV4m3zpwQIcmA1q1GizbJxM/RBRwjh/XHXHRoiJxcZ7/f+ZJOKdP9358Ti++qM/x9tveebwLF1yrh918s87L4g0HD+q8BOXL62NXqqQTauf1RMVqf/4pctNN8v95a3y52pNzriVAn9PXw/zyIjZWk4YRETps0tPE5JEjOuEkIFKxojaM8pLcTEnRIRDOSgBvDo2ZNs1VJp/X+Uooexs2iNx2m77fxYrpKlesthL5/XeR667T7W32bP8/f3y8Hi+N0f28HRL7Bw64hik3aGDPSfIvXhQZOlRjvPFGPZ540/79esyoV0+fIyxMpFs3kblzmagOBvv367nUNdd4r7Jr3jz9njRs6N9qvfwkLU1k/HhtXxQsKPLGG8E1vxITP0QUkA4e1KE70dGeDztxJmReecUnof2fw6EVIMbo3Ch5kZDgWoXq2Wd9s8pDaqrG6awECQ3V8vTY2MArc507V1feKlpUe8r8ZdYs+y73np5+6dAud5avz85PP+n8IIBWz8XHe/4YiYk6JBIQeeop33yvP/pIH79bt+A6gbOLH3/UhB2gVRGjRtlvXhurJSXppMmAyJtv+m9/OmuWDjOJjvZNZWteLVigHQ7G6EqUdpn4OyFBVzcDRB5/3Lf7DYdD98VPPeXqfClWTCsVR43SymZvVlaS7x07pqu+FS8usmmTdx/72281IVG3bvbzTJHn4uJc59kdOwbWcFR3MfFDRAFr1y49ob32Wk0EuePjj8Wt5TS95fx57S0sVCh31UkieqCPjNTl2fMy74kndu3SCT8jI/X9qlZNhwpMnKi9s3atnjh71jUBZ9Om1syhsHKlJp2uuUbkl1/8//xZyc3QLnekpYlMmqSvNzRU5Omn3X/szZs1eRsRoUMjfWnSJPn/qlJ2H1oSCBwOTSS0aqXva5kyIu+8Y5+Gux1duKDbnvP448vv4YULIoMH63O1aOH+8dEKp0/rJMjGaAXhokXWxvPNN3rcK1487x02nkpL0/mwBg3Shr1zAu4CBbRB+thjWjVm588zvzt7VueyiojQTjNfWLFCE7rVq3u/Ei0/OnNGVw4OCdE5Br/6KvA6O93FxA8RBbSNG7V3rE6dnIfXLF2qJ1AdO/q38ffXX9rALVfOs5Wk0tJc1UkNGliTxDh/XofDdeumiQzniWh4uCZWhg7VRvvOndZObOxwaJKlVi1tQDz7rLUN/Ph4/cwLF9aGhJVWrszb0C53JCZqYxZwTRyb3fPMnq3vTcWK/kuOjRun8d11l28qiy534YKW5o8cqXN4fP+9LvccyFVHDodOdumshqhQQWTCBA7pcpfD4dqnt2vnm7k69uzRoSCAJmIDJdG5bp1r2FOPHrqioD+lpOgE/YDIP/9pj4mXT5zQjp/nn9fKyEKFXMfgypV1CPOkSZpEt8MQvvwuJUXnSgwJ8X3ScO1a7Qy89lrtqCPPORz6OVWooOeNgwcH//xJTPwQUcBbsUITETfeePWS6E2bdNhPTIw1vdLbtmmCKibGvVUZkpJcFRr9+9ujYeVw6Lj1efO0QdG6tfY6OU9ES5bUIR8vvKCNQ2/NQeR87qQkrTaaM0erCx55RFd5q13bdUJcvnzeJv32piNHtAEWEiLywQf+f/70dJ2ryVtDu9yxfr2rVLpNG/3eXx7Tc8/p/c2b+38eqdGj5f9zPvmioZSerom2AQP0pBxwTeLqvISEiFSpoiu0DBigQ39mz9aKQLsOkUpP1+GTzpWNqlbVJKJdK//sbvp0naujdm3vDglduFC/dyVLWl85kxspKbo9FCyoFTfjxumwK1/3vh88KNKypX63Bw+27xw7KSl6DHzvPZE779SEvnO/UqyYnjO8/LJWDbH6zr/S00X69NHP4uOP/fOcmzbpyqxly3pnQYT85M8/XXPS1a/vm4U/7IiJHyIKCvPna4OqU6cre9T37dMDY6VK/u9FzGzpUo2xW7fsG53r12usERF6AmHnktO0NG3cT5umE0/HxGhVVeZeyZ49tcG9atXVE3MOh1ZsxcVpYmnMGJ3wt0sXLXnPnGByXiIjtWf2jjtEnnhCh6HZbeWF5GR9DYDIM8/4ryrKV0O73JGWpsvPRkXpd+Hxx3Vi7VOnRLp21Zgeesi6pMEbb2gM/fp57/PYvl2rzCpV0scuWlQnkF+2TBtrhw7pXDjTp4u89JI2EJo10yFSl3+vS5bU5FnPnvqdmTJFV+BLSPBPpVJmqala0Ve7tsZWs6aulhYoVSR2Fhurn3V0dN4bHSkpOjTXuZKa3eYX89Tu3VoR5dwmypXT/fyYMVrp4M19x7Jl+hkUKSIyc6b3HtcfHA79rL/4QhNW9eu7Es0hIXp8HDJEKzD377c62uDlcOhQIUCPL/60Y4duH6VKaQU8ZS81VfcjRYpo1fHo0fnreJZd4sfo/f7RuHFjiYuL89vzEVHw+fhjYOBA4J57gC++AEJCgJMngRYtgMOHgTVrgLp1rY1x0iRg6FDgySeBMWMuvU8E+OADYPhwoEIFYN48oFEja+LMi/Pngc2bgQ0bgJ9/1ktCgt4XEgLUqwc0bQoULgzs26f37dsHJCdf+jjFiwNVq+qlShXXpWpVoHJloEQJv76sXEtLA4YMAT76CLj7bmD6dCAiwnfPt2qVbgMnTgATJwIDBgDG+O75rub4ceD554EpU4AyZfTzTEgAJkwAHn7YmpicXn0VeOkl4MEHNb6QEM8f4/Bh4MsvdV+zZQtQoADQsSPQty/QrRtQpIh7j3P2rL4ve/YAe/e6rvfu1dtTU11/Gxqq3/3rrgOqVbvyulgxz19HVlJSgM8/B958U+OoX18/yx499HWSd+zcCdx6q36XZswA7rzT88c4eBDo1QtYuxZ45BFg7Fjf7l/8RQT49Vd9Xc7L3r16X3i4HhubN9dLs2ZAuXKePX56uu4HRo0C6tTR422tWt5/Hf52+jSwfr2e76xZo8fhc+f0vsqVgVatgJtu0kvNmtbuh4PFm28Czz0HDBsGvPee/9/TPXuAdu30fHfpUt0m6Eo//wwMGqTH6y5dgPff120iPzHGbBSRxlnex8QPEQUa5wF46FBg9GhtiK1dC/znP8DNN1sdnRo6VBNAU6YADz2kt507pwekmTO1ITBjBhAVZW2c3pSU5EoCOS8pKa6kTlbJnZIlrY3Zm0SAd94Bnn1Wk3q1armSWpkvZcrk/qTR4dDv/0svAdWrA3PmAA0aePd15EZcnCa+EhI0ptatrY5IvfSSNvwGDdKEqzvve3IysGCBJnt++EE/16ZNNdnTq5d+ft6Ung4cOnRpMihzgujEiUv/Pjo666TQddcBZcvmnOC6cAGYNg14+21NKDRpArzwAtC1a+6SY5SzpCSge3c9Tr31FjBihPv7gGXLgD599HObOlUTy8Hs6FFg3TpXIiguDrh4Ue+rUsWVBGreXJOVoaFXf5w+fXQb7tcPmDxZOyKCUVoasHUr8NNPwOrVeklM1Puio11JoFat9HjBxK5npk7V87jevfW8zar95P79wC23aBJ58WKgbVtr4rCj06e142LyZE0QT5gA3HFH/kx6MvFDREFFRKtp3ntPG9e//67JlN69rY7MJS1NKwKWLwe+/x6oWFF70uPjtSE6cmT+aGSJ5L8D79dfA199pUmQhARt9GVWuLAr8ZX5klMyLDERuPdebQjec49WF3mr+sMbRPR7HxZmdSQuIpokfust4NFHtToqq+9jaqq+r198ASxapI3satU02dOnD3D99f6P3enUqSuTQc7r/fs1GehUsKDGnVVSKDoa+PRTrUI8ehRo2RJ48UWgffv8t41a4e+/gfvv133Dgw9qIjK7bSU9XStVXn1Vq1jnzg2OahVPpaRodWnmqqDDh/W+woWBG25wJYJuvBEoVQr48UdNkJ08qQ3B+++39jX4mwiwa5crCfTjj1pxC+gxo0ULVzKoSRPdb1DWFi7UKr0OHfTYEB5ubTx//aX77N27tYKta1dr47GaiL4Pw4bpezNkCPDaa1p9nF8x8UNEQcfhAPr3196XN97QRIrdnDmjJ1gHDmi8ERHArFl60Kb84+xZ13A355A3588JCfo9yaxkySuTQkWLagLj+HHtyXroITbW3SUCPP008O67OsRy7Fh970SAX37RZM9XX2mCLipKG4x9+2oj0u7vcUqKJn+ySgrt2eMa/pFZ+/Za4dOqlf/jze8cDuDf/9aGSbt22mDJKtGbmKgdGfmhWsVTInpMXbvWVRm0ebMmygBN0u7Zo8nOuXO1Koi0us+ZBFq9GtixQ2+PiNCKRmdFUPPmeetQSE3VZLXzcvLkpT+fOaNVWoUKacKpUCH3fy5USBMv/tovr1qlFeUxMbotuju019eOH9e4Nm7UKs+6dXV4fd26rkugDJPPi4QE7dBZuhRo2FA7wxpnme7IX5j4IaKglJYGbN+upct2baD9+af2RlaurENgKlWyOiKyExE9Gc6cCMqcHNq3TysFAG3Q2GVoV6ARAR5/HBg/Xodhli6tCZ/du7Xh062bJns6dbK+R9dbRDSZ5UwGHTigQ2FvuMHqyGj6dE3eXn89sGSJJnedVq/WIYUnT+r8FA88YFmYAePcOR0S5kwEVayoQxntVBFpN8ePXzo0bONGTZ6FhGiiwzlPUFRU1gmcq/2cVbI5s5CQS6sUPWWMKxnkvC5SRPfp0dHZX5cu7X5F6pYtOmS5QgV9f0qVyn3MvnDmjA7Z/fVXTeLt2KFzLzpVrOhKBjmv69TxTfIqNRU4ckSHLB86pElG5/Xhw/qZFSt26aV4cfdui4i48vw+NVU7cF55RYctvvqqVvpcbdhnfsPEDxGRhf7+O+uDF1FOHA4dlnPwoJ64sdc/90T05HDyZN0W27TRZE+PHvmjd5TsJTZW56AID9f5Opo00WF4zz2nQ/TmzmWSl/zn7FmdMNpZFbR+vavT4XIlSgCRkVqt5rx29+fChXVffPGiDqm9cEGfJ/N1Trddfn9yMnDsmF6SkjQBdTUlSmSdEMp8W3g4cN99er1mTWB02Dkc2tG4fbsrEbR9O/Dbb645sgBNMmdOBtWrp0NIrzbc79y5K5M5l18fPaqfaWYREZp8Kl9ej7fJya7LmTP62bkjLOzKZFBionZodO+uFdCB8Pn4ExM/RERElO85HMB332mDmieLZLXff9eJ/o8c0eE2P/6o84lMm5a/56gg66WkAJs2aQM9cwKnWDH7Tw6dlqYVTc5E0OXXWd2WknLpY0RFaUVU7drWvAZvSU/XJIkzEeS83rlT3ydAq7CqV9dEUGTkpYmdU6eufMzISK2EqlBBkztZXUdFZd/ZmZamycbMyaDLk0OZf898m8Ohc/rcfrtv3rNAx8QPEREREZHNJCVpAyYuTuehGjKE1aFE/iSiSYjMiaAGDTSJEaxSU3Woc+Zk0I4dmljJnMDJKqnDymN7Y+KHiIiIiMiGUlO1QqFsWasjISKiQJZd4icfLCZMRERERGRPYWFM+hARkW8x8UNEREREREREFKSY+CEiIiIiIiIiClJM/BARERERERERBSkmfoiIiIiIiIiIghQTP0REREREREREQYqJHyIiIiIiIiKiIMXEDxERERERERFRkGLih4iIiIiIiIgoSDHxQ0REREREREQUpJj4ISIiIiIiIiIKUkZE/PdkxiQB+NNvT+hbpQEcszoIogDEbYcod7jtEOUOtx2i3OG2Q5Q7Vm07lUUkOqs7/Jr4CSbGmDgRaWx1HESBhtsOUe5w2yHKHW47RLnDbYcod+y47XCoFxERERERERFRkGLih4iIiIiIiIgoSDHxk3tTrA6AKEBx2yHKHW47RLnDbYcod7jtEOWO7bYdzvFDRERERERERBSkWPFDRERERERERBSkmPjxkDGmkzFmpzHmD2PMs1bHQ2RnxphPjDGJxpjtmW6LMsYsN8bszriOtDJGIrsxxlQyxsQaY+KNMTuMMcMybue2Q5QNY0xBY8zPxpitGdvOKxm3VzXGbMg4d5ttjAm3OlYiOzLGFDDGbDbGLMn4ndsOUQ6MMfuMMduMMVuMMXEZt9nunI2JHw8YYwoAeB9AZwB1ANxjjKljbVREtjYdQKfLbnsWwA8iUgPADxm/E5FLGoAnRaQOgBsBPJpxrOG2Q5S9iwDaikgDADEAOhljbgTwNoD3RKQ6gJMAHrQwRiI7Gwbgt0y/c9shcs/NIhKTaQl3252zMfHjmaYA/hCRvSKSAuArALdbHBORbYnIjwBOXHbz7QA+y/j5MwDd/RoUkc2JyBER2ZTxczL0JLwCuO0QZUvU2YxfwzIuAqAtgHkZt3PbIcqCMaYigC4Apmb8bsBthyi3bHfOxsSPZyoAOJDp94MZtxGR+64RkSMZP/8F4BorgyGyM2NMFQD/BLAB3HaIcpQxVGULgEQAywHsAXBKRNIy/oTnbkRZGwdgBABHxu+lwG2HyB0CYJkxZqMxZmDGbbY7Zwu1OgAiyr9ERIwxXFqQKAvGmKIA5gMYLiJntPNVcdshypqIpAOIMcaUBLAQQC2LQyKyPWNMVwCJIrLRGNPG6niIAkxLETlkjCkDYLkx5vfMd9rlnI159Y7LAAABy0lEQVQVP545BKBSpt8rZtxGRO47aowpBwAZ14kWx0NkO8aYMGjSZ6aILMi4mdsOkZtE5BSAWADNAJQ0xjg7O3nuRnSlFgC6GWP2QaeyaAtgPLjtEOVIRA5lXCdCOxyawobnbEz8eOYXADUyZrgPB3A3gMUWx0QUaBYD6Jfxcz8AiyyMhch2MuZVmAbgNxEZm+kubjtE2TDGRGdU+sAYUwhAe+gcWbEA7sz4M247RJcRkZEiUlFEqkDbNytEpA+47RBlyxhTxBhTzPkzgA4AtsOG52xGxPKqo4BijLkVOga2AIBPROR1i0Misi1jzJcA2gAoDeAogH8D+BrAHADXAvgTwF0icvkE0ET5ljGmJYDVALbBNdfCc9B5frjtEF2FMaY+dBLNAtDOzTkiMsoYUw1axRAFYDOAviJy0bpIiewrY6jXUyLSldsOUfYytpGFGb+GApglIq8bY0rBZudsTPwQEREREREREQUpDvUiIiIiIiIiIgpSTPwQEREREREREQUpJn6IiIiIiIiIiIIUEz9EREREREREREGKiR8iIiIiIiIioiDFxA8RERERERERUZBi4oeIiIiIiIiIKEgx8UNEREREREREFKT+BzIrUaNq/dWuAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "id": "7b0ceb88",
      "metadata": {
        "id": "7b0ceb88"
      },
      "source": [
        "# One fully-connected layer after the LSTM"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6f30c3bf",
      "metadata": {
        "id": "6f30c3bf"
      },
      "outputs": [],
      "source": [
        "from torch import nn\n",
        "\n",
        "\n",
        "class DenseLSTM(nn.Module):\n",
        "    def __init__(self, input_dim, hidden_dim, lstm_layers=1, bidirectional=False, dense=False):\n",
        "        super(DenseLSTM, self).__init__()\n",
        "        self.input_dim = input_dim\n",
        "        self.hidden_dim = hidden_dim\n",
        "        self.layers = lstm_layers\n",
        "        self.bidirectional = bidirectional\n",
        "        self.dense = dense\n",
        "        # define the LSTM layer\n",
        "        self.lstm = nn.LSTM(input_size=self.input_dim, \n",
        "                            hidden_size=self.hidden_dim, \n",
        "                            num_layers=self.layers,\n",
        "                            bidirectional=self.bidirectional)\n",
        "        self.act1 = nn.ReLU()\n",
        "        # change linear layer inputs depending on if lstm is bidrectional\n",
        "        if not bidirectional:\n",
        "            self.linear = nn.Linear(self.hidden_dim, self.hidden_dim)\n",
        "        else:\n",
        "            self.linear = nn.Linear(self.hidden_dim * 2, self.hidden_dim)\n",
        "        self.act2 = nn.ReLU()\n",
        "        # change linear layer inputs depending on if lstm is bidrectional and extra dense layer isn't added\n",
        "        if bidirectional and not dense:\n",
        "            self.final = nn.Linear(self.hidden_dim * 2, 1)\n",
        "        else:\n",
        "            self.final = nn.Linear(self.hidden_dim, 1)\n",
        "\n",
        "\n",
        "    def forward(self, inputs, labels=None):\n",
        "        out = inputs.unsqueeze(1)\n",
        "        out, h = self.lstm(out)\n",
        "        out = self.act1(out)\n",
        "        if self.dense:\n",
        "            out = self.linear(out)\n",
        "            out = self.act2(out)\n",
        "        out = self.final(out)\n",
        "        return out"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6840d1a2",
      "metadata": {
        "id": "6840d1a2"
      },
      "outputs": [],
      "source": [
        "import time\n",
        "\n",
        "def fit(model, optimizer, criterion):\n",
        "    print(\"{:<8} {:<25} {:<25} {:<25}\".format('Epoch', \n",
        "                                          'Train Loss', \n",
        "                                          'Test Loss', \n",
        "                                          'Time (seconds)'))\n",
        "    for epoch in range(epochs):\n",
        "        model.train()\n",
        "        start = time.time()\n",
        "        epoch_loss = []\n",
        "        # for batch in train data\n",
        "        for step, batch in enumerate(train_dataloader):\n",
        "            # make gradient zero to avoid accumulation\n",
        "            model.zero_grad()\n",
        "            batch = tuple(t.to(device) for t in batch)\n",
        "            inputs, labels = batch\n",
        "            print(inputs.shape)\n",
        "            # get predictions\n",
        "            out = model(inputs)\n",
        "            out.to(device)\n",
        "            # get loss\n",
        "            loss = criterion(out, labels)\n",
        "            epoch_loss.append(loss.float().detach().cpu().numpy().mean())\n",
        "            # backpropagate\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "        test_epoch_loss = []\n",
        "        end = time.time()\n",
        "        model.eval()\n",
        "        # for batch in validation data\n",
        "        for step, batch in enumerate(test_dataloader):\n",
        "            batch = tuple(t.to(device) for t in batch)\n",
        "            inputs, labels = batch\n",
        "            # get predictions\n",
        "            out = model(inputs)\n",
        "            # get loss\n",
        "            loss = criterion(out, labels)     \n",
        "            test_epoch_loss.append(loss.float().detach().cpu().numpy().mean())\n",
        "        print(\"{:<8} {:<25} {:<25} {:<25}\".format(epoch+1, \n",
        "                                                  np.mean(epoch_loss), \n",
        "                                                  np.mean(test_epoch_loss), \n",
        "                                                  end-start))\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "069e91ca",
      "metadata": {
        "id": "069e91ca"
      },
      "outputs": [],
      "source": [
        "hidden_dim = 32\n",
        "epochs = 5\n",
        "\n",
        "model = DenseLSTM(18, hidden_dim, lstm_layers=2, bidirectional=True, dense=True)\n",
        "model.to(device)\n",
        "\n",
        "optimizer = torch.optim.Adam(model.parameters())\n",
        "criterion = nn.MSELoss()\n",
        "\n",
        "fit(model, optimizer, criterion)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "512122d1",
      "metadata": {
        "id": "512122d1"
      },
      "source": [
        "# 평가"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1e971353",
      "metadata": {
        "id": "1e971353"
      },
      "outputs": [],
      "source": [
        "model.eval()\n",
        "test_predict = model(testX)\n",
        "numpy_test = test_predict.cpu().detach().numpy()\n",
        "\n",
        "# data_predict = test_predict.data.numpy()\n",
        "# dataY_plot = testY.data.numpy()\n",
        "\n",
        "# data_predict = sc_t.inverse_transform(data_predict)\n",
        "# dataY_plot = sc_t.inverse_transform(dataY_plot)\n",
        "\n",
        "# plt.axvline(x=train_size, c='r', linestyle='--')\n",
        "\n",
        "# plt.plot(dataY_plot)\n",
        "# plt.plot(data_predict)\n",
        "# plt.suptitle('Time-Series Prediction')\n",
        "# plt.show()\n",
        "numpy_test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d5e21fe5",
      "metadata": {
        "id": "d5e21fe5"
      },
      "outputs": [],
      "source": [
        "data_predict = sc_t.inverse_transform(numpy_test)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data_predict"
      ],
      "metadata": {
        "id": "KJDZMTtfxlJx"
      },
      "id": "KJDZMTtfxlJx",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "de2b8eba",
      "metadata": {
        "id": "de2b8eba"
      },
      "outputs": [],
      "source": [
        "path = '/content/drive/MyDrive/Colab_Notebooks/data/test/meta/'\n",
        "test = pd.DataFrame()\n",
        "\n",
        "file_list = os.listdir(path)\n",
        "\n",
        "for l in file_list:\n",
        "    data = pd.read_csv(path + l)\n",
        "    test = pd.concat([test, data])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6baaf4b9",
      "metadata": {
        "id": "6baaf4b9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "378b1c01-1a79-4802-cb3d-1b8c20504f8b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(662400, 6)\n"
          ]
        }
      ],
      "source": [
        "test = test[['내부온도관측치', '내부습도관측치', 'CO2관측치', '최근분무량','냉방온도', '난방온도']]\n",
        "test = test.fillna(method='bfill')\n",
        "test = test.fillna('0')\n",
        "print(test.shape)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test"
      ],
      "metadata": {
        "id": "emQ-Vv4JDV22"
      },
      "id": "emQ-Vv4JDV22",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "40175df0",
      "metadata": {
        "id": "40175df0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a9d35d1a-e7a6-4774-e462-b2b2cf7c972d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(662400, 6)\n"
          ]
        }
      ],
      "source": [
        "test_dataset = test.to_numpy()\n",
        "test_dataset = test_dataset[:, :]\n",
        "print(test_dataset.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "de371238",
      "metadata": {
        "id": "de371238",
        "outputId": "cc88915d-61b8-4b36-e571-28cba687c0e7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[24.89999961853028, 21.600000381469727, 47.59999847412109, ...,\n",
              "        309.41, 16.548000000000002, 29.7635],\n",
              "       [24.799999237060547, 21.600000381469727, 48.400001525878906, ...,\n",
              "        309.41, 16.548000000000002, 29.7635],\n",
              "       [24.5, 21.700000762939453, 49.400001525878906, ..., 309.41,\n",
              "        16.548000000000002, 29.7635],\n",
              "       ...,\n",
              "       [25.6000003814697, 25.8999996185302, 83.6999969482421, ..., 0.0,\n",
              "        0.0, 0.0],\n",
              "       [25.5, 25.8999996185302, 83.6999969482421, ..., 0.0, 0.0, 0.0],\n",
              "       [25.5, 25.8999996185302, 83.6999969482421, ..., 0.0, 0.0, 0.0]],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 57
        }
      ],
      "source": [
        "test_dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3ed2e1b3",
      "metadata": {
        "id": "3ed2e1b3",
        "outputId": "94d89f92-a1a0-4893-e56b-4a57469ffec6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(662400, 6)"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ],
      "source": [
        "len(test_dataset)/1440\n",
        "test_dataset.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e3b1423d",
      "metadata": {
        "id": "e3b1423d"
      },
      "outputs": [],
      "source": [
        "def test_windows(data):\n",
        "    x = []\n",
        "\n",
        "    for i in range(int(len(data)/1440)):\n",
        "        _x = data[(i * 1440): ((i + 1) * 1440)]\n",
        "        x.append(_x)\n",
        "\n",
        "    return np.array(x)\n",
        "\n",
        "sc_real = MinMaxScaler()\n",
        "testing_data = sc_df.fit_transform(test_dataset)\n",
        "\n",
        "t = test_windows(testing_data)\n",
        "\n",
        "test_T = Variable(torch.Tensor(np.array(t))).to(device)\n",
        "test_set = DataLoader(test_T, batch_size=20, shuffle=False)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def predict(model, test_loader, device):\n",
        "    model.eval()\n",
        "    model_pred = []\n",
        "    with torch.no_grad():\n",
        "        for sample in tqdm(iter(test_loader)):\n",
        "            meta = sample.float().to(device)\n",
        "\n",
        "            pred_logit = model(sample)\n",
        "            pred_logit = pred_logit.detach().cpu()\n",
        "\n",
        "            model_pred.extend(pred_logit.tolist())\n",
        "    return model_pred"
      ],
      "metadata": {
        "id": "Fq2DoigqDmz7"
      },
      "id": "Fq2DoigqDmz7",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e37d8cae",
      "metadata": {
        "id": "e37d8cae"
      },
      "outputs": [],
      "source": [
        "checkpoint = torch.load('/content/drive/MyDrive/Colab_Notebooks/competition/meta_LSTM.pth')\n",
        "model = LSTM(6, 512, 1, bidirectional=True).to(device)\n",
        "\n",
        "test_predict = predict(model, test_set, device)\n",
        "\n",
        "\n",
        "test_predict"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_predict2 = np.array(test_predict)\n",
        "test_predict2.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bFYPOOH-Ykry",
        "outputId": "fb989624-fb00-4312-8575-270fbaae25a6"
      },
      "id": "bFYPOOH-Ykry",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(460, 1)"
            ]
          },
          "metadata": {},
          "execution_count": 111
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "out = sc_t.inverse_transform(test_predict)"
      ],
      "metadata": {
        "id": "4Zph9914HN9G"
      },
      "id": "4Zph9914HN9G",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "out"
      ],
      "metadata": {
        "id": "g5sapbqhHiGR"
      },
      "id": "g5sapbqhHiGR",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "140342c4",
      "metadata": {
        "id": "140342c4"
      },
      "outputs": [],
      "source": [
        "submission = pd.read_csv('./data/open/sample_submission.csv')\n",
        "submission['leaf_weight'] = data_predict\n",
        "submission.to_csv('./submit_lstm2.csv', index=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b2d0c494",
      "metadata": {
        "id": "b2d0c494"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "colab": {
      "name": "LSTM.ipynb",
      "provenance": [],
      "machine_shape": "hm",
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}