vilbert/vilbert-multi-task-master/ME_train_reg_test-set.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.utils.data as Data\n",
    "from torch.autograd import Variable\n",
    "from statistics import mean\n",
    "import matplotlib.pyplot as plt\n",
    "import _pickle as cPickle\n",
    "from tqdm import tqdm\n",
    "from scipy.stats import spearmanr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, inp, target):\n",
    "    loss_func = torch.nn.MSELoss()\n",
    "    torch_dataset_val = Data.TensorDataset(inp, target)\n",
    "\n",
    "    loader_val = Data.DataLoader(\n",
    "        dataset=torch_dataset_val, \n",
    "        batch_size=VAL_BATCH_SIZE, \n",
    "        shuffle=False\n",
    "    )\n",
    "\n",
    "    dataiter_val = iter(loader_val)\n",
    "    in_, out_ = dataiter_val.next()\n",
    "\n",
    "    model.eval()\n",
    "    pred_scores = model(in_)\n",
    "\n",
    "    loss = loss_func(pred_scores, out_)\n",
    "    model.train()\n",
    "    r_1, _ = spearmanr(\n",
    "        pred_scores.cpu().detach().numpy()[:,0], \n",
    "        out_.cpu().detach().numpy()[:,0], \n",
    "        axis=0\n",
    "    )\n",
    "\n",
    "    r_2, _ = spearmanr(\n",
    "        pred_scores.cpu().detach().numpy()[:,1], \n",
    "        out_.cpu().detach().numpy()[:,1], \n",
    "        axis=0\n",
    "    )\n",
    "    \n",
    "    return r_1, r_2, loss.item()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_save_path = \"datasets/ME/out_features/train_features_NLVR2.pkl\"\n",
    "pooled_dict = cPickle.load(open(train_save_path, 'rb'))\n",
    "pooled_output_mul = pooled_dict[\"pooled_output_mul\"]\n",
    "pooled_output_sum = pooled_dict[\"pooled_output_sum\"]\n",
    "pooled_output_t = pooled_dict[\"pooled_output_t\"]\n",
    "pooled_output_v = pooled_dict[\"pooled_output_v\"]\n",
    "concat_pooled_output = pooled_dict[\"concat_pooled_output\"]\n",
    "targets = pooled_dict[\"targets\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_save_path = \"datasets/ME/out_features/test_features_NLVR2.pkl\"\n",
    "pooled_dict_test = cPickle.load(open(test_save_path, 'rb'))\n",
    "pooled_output_mul_test = pooled_dict_test[\"pooled_output_mul\"]\n",
    "pooled_output_sum_test = pooled_dict_test[\"pooled_output_sum\"]\n",
    "pooled_output_t_test = pooled_dict_test[\"pooled_output_t\"]\n",
    "pooled_output_v_test = pooled_dict_test[\"pooled_output_v\"]\n",
    "concat_pooled_output_test = pooled_dict_test[\"concat_pooled_output\"]\n",
    "targets_test = pooled_dict_test[\"targets\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SigLinNet(nn.Module):\n",
    "    def __init__(self, input_size, \n",
    "                 hidden_size_1,\n",
    "                 hidden_size_2, \n",
    "                 hidden_size_3, \n",
    "                 num_scores):\n",
    "        super(SigLinNet, self).__init__()\n",
    "        self.out = nn.Sequential(\n",
    "            nn.Linear(input_size, hidden_size_1),\n",
    "            nn.Sigmoid(),\n",
    "            nn.Linear(hidden_size_1, hidden_size_2),\n",
    "            nn.Sigmoid(),\n",
    "            nn.Linear(hidden_size_2, hidden_size_3),\n",
    "            nn.Sigmoid(),\n",
    "            nn.Linear(hidden_size_3, num_scores),\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.out(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128\n",
    "VAL_BATCH_SIZE = 2000\n",
    "EPOCH = 100\n",
    "lr = 4e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:15<00:00,  6.48it/s]\n"
     ]
    }
   ],
   "source": [
    "torch.manual_seed(42)\n",
    "\n",
    "torch_dataset = Data.TensorDataset(concat_pooled_output, targets)\n",
    "\n",
    "loader = Data.DataLoader(\n",
    "    dataset=torch_dataset, \n",
    "    batch_size=BATCH_SIZE, \n",
    "    shuffle=True\n",
    ")\n",
    "\n",
    "net = SigLinNet(1024*2, 512, 64, 32, 2)\n",
    "net.cuda()\n",
    "\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr=lr, weight_decay=1e-4)\n",
    "loss_func = torch.nn.MSELoss()\n",
    "\n",
    "summary = {\n",
    "    \"losses\" : [],\n",
    "    \"r1s\" : [],\n",
    "    \"r2s\" : [],\n",
    "    \"eval_losses\" : []\n",
    "}\n",
    "\n",
    "net.train()\n",
    "for _ in tqdm(range(EPOCH)):\n",
    "    errors = []\n",
    "    # r1s, r2s, ls = list(), list(), list()\n",
    "    for step, (batch_in, batch_out) in enumerate(loader):\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        b_in = Variable(batch_in)\n",
    "        b_out = Variable(batch_out)\n",
    "\n",
    "        prediction = net(b_in)\n",
    "\n",
    "        loss = loss_func(prediction, b_out)\n",
    "        errors.append(loss.item())\n",
    "\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "    \"\"\"r1, r2, _ = evaluate(net, pooled_output_mul_test, targets_test)\n",
    "    summary[\"r1s\"].append(r1)\n",
    "    summary[\"r2s\"].append(r2)\"\"\"\n",
    "    summary[\"losses\"].append(mean(errors))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f98a8567400>]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAD4CAYAAADLhBA1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy86wFpkAAAACXBIWXMAAAsTAAALEwEAmpwYAAA8MklEQVR4nO3deXxU1f3/8ddnsu87ISSEJBAgQWQRcEFwAcUda9W6fNVarV3028Wvbe1iF/ttq/221Vr9fq1WW6t1+6FVVFwBd9kRJBAghC0hZAMSsm+f3x9zEyYbmZAJA+TzfDzmwcy5Z+6c6+C8ueece66oKsYYY8xAufzdAGOMMScGCxRjjDE+YYFijDHGJyxQjDHG+IQFijHGGJ8I9HcDBlNiYqJmZGT4uxnGGHNcWb16dYWqJvX3fSd0oGRkZLBq1Sp/N8MYY44rIrLzSN5nXV7GGGN8wgLFGGOMT1igGGOM8QkLFGOMMT5hgWKMMcYnLFCMMcb4hAWKMcYYn7BA6UFdUwv3vZnP7n11/m6KMcYcNyxQenCgrpmnP9vBT1/ZgN0vxhhjvGOB0oMRsWH8YN44PtxSzsJ1e/zdHGOMOS5YoPTihtMzmDwylntf28j+2iZ/N8cYY455Fii9CHAJv7tiIlX1zfxm0SZ/N8cYY455FiiHkZMSzW2zs1iwuohPCyr83RxjjDmmWaD04TtzshkWFcK/lu/yd1OMMeaYZoHSh9CgAE7NSmDNrv3+booxxhzTLFC8MDU9lpKqBkqq6v3dFGOMOWZ5FSgicoGIbBaRAhG5u4ftISLygrN9uYhkOOUJIrJURGpE5OEu73lLRNaJSJ6IPCoiAU75CyLyufPYISKfO+UZIlLvse3RgR68t6amxwGwZueBo/WRxhhz3Onzjo3OD/0jwHlAEbBSRBaq6kaParcA+1V1jIhcA9wPfAVoAO4BTnIenq5W1WoREWABcBXwvKp+xeOz/whUebxnm6pO7ucxDlhOSjQhgS7W7NrPxSenHO2PN8aY44I3ZygzgAJVLVTVJuB5YH6XOvOBp5znC4A5IiKqWquqH+MOlk5Utdp5GggEA50uSXeC5mrgOW8PZrAEB7qYmBpj4yjGGHMY3gRKKrDb43WRU9ZjHVVtwX1WkdDXjkXkbaAMOIg7iDzNAkpVdatHWaaIrBWRD0RkVi/7vE1EVonIqvLy8r6a4LWpo+LIK66msaXVZ/s0xpgTiV8H5VV1HpAChADndtl8LZ3PTkqAdFWdAtwJPCsi0T3s8zFVnaaq05KSknzW1qnpsTS1tpG3p7rvysYYMwR5EyjFwEiP12lOWY91RCQQiAEqvWmAqjYAr+LRjebs4wrgBY96japa6TxfDWwDxnrzGb5waGDeur2MMaYn3gTKSiBbRDJFJBi4BljYpc5C4Cbn+ZXAEj3MMr0iEikiKc7zQOBiIN+jylwgX1WLPN6T5DETLAvIBgq9aL9PDIsOJTU2jLW7DhytjzTGmONKn7O8VLVFRO4A3gYCgCdVNU9E7gVWqepC4AngaREpAPbhDh0ARGQHEA0Ei8jlwPm4z14WikgI7lBbCnhOA76G7oPxs4F7RaQZaAO+qar7+n/IR27qqDhW7TiqH2mMMceNPgMFQFUXAYu6lP3c43kD7mm/Pb03o5fdTj/M5321h7KXgJf6bu3gmZoey2vr9lBSVU9KTJg/m2KMMcccu1K+H6bYBY7GGNMrC5R+yPW4wNEYY0xnFij9EBzoYsKIaL4oruq7sjHGDDEWKP2UGhdO+cFGfzfDGGOOORYo/ZQUGWKBYowxPbBA6aekqBBqGluoa2rxd1OMMeaYYoHST0lRIQBUHGzyc0uMMebYYoHST+2BUl7TbQFlY4wZ0ixQ+ikp0h0oZdU2jmKMMZ4sUPrp0BmKBYoxxniyQOmn+IhgXILN9DLGmC4sUPopwCUk2NRhY4zpxgLlCNi1KMYY050FyhFIigqxMRRjjOnCAuUIDIuyMxRjjOnKAuUIJEWFUFHTSFtbrzelNMaYIccC5QgkRYXQ3KpU1Tf7uynGGHPMsEA5AnYtijHGdGeBcgTar5a3cRRjjDnEq0ARkQtEZLOIFIjI3T1sDxGRF5zty0UkwylPEJGlIlIjIg93ec9bIrJORPJE5FERCXDKfykixSLyufO4yOM9P3Y+Y7OIzBvQkQ9AxxmKBYoxxnToM1CcH/pHgAuBXOBaEcntUu0WYL+qjgEeAO53yhuAe4C7etj11ao6CTgJSAKu8tj2gKpOdh6LnHbkAtcAE4ALgP9tD6GjzQLFGGO68+YMZQZQoKqFqtoEPA/M71JnPvCU83wBMEdERFVrVfVj3MHSiapWO08DgWCgrylT84HnVbVRVbcDBU7bjrrIkEBCg1w2hmKMMR68CZRUYLfH6yKnrMc6qtoCVAEJfe1YRN4GyoCDuIOo3R0isl5EnhSRuH6046gQEffFjXaGYowxHfw6KK+q84AUIAQ41yn+P2A0MBkoAf7Yn32KyG0iskpEVpWXl/uwtZ0lRYZQdtDuiWKMMe28CZRiYKTH6zSnrMc6IhIIxACV3jRAVRuAV3G60VS1VFVbVbUNeJxD3VretANVfUxVp6nqtKSkJG+acETsDMUYYzrzJlBWAtkikikiwbgHxhd2qbMQuMl5fiWwRFV7HRMRkUgRSXGeBwIXA/nO6xSPql8CNnh8xjXOjLJMIBtY4UX7B4UFijHGdBbYVwVVbRGRO4C3gQDgSVXNE5F7gVWquhB4AnhaRAqAfbhDBwAR2QFEA8EicjlwPu6zl4UiEoI71JYCjzpv+b2ITMY9SL8D+IbTjjwReRHYCLQAt6tq64COfgCSIkPZX9dMU0sbwYF2OY8xxvQZKADO1N1FXcp+7vG8gc7Tfj3rZfSy2+m91L/hMO34DfCbPpp7VAyLdk8drqxtJCUmzM+tMcYY/7N/Wh8hu1reGGM6s0A5QnZxozHGdGaBcoQsUIwxpjMLlCOUEBkMWKAYY0w7C5QjFBIYQGx4kC2/YowxDguUAUiKtGtRjDGmnQXKANjFjcYYc4gFygAkRYVYl5cxxjgsUAYg0bq8jDGmgwXKAMRHBFPX1EpDs99WgDHGmGOGBcoAxEe4pw7vr2vyc0uMMcb/LFAGoD1QKmssUIwxxgJlAOwMxRhjDrFAGYD2QNlXa4FijDEWKAMQH26BYowx7SxQBiAmLAiXWKAYYwxYoAyIyyXEhQdboBhjDBYoAxYfYYFijDFggTJgcRYoxhgDWKAMWIIFijHGAF4GiohcICKbRaRARO7uYXuIiLzgbF8uIhlOeYKILBWRGhF5uMt73hKRdSKSJyKPikiAU/4/IpIvIutF5N8iEuuUZ4hIvYh87jweHejB+0JcRLBdh2KMMXgRKM4P/SPAhUAucK2I5HapdguwX1XHAA8A9zvlDcA9wF097PpqVZ0EnAQkAVc55e8CJ6nqycAW4Mce79mmqpOdxze9OcDBlhARzP66Ztra1N9NMcYYv/LmDGUGUKCqharaBDwPzO9SZz7wlPN8ATBHRERVa1X1Y9zB0omqVjtPA4FgQJ3yd1S1xdm2DEjrzwEdbXHhwbS2KdUNzf5uijHG+JU3gZIK7PZ4XeSU9VjHCYMqIKGvHYvI20AZcBB3EHX1NeBNj9eZIrJWRD4QkVm97PM2EVklIqvKy8v7asKAdaznZeMoxpghzq+D8qo6D0gBQoBzPbeJyE+BFuBfTlEJkK6qU4A7gWdFJLqHfT6mqtNUdVpSUtKgth881vOyQDHGDHHeBEoxMNLjdZpT1mMdEQkEYoBKbxqgqg3Aq3h0o4nIV4FLgOtVtb0rrFFVK53nq4FtwFhvPmMw2Xpexhjj5k2grASyRSRTRIKBa4CFXeosBG5ynl8JLGkPgp6ISKSIpDjPA4GLgXzn9QXAD4HLVLXO4z1JHjPBsoBsoNCL9g8qCxRjjHEL7KuCqraIyB3A20AA8KSq5onIvcAqVV0IPAE8LSIFwD7coQOAiOwAooFgEbkcOB/32ctCEQnBHWpLgfZpwA/j7gJ7V0QAljkzumYD94pIM9AGfFNV9w3w+AesI1Bs6rAxZojrM1AAVHURsKhL2c89njdwaNpv1/dm9LLb6b3UH9NL+UvAS14096gKDQogPDiAfXaTLWPMEGdXyvtAXHiwnaEYY4Y8CxQfSIi05VeMMcYCxQfiwoNt2rAxZsizQPGBhIhgu7DRGDPkWaD4QFyEnaEYY4wFig/ERwRT29RKQ3Orv5tijDF+Y4HiAx3Lr9hML2PMEGaB4gMdC0TatSjGmCHMAsUH7AzFGGMsUHzC1vMyxhgLFJ+ID7dAMcYYCxQfiAkLwiUWKMaYoc0CxQdcLnGv52WBYowZwixQfCQuwgLFGDO0WaD4SLwFijFmiLNA8ZH48GCbNmyMGdIsUHwk3pawN8YMcRYoPuI+Q2mmrU393RRjjPELCxQfiY8IprVNqapv9ndTjDHGL7wKFBG5QEQ2i0iBiNzdw/YQEXnB2b5cRDKc8gQRWSoiNSLycJf3vCUi60QkT0QeFZEApzxeRN4Vka3On3FOuYjIQ85nrBeRqQM+eh9KjQsDYPf+Oj+3xBhj/KPPQHF+6B8BLgRygWtFJLdLtVuA/ao6BngAuN8pbwDuAe7qYddXq+ok4CQgCbjKKb8bWKyq2cBi5zXO52c7j9uA//PmAI+W0UmRAGwrr/FzS4wxxj+8OUOZARSoaqGqNgHPA/O71JkPPOU8XwDMERFR1VpV/Rh3sHSiqtXO00AgGGgffPDc11PA5R7l/1S3ZUCsiKR40f6jYlRCOIEuoaDMAsUYMzR5EyipwG6P10VOWY91VLUFqAIS+tqxiLwNlAEHcQcRQLKqljjP9wLJ/WgHInKbiKwSkVXl5eV9NcFnggJcjEoIZ1tZ7VH7TGOMOZb4dVBeVecBKUAIcG4P25VDZy7e7vMxVZ2mqtOSkpJ801AvjU6KtC4vY8yQ5U2gFAMjPV6nOWU91hGRQCAGqPSmAaraALzKoW600vauLOfPsn60w69GD4tkR2UtLa1t/m6KMcYcdd4EykogW0QyRSQYuAZY2KXOQuAm5/mVwBLn7KJHIhLpERqBwMVAfg/7ugl32LSX3+jM9joNqPLoGjsmjE6KpLlV2b2/3t9NMcaYoy6wrwqq2iIidwBvAwHAk6qaJyL3AqtUdSHwBPC0iBQA+3CHDgAisgOIBoJF5HLgfNxnLwtFJAR3qC0FHnXech/woojcAuwErnbKFwEXAQVAHXDzAI57UIxOigBgW1kNmYkRfm6NMcYcXX0GCoCqLsL9g+5Z9nOP5w0cmvbb9b0Zvex2ei/1K4E5PZQrcLs37fWX0cPcU4cLymuY2zGXwBhjhga7Ut6HokODGBYVwjabOmyMGYIsUHzMZnoZY4YqCxQfGz0sgm3ltRxmToIxxpyQLFB8bHRSJFX1zVTaUvbGmCHGAsXHOtb0snEUY8wQY4HiY54zvYwxZiixQPGxlOhQwoMDbE0vY8yQY4HiYy6XkJUUYTO9jDFDjgXKILCpw8aYocgCZRCMToqk+EA99U2t/m6KMcYcNRYog2B0UiSqdvdGY8zQYoEyCE4ZFUdokIsH3t1iFzgaY4YMC5RBMDwmlB/MG8/i/DJeXnNM3bLFGGMGjQXKILn5jAymZ8Txq9fyKK1u8HdzjDFm0FmgDBKXS/j9lZNoam3jxy9/YV1fxpgTngXKIMpMjOAH88azJL+MNzfs9XdzjDFmUFmgDLKbz8ggJSaUV9baWIox5sRmgTLIXC7h/NxkPtxabtelGGNOaBYoR8H5E4bT0NzGR1vLvapfUFbDnS98TnVD8yC3zBhjfMerQBGRC0Rks4gUiMjdPWwPEZEXnO3LRSTDKU8QkaUiUiMiD3vUDxeRN0QkX0TyROQ+j20PiMjnzmOLiBzw2NbqsW3hQA78aJqRGU90aCBv55V6Vf+Flbt4eW0xv3w1b5BbZowxvtNnoIhIAPAIcCGQC1wrIrldqt0C7FfVMcADwP1OeQNwD3BXD7v+g6qOB6YAM0XkQgBV/b6qTlbVycBfgJc93lPfvk1VL/P2IP0tKMDFnJxkFueX0tLa1mf9D7dUEBzo4uW1xbyxvuQotNAYYwbOmzOUGUCBqhaqahPwPDC/S535wFPO8wXAHBERVa1V1Y9xB0sHVa1T1aXO8yZgDZDWw2dfCzzn9dEcw87PTeZAXTMrd+w/bL2Sqno2lx7ku3OymTQylp/8+wv2Vtl1LMaYY583gZIK7PZ4XeSU9VhHVVuAKiDBmwaISCxwKbC4S/koIBNY4lEcKiKrRGSZiFzey/5uc+qsKi/3bsziaDhrXBIhgS7e2eiePtzWpvxl8VYe/7CwU70Pt7jbPDcnmQe/MpmmljZ+sGAdbW12HYsx5tjm10F5EQnEfQbykKoWdtl8DbBAVT2nRo1S1WnAdcCDIjK66z5V9TFVnaaq05KSkgat7f0VHhzIrOxE3slzd3v96KX1/PHdLfz+7XzKPK6k/2BLOcOjQxmbHElmYgQ/uySHj7ZWsGB1kR9bb4wxffMmUIqBkR6v05yyHus4IREDVHqx78eArar6YA/brqFLd5eqFjt/FgLv4x5/OW6cnzuc4gP1XPv4Mv7f6iKuOzWdljblmeW7AGhpbeOjrRXMHpuIiABw3Yx0pqbH8vu3N1PT2DLgNuytarCr9o0xg8KbQFkJZItIpogE4/6h7zrDaiFwk/P8SmCJ9vGrJSL/jTt4vtfDtvFAHPCZR1mciIQ4zxOBmcBGL9p/zJiTMwyXwMod+/nZxTn89ksTOXfcMJ5dvpPGllbWFR3gYEMLZ40d1vEeEeHnl06goqaR/3u/YECfv3nvQU6/bzEfbDl2ugKNMSeOPgPFGRO5A3gb2AS8qKp5InKviLTPtHoCSBCRAuBOoGNqsYjsAP4EfFVEikQkV0TSgJ/injW2xpkGfKvHx14DPN8llHKAVSKyDlgK3Keqx1WgJESG8JOLcnjo2incOisLgJtnZlJR08Tr60r4YHM5LoEzxyR2et/kkbF8aUoqj3+0nd376o748z/aWo4qrNi+b0DHYYwxPZETuftj2rRpumrVKn8347BUlfMf+JCQIBcuEYICXLz0rTO61SupquecP7zP3JxkHr5u6hF91tf/uYp3N5Zy5phEnrn11IE23RhzghKR1c54db/YlfJ+JiJ8dWYGG4qrWV9UxezsnicSpMSE8Y3Zo3l9fQkL1+3p9zhIW5uycof7zGR90QEbRzHG+JwFyjHgS1NSiQkLAtzTi3vzjbOyGD88iu88t5YbnljB5r0He6zX1qY0NHdeN2xL2UEO1DUzbVQc1Q0t7BpA15kxxvTEAuUYEB4cyNdmZjIqIZyJqTGHrffaf57JLy7N5YviKi7884f8a/nObvUeXlrAmfcvpa7p0Kyw9nGT9rGb9UVVPj6Kwffge1u45rHP7JocY45RFijHiO/MGcP7d51NgEsOWy8owMXNMzN5/66zmZoex0OLt3ZazqW1TXl2+S4qahp5fd2hZVuWF+5jREwo544fRnCgiy+KBy9Qnv5sB396dws7K2t9ts+W1jae/mwnywr38cYXthyNMceiQH83wLi1X3firbiIYG6dlcU3n1nNR1srOGe8e6rxJwUV7K1uIDjQxb9W7OLq6SNRVZZv38es7ESCA13kpESzbveBTvurqGnEJUJ8RPCAjmNDcRU/X5iHKjy0eCszMuOZNSaRoEAXgS5hZHw48yYM7/d+PyuspLK2ibCgAB58bwsXTUzpNXwbmlsJDQoY0HEYY/rPzlCOY+eOH0ZCRDAvrDy0Ms5La4qICQvi+3PHsm73AfL2VFFYUUtFTSMzMuMBmJQWw4biqo6uo5bWNi5/5BNm/OY9vvH0KhZv8m4Ry65UlXtf20h8eDDvfH82P5g3joqDjfzx3S3c92Y+//3GJr7x9Oojmvr82ro9RIYE8rsrJrKtvJaF67rfsGzd7gN865nV5Pz8Ld6yO2Qac9RZoBzHggNdXDE1lfc2lVJR08jBhmbeztvLpZNSuHbGSIIDXTy3YlfH+El7oExMjaG2qZXCCneX1HubSinaX8/cnGRW79zPLU+t4sYnV/R7JtgbX5SwYsc+7po3jrHJUdx+zhgW/9dZ5P/6AjbeO493vz8bgNf7uYJyY0srb23Yy/kTkrls0ghyU6L583uHuvrW7T7AdY8vY/4jn/BJQQUhgS6W5pf16zOMMQNngXKcu3raSFralH+vKWbRFyU0NLfx5alpxIYHc8nEFF5Zu4el+WUkRoaQlRgBwMlpsQB8UXwAgH9+tpPU2DAeuX4qn/14Dt+Zk82n2yr5vEu32OHUN7Xyu0X55KREc/W0Qyv1iAihQQGEBweSnRzF5JGxvLZuT6f3qirv5O3tNjOt3UdbKqhuaOHSSSNwuYTvnzeWHZV1/O3j7fxwwTrmP/IJW0pr+MlF4/nk7nM5c0xixxTp/io+UM+NT66wFZ6NOQIWKMe57OQopqTH8uKq3by0upispAgmj4wF4NpT06lpbOGdjaWcmhnfMU4zOimCsKAA1hdVsbX0IJ9uq+T609IJcLkvrPz6rEwiggN4eln3GWS9+euH2yg+UM8vLs097MSCSyeNYGNJNYXlNR1lb+ft5banV/PkJ9t7fM9r6/cQGx7UsYLA3JxhTEqL4b4383l5TTG3zc5i6V1ncdvs0USFBjEtI57CilrKDzZ63f52L67czYdbynl+5a5u25bkl7K1tOep2sYYC5QTwlemjWRrWQ0rduzjy1PTOoJj2qg4sodFAnBqVnxH/cAAFxNGRLO+qIqnl+0kOMDFVzzOKqJCg7hiahqvry9hX21Tj5+pqmwtPchfFm/lwj9/xIPvbeXiiSmclnX4uxZcPDEFkUPdXm1tyoPvbQXg1bV7utWvb2rlvY2lXHhSCkEB7r+uIsKvLz+Jq6el8db3ZvOTi3KICg3qeM/0DPexrt7Zv7MUVe2YQfbvtcWduvyK9tdx61OruOqvn1moGNMLC5QTwCWTRhAWFIAIXDH10K1qRIQbTh8FwBmjO//Qn5wWS96eKl5eU8wlJ6eQEBnSafsNp4+iqaWt04B/fVMrT3+2g9v/tYbpv1nMeQ98yB/f3UJ4cAA/uziH3195cp9tHR4TyvRR8R3dXm/l7SV/70FOy4pnc+lBNpVUd6q/dHMZtU2tXDoppVv7f3/lJMY4gelpYmoMIYEuVmw//M3MutpcepCCshpOGRXHzso61uw69P6nl+1ERAh0ubjhiRUU7bcLQ43pygLlBBAZEshts7O4bkY6KTFhnbb9x6mjePt7sxkzLKpT+clpMTQ0t1HT2NIROp7GJkdxWlY8zyzbSWubUlXXzH88sZx7Xs1j7a79zMpO5LdfmsiyH8/hpW+dwa2zsogI8W4W+qWTUthaVsOmkmoefG8Lo5Mi+Mu1Uwl0Ca+s7Tx7699ri0mKCuHUTK/u1wa4JytMHhnb73GUN9aX4BJ44OrJhAUF8NIad1vqmlp4fsVu5k1I5ulbZlDX1MKNT6ygoqb/XWrGnMgsUE4Q3z9vLL/50sRu5S6XMG54VLfyiWnuK/InpsZ0jLl0dePpGRQfqOfFVbv5ymOfsb7oAA9fN4VPfzyHB74ymetOTWd4TGi/23rBSSm4BP7rxXVsKa3hu3PHkhQVwlljk1i4bk/HdOblhZW8u7GUa6eP7POCz65mZMaTt6fK63vIqCqvry/h9NEJpCeEM29CMq+v20NDcyuvrN1DVX0zN8/MJCclmie/Op09VfX8aMH6fh/7YPj+C5/zj17Gn4w5mixQhqjMhAgumjic/zp/bK8XVZ6Xm0xydAg/fvkLdlbW8cRN07nk5BED/uykqBBOH53AxpJqsodFcvFEd3fW/CmplFQ1sHz7Pppb27jn1Q2kxobxrbPH9PszpmfE06awdpd33V4bS6rZXlHbcXxXTE2juqGFJfll/OPT7UwYEc20UXEATMuI52szM3l/S3mvY0xHi6ry5oYSXuvnVGxjBoMFyhDlcgn/e/0pnD1uWK91ggJcfPOs0SRGhvDMracye6zvbql82ST3D/d352Z3nH2cl5NMRHAAr6wt5u+fbGdLaQ2/vGwCYcH9v+p9Snqs+2ZmXt775fX1JQS4pOMq/pljEhkWFcJv3tjEltIabp6Z2Sl4L5qYQmube7qzPx2oa6ahuY2Ne6pptTXOjJ/Z0ivmsG6emclNp2fg6meXU1+uPGUkI+PDOd1jVlhYcADzThrOoi9KaFVlbs4wzstNPqL9R4UGkTsimpU7+j5DUVXeWF/CzDGJHUvPBLiEL01J5a8fFpIQEcwlJ3eeFDBhRDTp8eEs2rCXa2akH1EbfaH4QD0A9c2tFJbXkJ3cvXvTmKPFzlBMn3wdJuD+wT5jdGK37rbLJ6dysLGF1jblF5dOGNBnTM+IZ+3u/TS1HH4Zmc93H2DXvjoumdg5NK6YmgbA9aeN6rY2mIhw0cQUPi2o4EDdoW6v1jZlR4XvFsXsS4nHBZgb9hx/K0ibE4sFijmmnDE6gSnpsdx94XhGxocPaF/TM+JpaG477A/textL+do/VhIVGsj5EzqfDY0bHsUrt8/kjnN6HsO5eGIKLW3KO3mlHWW/eWMTc/70wVELlZIq9xmKS+CLouo+ahszuCxQzDElMMDFv789k5tnZg54X+0XOC4v7D6O0tDcyi8X5nHrP1eREhPGK7fPJDa8+0rLk0fGEhzY8/8mJ6VGkxYXxqIN7gHxFdv38fdPt9Papry8pmjA7ffGngMNBAUIE9Ni7QzF+J1XgSIiF4jIZhEpEJG7e9geIiIvONuXi0iGU54gIktFpEZEHvaoHy4ib4hIvojkich9Htu+KiLlIvK587jVY9tNIrLVedw0oCM3J7ykqBBOGRXHPz/b0W2dsB8sWM8/Pt3BzTMz+PftZzA6qfsFkn0RES6emOK+ZUBVAz9csI60uDBmZMTz0priXm8Epqp8vLWC9zaWej1LrKq+mXte2cD+LvVLquoZHhPKyakxbNxTbTcfM37V56C8iAQAjwDnAUXAShFZqKobPardAuxX1TEicg1wP/AVoAG4BzjJeXj6g6ouFZFgYLGIXKiqbzrbXlDVO7q0Ix74BTANUGC1047+XQ5thpS7zh/HtY8v45+f7eC22aMB9z1jXlu3h+/MyebO88YOaP8XTUzhrx8Wcv3flrGjso5nv34qZdWNfO+Fz1mxY1+3pWj21zbxs1c38IbHNN+sxAgyEiMIdNZSyx0Rze1dutmeWbaTp5ftZOqoWL40Ja2jvORAAykxYUxMjeHpZTvZUVlL1hGEozG+4M0ZygygQFULVbUJeB6Y36XOfOAp5/kCYI6IiKrWqurHuIOlg6rWqepS53kTsAZI4/DmAe+q6j4nRN4FLvCi/WYIO310AmeNTeKRpduoqm+mqaWNXyzMY2R8GN8+e/SA939yWgypsWFsK6/lhtNGccboROZNGE5kSCAvre7c7bU0v4zzH/yQd/L28oN543jhttP40QXjyUqKpOxgA7v21fH57gP8z9ubWb3z0L+TWlrbeMZZqLOwvPPYzJ6qekbEhDIhNRqADXtOzHGUZ5fv4k27U+cxz5tpw6nAbo/XRcCpvdVR1RYRqQISgIq+di4iscClwJ89ir8sIrOBLcD3VXV3L+1IxZg+/PCCcVz80Mc8+sE24sODKSir4W83TvPJXR1FhOtOTWfh53u4+8LxgHv680UTh/PG+hJ+NX8C4cGBvL5+D//53FrGJUfx1M0zyB3hDoBTu5zB1DW1cOb9S/nz4q3882szAHhnYyklVQ24pHOgtLUppdUNpMSGMTY5iuAAF3nFVR3X+JwoVJU/vrOZzMQILuwyE88cW/x6HYqIBALPAQ+paqFT/BrwnKo2isg3cJ/5nNuPfd4G3AaQnu6/6wPMsWPCiBjmTx7B3z/ZToAI544fxtwjvL6lJ7efM6ZbF9UVU9N4cVURb+ftJSUmjDtfWMe0UXE8fcuphw2y8OBAvj4ri/vfymftrv1MSY/jH5/uIC0ujNFJkWzzWPa/oqaR5lZlREwoQQEuxqdEnZAD8+UHG6n084oExjvedHkVAyM9Xqc5ZT3WcUIiBqj0Yt+PAVtV9cH2AlWtVNX2Vff+BpzSj3agqo+p6jRVnZaU5Lsru83x7b/OG0dLq9Lcpvzi0txB/7wZGfGkxYXx1w8K+fo/V5GeEM7jXp4V3Xj6KOLCg3ho8VY2lVSzYvs+bjhtFGOTI9leUdsx8N5+UWP7gqATRsSwobi633faPNblOStQV9Y2UVXX7OfWmMPxJlBWAtkikukMoF8DLOxSZyHQPuvqSmCJ9vG3WkT+G3fwfK9Luec57WXAJuf528D5IhInInHA+U6ZMX1KTwjn91eezANXT2ZUQsSgf57LJVwxNY38vQcJCwrgqa/N6HFack8iQgK5dVYWSzeX87NXNhAa5OIr00eSlRRJY0tbR5C0X9SYEuteoHNiagxV9c0U7a9HVbn/rXyue3xZv2Z+3f3Sep76dEf/DrYXS/JLvV5L7XA2eowLba88eheNmv7rs8vLGRO5A/ePdwDwpKrmici9wCpVXQg8ATwtIgXAPtyhA4CI7ACigWARuRx3EFQDPwXygTXO1dIPq+rfgO+IyGVAi7Ovrzrt2Cciv8YdcAD3quqR3efVDEntV74fLdefms6WvQf5zpxsUmPD+n6DhxtPH8VjHxayeud+rpk+ktjw4I5bOBdW1DIyPpw9TrCMcM5QTnIG5r8oruKJj7fzDycYviiuYlIvK0p72lZew/MrdzM2OZKbzsjoV3u7qqxp5Nv/WkNMWBBL7zqb8OAj713fWFJNcICLptY2Cstrel0d2/ifV9+yqi4CFnUp+7nH8wbgql7em9HLbntcz0NVfwz8uJdtTwJP9t1iY/wvOTqUR284pe+KPYgKDeK22Vn84Z3NHT/u7dOBC8trOGtsEiVVDYQGuYgNd9+tcmxyFIEu4Vev5VFa3ci1M0bywsrdvLeptFugbN57kLHJkZ2WvmmflbaltIaKmkYSu9x0rT+e+nQHDc1tNDQ38ugHhQOanr1pTzVnZify/uYyth/FZW1M/9mV8sYco7551mjeu/MsclLcZx6JkcFEhwZ2zPQqqapnRExYRyiEBgUwNjmK0upGvj4rk99+aSLTMuJ5b1NZp/1+WlDBvAc/7JiKDDhX9xd3nEn1tLqAt2obW3jqs52cl5vMJSen8NiH2zrOpvqrrqmF7ZW1TEqLZWR8OIUWKMc0CxRjjlEBLul0Bb+IkOUx02vPgYaO8ZN235ubzS8vzeUnF+UgIszNGcamkupOtyx+2gmSB9/bysEG9yD3R1vL2VvdwI8uHE9EcACfFfY54x+A5tY2/vZRYce4DsDzK3dTVd/MN88azd0XjqdN4fdv5R/Rf4P8vQdRhdwR0WQmRrC93ALlWGaBYsxxJCspotMZStdbPp8/YThf9bh3y9wc9/ToJfnus5TS6gbe2VjKrOxEKmubeOxD92z9BauLiA0PYt6EZKZnxvPZNm8mabov1vzvNzZxxf9+wua9B2lqaeOJjwqZkRHPKaPiSIsL59YzM3nl8z1HNEDfPiDfESgVtSfcLLYTiQWKMceR0UmR7K1uoKq+mbKDjYzo4xbMWUmRZCVG8O5G94rIL67cTWubcu/8k7hs0gge/6iQLaUHeWdjKZdPTiUkMIDTsxLYVl5LWfWhBS6eXb6LS//yMc2tnW8FsCS/jAjnBmhXPfopv120iT1VDXzz7KyOOt8+ZwyJkSHc92b/z1I2llQTExbEiJhQshIjqG9upbS6se83Gr+wQDHmODI6yT3T67NtlahCihezx+bkDGN54T6q6pt5bsUuzhyTSGZiBD+YN47WNuWGJ5bT1NLGlae4Z8GdPtp99f4y526XDc2t/OndzXxRXMUnBYe6wtralCX5ZZw9bhgvfesMEqNC+MenOxiXHMU5HncCjQwJ5OaZGSzfvq/fYykb91STkxKFiJCZ6ExKqKjp413GXyxQjDmOtM/0+rigHICUPs5QwN3t1dTaxq9f38ieqgauP9W9gsTI+HBuPD2D0upGxg+PYoKzHMyEETFEhQR2dHu9sHI3FTVNBAe4WPj5no795u2ppuxgI+eOH0ZaXDgLvnkGl08ewS8uy+1247SLnCVTFvVjPa7WNiV/bzW5KTEAZDph6jnTa1dlHX//ZLt1gx0jLFCMOY6MSgjHJfBJgfvH3pvrW04ZFUdMWBALVheRFBXSadmZO84ZQ2psGF/zGHcJcAkzMuNZVlhJU0sbf/1gG9NGxfGlKam8nbeX+ib3rQDe21SKCJwz3n02Eh8RzIPXTOGM0Ynd2pCZGEFuSnS/AmV7RS0NzW0d656lRIcSEujqNDD/4OIt/Oq1jZ3uXHmkyqobeHHVbgunAbBAMeY4EhIYQFpceMe/0r3p8goMcHGu86N/zfSRBAUc+t8+LiKYT+4+l6unj+z0ntNHJ7C9otY95beqgdvPHcP8KSOobWplcb57PGZJfhlT0+OIj/BuBYCLT05hza4DXnd7bXSWXMl1pk27XNIxMA/urrj2u2V+UTzwNcyeXraTHy5Yz4urdvdd2fTIAsWY40z7OEpUaCCRId5dgX7F1FQSIoK5doZ3C6a238flgfe2MmFENGePTeLUzASGRYXw6ud7KK1u4Iviqo6g8kZ/u7027qkmKEAYM+zQ1OmspEOB8v7mcmoaWwDY4INAWV/k3sevX9/UaZq18Z4FijHHmfZxlBEx3i/nMis7idX3nMcIL5eAyU2JJiYsiNY25fZzxiAiBLiESyeN4P3NZbyy1r0ua/u0ZG/0t9trU0k12cOiOt2COTMxgl376mhubeO1dXtIjAwme1jkgM9QVJUNxVXMHJOAqvKjl9Zb19cRsEAx5jiT5ZyhdL2o0ZdcLuHscUmMHx7FvAnDO8rnTx5Bc6vy4HtbSY0NY2xy/+4O6W23V0NzKxuKqzrGT9plJkbS0qZs3nuQxfmlXDQxhUkjY/miqGpAAVBS1UBlbRPzJgznJxfn8ElBJc8s33XE+xuqLFCMOc5kOdNnu17U6Gu/v/JkXv72GQS4Ds3YmpgaQ6ZzPcicnGHdZnP1xZtuL1Xlp//eQGVtE5ec3PmGWpnOApmPf1RIQ3Mbl04awcTUGCprmwY0MN/e3TUxNYbrZqQzKzuR3y3aRGn14ff59LKdPL/CgqedBYoxx5nRw9w/qmlxgxsoIYEB3VYJFpGOO0L2Z/yknTfdXk98vJ2X1hTxvbnZnD2u82e0r7j82ro9pMSEckp6HBPT3NOKB9LttaG4igCXkJMSjYjwi0tzqWtq5Z28vYd9318/2MbfPt5+xJ97orFAMeY4MywqlEf/4xSvB9h97WszM/nZxTmcOab79GBvXDppBGt2HehxIP2DLeX8dtEmLjxpON85N7vb9riIYGLDg2hTuOTkFFwuITclmgCXDGhg/oviKrKHRXbcAG10UiSjEsJZurm81/dUN7jvPbO9opbGltYj/uwTiQWKMcehC04a7vV0XV+LCQ/i1llZBAYc2c/H9aelEx0ayIPvbe1UXrS/jv98dg1jk6P4w1WTcLl67k5r7/a61DlTCg0KGNDAfPuA/MTUmI4yEeGcccP4dFsFDc09h8WWvQcB9wWY28ps0UqwQDHGHGXRoUF8fVYW720qZX3RAcD9o373S1/Q2qY8dsM0Ig4zHXrKyDhyUqI7BcBJqTFsKD6ygfn2Afn2rrN2Z41LoqG5jeXbe17Kf1PJoTtJbik92O/PPRFZoBhjjrqvzswgNjyo4yzl2RW7+Liggp9cnEN6Qvhh3/vTi3N45fYzOk0ImJgaQ0VNE3v7GETvSfuZzUmpnQPl9KwEQgJdLM0v6+ltbNp7kKjQQIIChM1dAqWqvpmX1xTR1NLW43tPVBYoxpijLso5S1mSX8br6/fw2zc2ceaYRK7zYlwowCWEBAZ0KmsPg/bZWq1tyoLVRZ3u09Kb9gH59ivy24UGBXDG6ATe39xzoOSXVJObEk1WYiSb93YOlCc+3s6dL67jqkc/ZWfl0OkOs0AxxvjFTWdkEB8RzB3PrkVEuO/LE/s9Dbldbko0Ljl0xfx9b27irv+3jjl/fJ+HFm/tdRwE3CHkOSDv6Zzxw9hRWdft1sNtbUr+3oPkpEQzdnhUt0BZsb2S5OgQtlfUcvFDH/Pq58VHdFzHGwsUY4xfRIYE8o3Z7vum/OSiHNLiDt/VdThhwQFkD4vii+Iqnluxi8c/2s5Vp6QxZ3wyf3p3C+c98AF5e7oP2vc0IO/p7LHuactdz1J276+jrqmVnJQoxg+PovhAfcfdLxtbWlm76wCXnDyCN783m/HDo/ju85/zx3c2H7Wr7//+yXYe/WDbUfksT14FiohcICKbRaRARO7uYXuIiLzgbF8uIhlOeYKILBWRGhF52KN+uIi8ISL5IpInIvd5bLtTRDaKyHoRWSwiozy2tYrI585j4YCO3Bjjd1+flcXCO2Zy7YyRfVfuw0mpMazYvo97XtnA7LFJ/O6KiTxy/VSevfVUquqa+dtH3a8X6W1Avl16QjhZSRHdpg+3D8iPHx7N2OQoALaWue/Tsr6oisaWNmZkxpMaG8bzt53GNdNH8pclBfxyYR5tbYMfKq98voePtvY+5Xmw9BkoIhIAPAJcCOQC14pIbpdqtwD7VXUM8ABwv1PeANwD3NXDrv+gquOBKcBMEbnQKV8LTFPVk4EFwO893lOvqpOdx2VeHaEx5pjlcgknp8UecVeXp4mp0dQ1tZKZGMHD103pmNZ8xphEzhidyOqd3W9B3NuAvKdzxg1jWWFlx7L9AJtKDuISGJscxTgnUNq7vVY4s8KmZ8QD7tWef3fFRL4+K5OnPtvJXQvW0dI6eIP1qsq2shrGJPVvWRxf8OYMZQZQoKqFqtoEPA/M71JnPvCU83wBMEdERFVrVfVj3MHSQVXrVHWp87wJWAOkOa+Xqmr7Up/L2suNMeZw5uYmc15uMk/cNJ3o0KBO204ZFceufXWUH+x8++DeBuQ9nTNuGE0tbXzscbfK/L3VZCRGEBYcQFpcGOHBAR2Bsnz7PsYmR3a6TkhE+MlFOdx1/lheXlPM3z/Z4YMj7tne6gZqGls6rdJ8tHgTKKmA5w0CipyyHuuoagtQBSR40wARiQUuBRb3sPkW4E2P16EiskpElonI5b3s7zanzqry8qN/ymeM8Y+0uHAev3Faj9OOp46KBWDNrs5nKcsKK8lNie5xQL7djMx4hkWF8KTHEiubSg6SM/zQfVqyk6PYUnqQltY2Vu/Yx6mZ3X/+RIQ7zs1mRkY8zyzfOWhdXwVO19voYzRQBo2IBALPAQ+pamGXbf8BTAP+x6N4lKpOA64DHhSR0V33qaqPqeo0VZ2WlJQ0iK03xhwvJoyIITjA1SlQquqbWbPrAGeNPfzvRHCgi9tmZ/FZYSWrduyjprGFXfvqyEmJ6qgzLtk9dXhjSTW1Ta3MyIzvdX/Xn5bOzso6PtlW0WudgWgPlGP1DKUY8BwxS3PKeqzjhEQMUOnFvh8Dtqrqg56FIjIX+Clwmap2nKOqarHzZyHwPu7xF2OMOazQoAAmpEazxmMc5ZOCClrblLPH9f0Pz+tOTSc+IpiHlxZ0dG2NH36om2zc8Ggqa5t4w1n08nCB0r5szrODtDx+QVkN0aGBJEWGDMr+D8ebQFkJZItIpogEA9cAXWdYLQRucp5fCSzRPubHich/4w6e73UpnwL8FXeYlHmUx4lIiPM8EZgJbPSi/cYYw9T0ONYXVXVcvf7+5jKiQwOZPDK2z/eGBwdyy5mZvL+5nAWr3SMA4zudobifv7hyNxkJ4SRH936vmpDAAK46JY13Npb2uTz+kSgoq2HMsEifTHTorz4DxRkTuQN4G9gEvKiqeSJyr4i0z7R6AkgQkQLgTqBjarGI7AD+BHxVRIpEJFdE0nCfgeQCa5xpwLc6b/kfIBL4f12mB+cAq0RkHbAUuE9VLVCMMV45ZVQcjS1tbCypRlX5YEs5s7KTvF7k8obTRxEVGshzK3YTFRpIqsfdL8cOd3cv7a9rPuzZSbtrZ6TT2qa8uNL396/fVl7jl+4uAK9uSK2qi4BFXcp+7vG8Abiql/dm9LLbHuNTVef2Uv4pMNGL5hpjTDdT0+MAWLNzPyGBLkqrGznLi+6udtGhQdx8RgYPLSkgZ3h0pzOApMgQ4sKD2F/X3OOAfFcZiRGcOSaR51bs4tvnjOl0E7OBOFDXREVN07EdKMYYc7wbHhNKamwYq3ftp9Hp9uprQL6rm2dm8vdPdjBpZOfrVkSEccOjWFa4z6szFIDrT03nW/9aww8WrONAXTP5JdVEhwVx9bSRfGlKKnFHcHsCfw7IgwWKMWYImZIey5qd+6k42EhOSvRhxzp6EhcRzDt3ziYmLKjbtjPHJFLT2OL1nTTn5iaTFhfGws/3MDopkumZ8eyorOPe1zdy35v53Hj6KH52SddryA+vI1CSovqoOTgsUIwxQ8Ypo+J4fX0Je6sb+MZZ3a468EpKTM+Bcce52dzRw10mexMU4OK9O89ChE6rJ28qqebhJQX87ePtXDktrdNssr4UlNUQEugidZBvD90bWxzSGDNktI+jtGn/u7sGQ2hQQLel+HNSovn15ScRHODi+RWdB+2bW9t4eU1Rr7PDCsprGJ0U6bMxmf6yQDHGDBk5KdGEBLqICgnklFFx/m5Or+IjgrngpOG8vKao0xpif//EfZ+VM+9fwo8WrGdbeU2n97VPGfYXCxRjzJARHOji/AnDmT9lBEFeThf2l2tnpFPd0MIi52LJfbVN/GVJAadnJXDN9HRe+byYuX/6gAWriwCob2ql+EC9XwPFxlCMMUPKX649PhbYOC0rnqzECJ5bsYsvn5LGQ4u3UtvYwq/mT2BschTfnZvNt59Zw72v5XHW2CRKqxtQ9d8ML7AzFGOMOSaJCNfOSGfVzv28tWEvzyzbyTUz0jvuv5IYGcJvr5hIfXMrv120qaP7ywLFGGNMN18+JY3gABffeW4tIYEuvj93bKftY4ZF8s2zRvPvtcU8u3wXAS4hIyHCT621QDHGmGNWfEQw804aTlNrG98+ZwxJUd0XfLz9nDGkx4ezfPs+RsWHExzov591CxRjjDmG/ee5Y7h6Whq3nJnZ4/bQoADunT8B8M89UDzZoLwxxhzDxiZH8fsrJx22ztnjhvGzi3PIHeH9RZCDwQLFGGNOALfOyvJ3E6zLyxhjjG9YoBhjjPEJCxRjjDE+YYFijDHGJyxQjDHG+IQFijHGGJ+wQDHGGOMTFijGGGN8QlTV320YNCJSDuwcwC4SgQofNed4Y8c+dA3l4x/Kxw6Hjn+Uqvb7lpYndKAMlIisUtVp/m6HP9ixD81jh6F9/EP52GHgx29dXsYYY3zCAsUYY4xPWKAc3mP+boAf2bEPXUP5+IfyscMAj9/GUIwxxviEnaEYY4zxCQsUY4wxPmGB0gMRuUBENotIgYjc7e/2DDYRGSkiS0Vko4jkich3nfJ4EXlXRLY6f8b5u62DRUQCRGStiLzuvM4UkeXO34EXRCTY320cDCISKyILRCRfRDaJyOlD7Hv/vvN3foOIPCcioSfqdy8iT4pImYhs8Cjr8bsWt4ec/wbrRWSqN59hgdKFiAQAjwAXArnAtSKS699WDboW4L9UNRc4DbjdOea7gcWqmg0sdl6fqL4LbPJ4fT/wgKqOAfYDt/ilVYPvz8BbqjoemIT7v8GQ+N5FJBX4DjBNVU8CAoBrOHG/+38AF3Qp6+27vhDIdh63Af/nzQdYoHQ3AyhQ1UJVbQKeB+b7uU2DSlVLVHWN8/wg7h+VVNzH/ZRT7Sngcr80cJCJSBpwMfA357UA5wILnCon5LGLSAwwG3gCQFWbVPUAQ+R7dwQCYSISCIQDJZyg372qfgjs61Lc23c9H/inui0DYkUkpa/PsEDpLhXY7fG6yCkbEkQkA5gCLAeSVbXE2bQXSPZXuwbZg8APgTbndQJwQFVbnNcn6t+BTKAc+LvT3fc3EYlgiHzvqloM/AHYhTtIqoDVDI3vvl1v3/UR/Q5aoJgOIhIJvAR8T1WrPbepe375CTfHXEQuAcpUdbW/2+IHgcBU4P9UdQpQS5furRP1ewdwxgvm4w7WEUAE3buEhgxffNcWKN0VAyM9Xqc5ZSc0EQnCHSb/UtWXneLS9tNc588yf7VvEM0ELhORHbi7N8/FPa4Q63SDwIn7d6AIKFLV5c7rBbgDZih87wBzge2qWq6qzcDLuP8+DIXvvl1v3/UR/Q5aoHS3Esh2ZnoE4x6kW+jnNg0qZ8zgCWCTqv7JY9NC4Cbn+U3Aq0e7bYNNVX+sqmmqmoH7u16iqtcDS4ErnWon6rHvBXaLyDinaA6wkSHwvTt2AaeJSLjz/0D78Z/w372H3r7rhcCNzmyv04Aqj66xXtmV8j0QkYtw96sHAE+q6m/826LBJSJnAh8BX3BoHOEnuMdRXgTScd8G4GpV7Tqod8IQkbOBu1T1EhHJwn3GEg+sBf5DVRv92LxBISKTcU9GCAYKgZtx/0NzSHzvIvIr4Cu4ZzquBW7FPVZwwn33IvIccDbuJepLgV8Ar9DDd+0E7MO4uwDrgJtVdVWfn2GBYowxxhesy8sYY4xPWKAYY4zxCQsUY4wxPmGBYowxxicsUIwxxviEBYoxxhifsEAxxhjjE/8fW8iNM6kuEoYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(summary[\"losses\"][3:])\n",
    "#plt.yscale('log')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "losses = list()\n",
    "\n",
    "torch_dataset_val = Data.TensorDataset(concat_pooled_output_test, targets_test)\n",
    "\n",
    "loader_val = Data.DataLoader(\n",
    "    dataset=torch_dataset_val, \n",
    "    batch_size=VAL_BATCH_SIZE, \n",
    "    shuffle=False\n",
    ")\n",
    "\n",
    "dataiter_val = iter(loader_val)\n",
    "in_, out_ = dataiter_val.next()\n",
    "\n",
    "net.eval()\n",
    "pred_scores = net(in_)\n",
    "\n",
    "loss = loss_func(pred_scores, out_)\n",
    "losses.append(loss.item())\n",
    "\n",
    "r_1, _ = spearmanr(\n",
    "    pred_scores.cpu().detach().numpy()[:,0], \n",
    "    out_.cpu().detach().numpy()[:,0], \n",
    "    axis=0\n",
    ")\n",
    "\n",
    "r_2, _ = spearmanr(\n",
    "    pred_scores.cpu().detach().numpy()[:,1], \n",
    "    out_.cpu().detach().numpy()[:,1], \n",
    "    axis=0\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.41671109171725407, 0.20942198041364313)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r_1, r_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}