tutorial-pytorch-lightning-slightly_fancy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qdB23YiUA02F"
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%load_ext line_profiler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "python 3.8.6 | packaged by conda-forge | (default, Dec 26 2020, 05:05:16) \n",
      "[GCC 9.3.0] /home/mtadmin/miniconda3/envs/dev_pytorch17/bin/python\n",
      "pytorch 1.7.1\n",
      "torchvision 0.8.2\n",
      "pytorch-lightning 1.1.3\n"
     ]
    }
   ],
   "source": [
    "import sys, os, warnings, time, logging\n",
    "import torch\n",
    "import pytorch_lightning as pl\n",
    "from pytorch_lightning.core.decorators import auto_move_data\n",
    "from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor, ModelCheckpoint\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torchvision\n",
    "from tqdm.autonotebook import tqdm\n",
    "\n",
    "from lightning_addons.progressbar import ProgressBar\n",
    "from lightning_addons.progressplotter import ProgressPlotter\n",
    "from lightning_addons.progressprinter import ProgressPrinter\n",
    "\n",
    "warnings.filterwarnings(\"ignore\", category=UserWarning)\n",
    "warnings.filterwarnings(\"ignore\", category=DeprecationWarning)\n",
    "# this makes lightning reports not look like errors\n",
    "pl._logger.handlers = [logging.StreamHandler(sys.stdout)]\n",
    "\n",
    "print(\"python\", sys.version, sys.executable)\n",
    "print(\"pytorch\", torch.__version__)\n",
    "print(\"torchvision\", torchvision.__version__)\n",
    "print(\"pytorch-lightning\", pl.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_3channel(image):\n",
    "    return image.repeat((3, 1, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((3, 64, 64), 42000, 18000, 10000)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class MnistDataModule(pl.LightningDataModule):\n",
    "    def __init__(self, batch_size, n_workers, train_fraction=0.7, extra_transforms=None, data_dir=\"data/mnist\"):\n",
    "        super().__init__()\n",
    "        self.batch_size = batch_size\n",
    "        self.n_workers = n_workers\n",
    "        self.train_fraction = train_fraction\n",
    "        self.data_dir = data_dir\n",
    "        transforms = [torchvision.transforms.ToTensor(),]\n",
    "        if extra_transforms is not None:\n",
    "            transforms += extra_transforms\n",
    "        self.transform = torchvision.transforms.Compose(transforms)\n",
    "        \n",
    "    def prepare_data(self):\n",
    "        # this is intended to just run the download\n",
    "        torchvision.datasets.MNIST(self.data_dir, download=True, train=True)\n",
    "        torchvision.datasets.MNIST(self.data_dir, download=True, train=False)\n",
    "        \n",
    "    def setup(self, stage=None):\n",
    "        # this can potentially be run on different machines and GPUs hence the actual preperation should be done here\n",
    "        if stage == \"fit\" or stage is None:\n",
    "            trainval_data = torchvision.datasets.MNIST(self.data_dir, train=True, transform=self.transform)\n",
    "            n_trainval = len(trainval_data)\n",
    "            n_train = int(self.train_fraction * n_trainval)\n",
    "            self.train_data = torch.utils.data.Subset(trainval_data, range(0, n_train))\n",
    "            self.val_data = torch.utils.data.Subset(trainval_data, range(n_train, n_trainval))\n",
    "            # if we set this here we get the actual transformed size, which is nice at it does not require prior knowledge as in __init__()\n",
    "            self.dims = tuple(self.train_data[0][0].shape)\n",
    "        if stage == \"test\" or stage is None:\n",
    "            self.test_data = torchvision.datasets.MNIST(self.data_dir, train=False, transform=self.transform)\n",
    "            self.dims = tuple(self.test_data[0][0].shape)\n",
    "        \n",
    "    def train_dataloader(self):\n",
    "        return torch.utils.data.DataLoader(self.train_data, batch_size=self.batch_size, num_workers=self.n_workers, shuffle=True)\n",
    "    \n",
    "    def val_dataloader(self):\n",
    "        return torch.utils.data.DataLoader(self.val_data, batch_size=self.batch_size, num_workers=self.n_workers)\n",
    "    \n",
    "    def test_dataloader(self):\n",
    "        return torch.utils.data.DataLoader(self.test_data, batch_size=self.batch_size, num_workers=self.n_workers)\n",
    "\n",
    "output_size = 64\n",
    "# imagenet\n",
    "norm_mean = [0.485, 0.456, 0.406]\n",
    "norm_std = [0.229, 0.224, 0.225]\n",
    "# default\n",
    "# norm_mean = 0.5\n",
    "# norm_std = 0.5\n",
    "extra_transforms = [\n",
    "    torchvision.transforms.Resize(output_size),\n",
    "    to_3channel,  # we use 3 channels to directly use (pretrained) ResNets\n",
    "    torchvision.transforms.Normalize(norm_mean, norm_std),\n",
    "]\n",
    "mnist_datamodule = MnistDataModule(32, 8, extra_transforms=extra_transforms)\n",
    "mnist_datamodule.prepare_data()\n",
    "mnist_datamodule.setup()\n",
    "mnist_datamodule.size(), len(mnist_datamodule.train_dataloader().dataset), len(mnist_datamodule.val_dataloader().dataset), len(mnist_datamodule.test_dataloader().dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAAD3CAYAAADmIkO7AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAOyUlEQVR4nO3de7yVUx7H8d8ySscoyq0LStQgo6YSklJya1yOS2MmMnR1P1QTMYNxKU3mJZkhlJESyUjuJU2JoZpGGlNuka4iHIUOU9b84cx6nvXMOfvss/dzzv49e3/er1ev12+dZ+31rENfz1r2fvZjrLUCQJ8dcj0BABUjnIBShBNQinACShFOQCnCCShFOHPIGLPKGNMzzb7WGHNghufJ+LXIHcIJjzGmkTFmmjFmU/mfh40xDXI9r0JEOBF1i4g0FJGWInKAiOwtIjfmckKFinAqYYzpZIx5zRhTaozZYIz5kzGmbqRbL2PMB+VXtDHGmB1Cr+9njFlhjPnCGDPLGNM8w6nsLyJPWms3W2u/FJEZItImw7GQBcKpx3YRuUpE9hCRo0TkOBG5JNLnDBHpKCLtReR0EeknImKMKRaRa0XkTBHZU0QWiMgjFZ3EGNPHGLMsxTz+LCKnGGMaGmMaishZIvJ8Zr8SskE4lbDWLrHWvm6t3WatXSUi94pIt0i30dbaz621q0VkrIj8qvzng0VklLV2hbV2m4iMFJF2FV09rbVTrbWHpZjKP0Wkroh8Vv5nu4jcncWvhgwRTiWMMa2NMc8YYz42xmyWHwK2R6TbmlD9kYg0La+bi8id5UviUhH5XESMiDTLYCrTReRdEakvIg1EZKWITMlgHGSJcOpxj4i8LSKtrLUN5Idlqon02TdU7yci68vrNSIy2Fq7W+hPkbX27xnMo62I3Gut/dpa+5WIjBeRXhmMgywRTj3qi8hmEfnKGHOQiFxcQZ/flO8F9xWREhGZVv7z8SIywhjTRkTEGLOrMaZ3hvNYLCIDjDFFxpgiERkkIm9mOBayQDj1GCYifURki4jcL0HwwmaKyBIRWSoiz4rIRBERa+0MERktIo+WL4nfEpGTKzqJMeZcY8y/U8yjn4i0EJG1IrJOfnhL5YLq/jLInuFma0AnrpyAUoQTUIpwAkoRTkCpHVMdNMbwf4uAGmatjb6fLSJcOQG1CCegFOEElCKcgFKEE1CKcAJKEU5AKcIJKEU4AaUIJ6AU4QSUIpyAUoQTUIpwAkoRTkApwgkoRTgBpQgnoBThBJQinIBShBNQinACShFOQCnCCShFOAGlCCegFOEElCKcgFKEE1CKcAJKpXwEIPJNvVB9sHekydE9XN29R1C/+847Xr9/zHkpaHz+bKyzg48rJ6AU4QSUMtZW/vBqnmydPAeecL2rrxwyxDvW8fBdXd2pkf+6Ch+tXIUbJr/ntW86v3UGo4AnWwMJQzgBpVjWqrKbqw7++VDvyFGdO7v68E6dvGPH9dzF1a1qZmJpmTC/zNUDi88IDpS+kIPZJAfLWiBhCCegFOEElGLPmWOX/PFVV/95SOcUPZPLmJ0jP9mak3loxZ4TSBjCCSjFB99rw37nuHLC5Eneof5dd6rt2YiIyMZIe0nowz7/XLLK1acVt/D6HVZPqu3IPqO99utTr6j+IAWIKyegFOEElCKcgFK8lVIDHl7k/2Prc3jNnm9TqJ40M9hNTp7k72/fnDE13MroXOdc+5SrH7311LReM2e13z6+eSb3wOQv3koBEoZwAkqxrI1JybhFrh57efzr2I9D9aXDn/COPXHXn4JG2d9iP7cv+O6h595b7h05+cCKX/FtpF2v4clBgztWWNYCSUM4AaX4hFAWRk1f4+przt4n6/HWher+l072js26+/ysx4/HClfNnrXEO3LygR0qfEX0M1B7tA6+a2jTIpa1leHKCShFOAGlCCegFHvO6mh2tteMY58Z/vBM+07B3RqfLb4r67Fr2tzwoxlERC6teM8Z1bhxY1dvStGv0HHlBJQinIBSLGur4fpRI7MeY8HHfrvrQScFjS9nZT1+bVr26iuRnwzPyTzyFVdOQCnCCShFOAGl2HNWZc/ghuLf9s3+SSQXDxrl/yBh+0zPp5ndsP3999/HPJH8xJUTUIpwAkqxrK1Cy7btXF0nhvEemjrCa3doOi9obJkdwxlqT5tTB2f0urVr18Y8k/zElRNQinACShFOQCn2nFVo27ZtrOO138Vvz10evJXSY99kfZ/rwIvS23N+F2lvXsOeMx1cOQGlCCegFMvaKqxbt67qTlkoLa3R4WN3yCm3uLqk1+5pvWZh5E4c+eyZGGeUv7hyAkoRTkAplrVVWPZmZh/uTtdTT86p0fHjds9911X7NS+9+F7VnfB/uHICShFOQCnCCSjFnrMKZSuCp0P/69uJ3rGfRh8CkoHVq1dX3SmHegyY4LW7Nqn+GBs2bIhpNoWFKyegFOEElGJZW6XtrnrmqTXekZ/23jfr0XffPfwpm/B4a6JdUwgeKX1Y8UBX9ziuh9erTp3gdvF6RUXesQ4dg8fydTgk+Hn2D5wQadioUQyjFB6unIBShBNQinACSrHnrIbXX3vN/0EMe86ju3Rx9bZhd7j65XnzvX7tO3ao8DUiIpefe4CrNe7uOh1xaK6nkEhcOQGlCCeglLHWVn7QmMoPFqDzf+8/OmHS9SfkaCbJ9ubWoB5S8oB3bO79V4ZaW2plPrlmra3wy6O4cgJKEU5AKZa11XDFuEVe+87LD8/RTApDw/aXuLr0jXtyOJOaxbIWSBjCCShFOAGl2HNWoWXP37p65Ys353Am+nwUaYefNJHeN9qmFr4NvXmzs/yD65+I4Qw6sOcEEoZwAkqxrI3oe+MLXvuhG07M0Ux0mLLQb/c9slWo9X6KVwY3Bdz+5GLvyNDT9856XoPHLHD1fcO7Zj1eLrGsBRKGcAJKEU5AKfacItLuzDGufuOvw3I4k9zZGKpLrnva1dNGnhb7ubpeeK+rpzwwyDuWye3rpnGx/4ONMzMYJXfYcwIJQzgBpQpyWXvWcP/TJY+PPiNHM6ld74bqW2+a7R176IaTQq3a/Ne+q9eavrTU1We3TW+ElyNPzu7WpGOotSSzadUilrVAwhBOQKmCXNam+p2TLvxh8fEPLPeOjRoyNGh86X8SSo+fuGrWB2+7+oT90x/h758G9dF7VbhiVIVlLZAwhBNQinACShXMnrPokP6u/ubfE1L0rFkPvrrda4+8+RZX161bJ1TX9fq9MT/0eIZNcyKjlsU2P1V2Dh5haL9+KaMh9j7qKq/9yetjs5lRjWDPCSQM4QSUKphlbcldwQ2/Yy/rmKJn/P7wxHpXX31Ws1o9d76Y8Zb/V7G4TXqvu3fuN177ouN+HNeUYsOyFkgYwgkoRTgBpfJ6z9n82BGuXvW3kbV23i8i7dZHlLh606JxtTaPfLLDAed57e3vT07rdesi7X2Mvo/zsecEEoZwAkrl9bJ2cWkw/Y67pugYszue/tRrDzltr9o7eYHo3v9+V8+dMCDt1x3ZJ9hWLHykJEXP2sOyFkgYwgkoldfL2lzdVH3b42u99oje7UKtz2p1Lvmrg6us/Ufarxo/5ytXX3x8/VhnlCmWtUDCEE5AKcIJKJVne07/vzXWbq+kX+Xei7RbN+rl6l7n+Z9S6XNeH1fXCe6Tli8+/9rrd1G/4EZvWT2t2nNCaqWRv8Op3jULv8m1lzkg1PogzilVC3tOIGEIJ6BUni1r/U/iWLuxkn6VC9+ULSIy7opOaZ2v2TG/dvXAwYO9Xi/NCb7zZ8GDF1V7Tkjty8jf4QZpvs4UdQ8aZfNim091sawFEoZwAkoRTkCpHXM9AW3WrF5ddSfnE1etWxA8HXteq1Zer67durl6/brfecdWvjg21NpSjXMXuJ2OdWW6e8yk4coJKEU4AaVY1kZs2ZL90nLeA+O9dptDD3X1gEGDvGMf9uzp6vuu7iZIz5lXXJ79IGVfV90nh7hyAkoRTkApPiEUMfy+ZV57zOC2Wc0I8Shq099rf/NWZk+Km70qqE/cX8fXZPIJISBhCCegFOEElOKtlIiftT8s11NABaZOi+dp5M89vbjqTkpw5QSUIpyAUnn2Voovk++tXR9pN2twYtDYMju7CaF6mp3tSrt2ekZDvPKJ3z5mbx1vn4TxVgqQMIQTUIpwAkrl9Vspt8/82NXDTm+c1muaRtq/uOxSVz82ij1nTWty9FBXL3vl9qzHu/KyO7IeI1e4cgJKEU5Aqbx+KyX8mLj564PHxHVtkv4I4VuvG+x5qn9w0zMZzgv/07TLMK/9Tui7mHbJcMz5G4L62Kb63jqJ4q0UIGEIJ6BUni9rA/XbBo9I2Lx0fIqeldscaZ/QZ5yrFz5SktGYhaGl15r94UpXH98i+9Efeu17r/3rzj/KftBaxLIWSBjCCShFOAGlCmbPGTbjLf/XKm6T/ZgDbpvn6okjulfesWAE+75n393mHenVKto3O8YcEfnJonhPUMPYcwIJQzgBpQpyWSvifwh+4ssfubrfMXWzHv1t///sy+iRwZOtH/zDGP9g0m7grhs8MqJ7377eod6/PMfVF/QMPt9TFMNpJ7263Wtf0KVzqJWsZWwUy1ogYQgnoBThBJQq0D1n5RZs9H/lLntV0jEmUxYG9cAL+7m6bMVfavbEKZiW57p6xtNTvGOnH1J785j9YVCf2FL/3SWZYs8JJAzhBJRiWVuFjucE30Ez/JorvWO929XceWcu99vTpz3v6scfe8w79u3KD4LGf74LHfHffhAT+sqo+g28Q1f/Mfi+ntsGHCo1qSxUj354pXds9MhRrt66fGKNzkMLlrVAwhBOQCnCCSjFnjMLLbpf6+r5c2919X65mIxiD77i35Vy4TEHhVr+nrMQsecEEoZwAkqxrI3NPq4qHuo/AuC04mJXN9zNfwLGLv47GpUq2xrUW7f6x7aFVo2pnnpoQounHSL/WS4K3TpSr6jyfmEbN/jtpW8sdfWsF2a5+l8zr6l8ELCsBZKGcAJKsawFcoxlLZAwhBNQinACShFOQCnCCShFOAGlCCegFOEElCKcgFKEE1CKcAJKEU5AKcIJKEU4AaUIJ6AU4QSUIpyAUoQTUIpwAkoRTkApwgkoRTgBpQgnoBThBJQinIBShBNQinACShFOQCnCCShFOAGlUj4CEEDucOUElCKcgFKEE1CKcAJKEU5AKcIJKPVfsSWIyv9MuXEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataloader = mnist_datamodule.train_dataloader()\n",
    "batch = next(iter(dataloader))\n",
    "# 0: input, 1: target\n",
    "sample_image = batch[0][0]\n",
    "sample_target = batch[1][0]\n",
    "f, ax = plt.subplots()\n",
    "ax.imshow(sample_image.permute(1, 2, 0).clamp(0.0, 1.0))\n",
    "ax.set_title(f\"label: {sample_target.numpy()}\");\n",
    "ax.axis(False);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"activation\":          softmax\n",
       "\"data_shape\":          (3, 64, 64)\n",
       "\"learning_rate\":       0.01\n",
       "\"num_classes\":         10\n",
       "\"pretrained\":          False\n",
       "\"scheduler\":           None\n",
       "\"sigmoid_soft_factor\": 1.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class MnistModel(pl.LightningModule):\n",
    "    def __init__(self, data_shape, num_classes, learning_rate, pretrained=False, activation=\"softmax\", sigmoid_soft_factor=1.0, scheduler=\"one_cycle\"):\n",
    "        super().__init__()\n",
    "        self.save_hyperparameters()\n",
    "\n",
    "        resnet = torchvision.models.resnet18(pretrained=pretrained)\n",
    "        fc_layer = torch.nn.Linear(512, num_classes)\n",
    "        resnet.fc = fc_layer\n",
    "        self.resnet = resnet\n",
    "        # this allows the trainer to show input and output sizes in the report (2 is just a sample batch size)\n",
    "        self.example_input_array = torch.zeros(2, *data_shape)\n",
    "        \n",
    "        self.train_acc = pl.metrics.Accuracy(compute_on_step=False)\n",
    "        self.val_acc = pl.metrics.Accuracy(compute_on_step=False)\n",
    "        self.test_acc = pl.metrics.Accuracy(compute_on_step=False)\n",
    "        \n",
    "    def configure_optimizers(self):\n",
    "        optimizer = torch.optim.SGD(self.parameters(), lr=self.hparams.learning_rate)\n",
    "        schedulers = []\n",
    "        if self.hparams.scheduler == \"one_cycle\":\n",
    "            steps_per_epoch = int(self.trainer.limit_train_batches * len(self.train_dataloader())) # same as batches per epoch\n",
    "            max_epochs = self.trainer.max_epochs\n",
    "            scheduler = torch.optim.lr_scheduler.OneCycleLR(\n",
    "                optimizer,\n",
    "                max_lr=self.hparams.learning_rate,\n",
    "                epochs=max_epochs,\n",
    "                steps_per_epoch=steps_per_epoch,\n",
    "                anneal_strategy=\"cos\", # can be \"linear\" or \"cos\"(default)\n",
    "            )\n",
    "            # \"interval: step\" is required to let the scheduler update per step rather than epoch\n",
    "            schedulers = [{\"scheduler\": scheduler, \"interval\": \"step\", \"frequency\": 1}]\n",
    "        elif self.hparams.scheduler == \"cosine_annealing\":\n",
    "            steps_per_epoch = int(self.trainer.limit_train_batches * len(self.train_dataloader())) # same as batches per epoch\n",
    "            max_epochs = self.trainer.max_epochs\n",
    "            max_steps = max_epochs * steps_per_epoch\n",
    "            scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(\n",
    "                optimizer,\n",
    "                T_max=max_steps,\n",
    "            )\n",
    "            schedulers = [{\"scheduler\": scheduler, \"interval\": \"step\", \"frequency\": 1}]\n",
    "        return [optimizer], schedulers\n",
    "    \n",
    "    def class_activation(self, x):\n",
    "        if self.hparams.activation == \"softmax\":\n",
    "            return x.softmax(1)\n",
    "        elif self.hparams.activation == \"sigmoid\":\n",
    "            return x.sigmoid()\n",
    "        \n",
    "    def class_loss(self, y_hat, y):\n",
    "        # y_hat: prediction, y: target\n",
    "        if self.hparams.activation == \"softmax\":\n",
    "            return torch.nn.functional.cross_entropy(y_hat, y)\n",
    "        elif self.hparams.activation == \"sigmoid\":\n",
    "            y_onehot = torch.nn.functional.one_hot(y, num_classes=self.hparams.num_classes).float() * self.hparams.sigmoid_soft_factor\n",
    "            return torch.nn.functional.binary_cross_entropy_with_logits(y_hat, y_onehot)\n",
    "    \n",
    "    def _forward(self, x):\n",
    "        return self.resnet(x)\n",
    "    \n",
    "    def training_step(self, batch, batch_nb):\n",
    "        images, targets = batch\n",
    "        \n",
    "        preds = self._forward(images)\n",
    "        loss = self.class_loss(preds, targets)\n",
    "        self.log(\"loss\", loss)\n",
    "        self.train_acc.update(self.class_activation(preds), targets)\n",
    "        # the accuracies are updated here, but only computed and reported at the epoch, see \"compute_on_step=False\", \"on_step=False\", \"on_epoch=True\"\n",
    "        self.log('train_acc', self.train_acc, prog_bar=False, on_step=False, on_epoch=True)\n",
    "        return loss\n",
    "    \n",
    "    def validation_step(self, batch, batch_nb):\n",
    "        images, targets = batch\n",
    "        \n",
    "        preds = self._forward(images)\n",
    "        loss = self.class_loss(preds, targets)\n",
    "        self.log(\"val_loss\", loss)\n",
    "        self.val_acc.update(self.class_activation(preds), targets)\n",
    "        self.log('val_acc', self.val_acc, prog_bar=False, on_step=False, on_epoch=True)\n",
    "        return loss\n",
    "    \n",
    "    def test_step(self, batch, batch_nb):\n",
    "        images, targets = batch\n",
    "        \n",
    "        preds = self._forward(images)\n",
    "        loss = self.class_loss(preds, targets)\n",
    "        self.log(\"test_loss\", loss)\n",
    "        self.test_acc.update(self.class_activation(preds), targets)\n",
    "        self.log('test_acc', self.test_acc, prog_bar=False, on_step=False, on_epoch=True)\n",
    "        return loss\n",
    "    \n",
    "    # this moves data to gpu when forward is called\n",
    "    @auto_move_data\n",
    "    def forward(self, x):\n",
    "        # in pl forward() is intended for inference, hence the activation is done here\n",
    "        logits = self._forward(x)\n",
    "        return self.class_activation(logits)\n",
    "\n",
    "num_classes = 10\n",
    "learning_rate = 1e-2\n",
    "pretrained = False\n",
    "activation = \"softmax\"\n",
    "sigmoid_soft_factor = 1.0 # soft label: this is a multiplier to reduce the one hot 1s in case of sigmoid to a smaller value\n",
    "scheduler = None #\"one_cycle\" #\"one_cycle\" # \"cosine_annealing\"\n",
    "model = MnistModel(\n",
    "    mnist_datamodule.size(), \n",
    "    num_classes, \n",
    "    learning_rate, \n",
    "    pretrained=pretrained, \n",
    "    activation=activation,\n",
    "    sigmoid_soft_factor=sigmoid_soft_factor,\n",
    "    scheduler=scheduler,\n",
    ")\n",
    "model.hparams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run this to show very detailed model information and if model has example_input_array also detailed feature sizes\n",
    "# pl.core.memory.ModelSummary(model, mode=\"full\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tb_logger = pl.loggers.TensorBoardLogger(\"/home/mtadmin/projects/tensorboard_logs\")\n",
    "\n",
    "plotter = ProgressPlotter()\n",
    "callbacks = [\n",
    "    ProgressBar(),\n",
    "    ProgressPrinter(),\n",
    "    plotter,\n",
    "    LearningRateMonitor(log_momentum=True),\n",
    "]\n",
    "\n",
    "# early_stop_callback = EarlyStopping(\n",
    "#     monitor=\"val_loss\",\n",
    "#     mode=\"min\",\n",
    "#     patience=5,\n",
    "#     verbose=True,\n",
    "# )\n",
    "# callbacks = [early_stop_callback] + callbacks\n",
    "\n",
    "# model_checker = ModelCheckpoint(\n",
    "#     monitor=\"val_loss\",\n",
    "#     mode=\"min\",\n",
    "#     save_last=True, \n",
    "#     save_top_k=1, \n",
    "#     filename=\"best_{epoch}\",\n",
    "# )\n",
    "# callbacks += [model_checker]\n",
    "\n",
    "max_epochs = 10\n",
    "limit_train_batches = 0.1 # 1.0 by default/full set, use this to check training on a smaller train set, also exists for val and test set\n",
    "trainer = pl.Trainer(\n",
    "    max_epochs=max_epochs, \n",
    "    gpus=1,\n",
    "    callbacks=callbacks,\n",
    "#     logger=[tb_logger],\n",
    "    precision=16,\n",
    "#     fast_dev_run=True, \n",
    "    limit_train_batches=limit_train_batches,\n",
    ")\n",
    "\n",
    "start_time = time.time()\n",
    "trainer.fit(model, mnist_datamodule)\n",
    "print(f\"runtime: {time.time() - start_time:.0f} s\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/mtadmin/projects/tutorials/lightning_logs/version_2'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.logger.log_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/mtadmin/projects/tutorials/default/13/checkpoints/epoch=8-step=1178.ckpt'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# this gives us the path to the best model from the default modelchecker\n",
    "trainer.checkpoint_callback.best_model_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'val_loss'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.checkpoint_callback."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaUAAAD4CAYAAABMtfkzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABQQElEQVR4nO3deXxU1fn48c8zk5nseyAEAoRNdgFBEFFccKUq7uIublXRqm2tVq217def1qpVq8WlotJSl6q4VBQRd+vCLvu+hUBC9n2Zmef3x52EIWSZwCSTmZz36zWvmbn3nnufSWCenHPPIqqKYRiGYXQGtmAHYBiGYRj1TFIyDMMwOg2TlAzDMIxOwyQlwzAMo9MwSckwDMPoNCKCdWGbzabR0dHBurxhGEZIqqysVFUN2wpF0JJSdHQ0FRUVwbq8YRhGSBKRqmDH0J7CNtsahmEYocckJcMwDKPTMEnJMAzD6DSCdk/JMIzwVldXR3Z2NtXV1cEOJSRFRUWRmZmJw+EIdigdyiQlwzDaRXZ2NvHx8WRlZSEiwQ4npKgqBQUFZGdn069fv2CH06FM851hGO2iurqa1NRUk5AOgYiQmpraJWuZJikZhtFuTEI6dF31ZxdySUlV+df3O1i/tzTYoRiGYbTNho9h7XvBjqJTC7l7SnllNdz/7mqyUmP44q6Tgh2OYYQmjwfKc6F4BxTvhOoSGH+DtW/lG5C3Btwu8LjAUwdRSXDK7639X/4F9v7k3ecCdx0k9YFznrb2z7sZclfRr7ICPrVbx2QcCRfPsfa/OAX2bbDO63FZ5x56Fpz9lLV/zTyISoSkvpDYGyKch/wx4+LiKC8vP+TybVawBfLWQeEWKNgMBVshMg4ue8Pa/9WjUFcNw6Z1XEwhJuSSUmWtG4DtBZVBjsQwOjFVqCqCou1W4inNgYkzrX0L7oMfXwR3zf7jo1P2J6UN82HDR2B3gC3CeiT33X9s8XbI32Rtt3v3u2v3749JgcTe1EZUEpWYDDYHpA3av3/IVOp6HU2VS6h0QUR1ERqRQYLLTaTdhr53G1Jb5j1YIKEnHHUVnHiPtWnlG5DYy0paCT3BZg/wD68F7joo2uGTdLZAZQFc/Kq1/9Pfw7oPrNcxaZA6wIqz3kWvQnRyx8UbgkIuKdW43MEOwTA6h5pyq5ZTvMP6ojzqSnDGwnd/h8//HzR8sXuNvsz6Quw5BibcaH1ZJmft/3KvV/8F25xpzx60SVVRj2KzCXnH/o6l24tYvXkH9rgU8stryN9Rw735FWSlxTLXeSH3fbX6oHN8OraKgd1ief3oN/jvl98zwJFPlr2A3lX72LS0gkvG1ZBqr4J5NzaUcUsEFVEZbBp0LSPPuQOn1lC47F1Ko3riSO2HRMVS6/LgsAu/+c1v+OijjxAR7r//fi655BL27NnDJZdcQmlpKS6Xi1mzZnHsMRO4++bLKd22nL5xLqZOOIIxv/kAIiLhk9/BD7P2Bx2VCKkDwVVr1egm/4aaY26nNrEfbmcCLo/i8Si28hrS4iIhqTd5ZdV0j/TnF9w1SbCWQ4+NjdVDmftuxa5izn32WwC2P/KzQIdlGJ2HqwZKsvfXdgZPhfgesPodmP9r6y90Xzd9Cz1GwKZPYdMnVu0mqa/3uY/1BdoGqkpplYt95TXkl9eQlRpLj8QotuVX8PyXW8gvr2FfeS35Zdb+564Yy0lDurNwbS43zFkCgAikxDhxq5KZFE1sZASVtS6KK+tw2G047NbNfLdHSYpxYrcJpdV1FFfW4fYoHlXcHusxKD0OhygUbcdZnk0meWTKPnpLHv91H8PTf3yAyIINMGtiw2eo0EiytRtPuC9hy49fs+Cdubzz9mvM/n4vfYeOxZG/jp66lz1Z0/jrjCnU/fAS9k9+iwNXwzkqieKutL/z7K0XQPYSZr+7gEX7EtjqSafAE4dbYXB6PPNvPx6Ac575hp+ySw74WR6dlcx/bjoWgF//ZyWPXTTKr9/BunXrGDp06AHbRKRSVWPb9MsMIaFXU6ozNSUjjKhatZ3cNVZCSeoDO76Dt66Fsj2Azx+NCb2spJTUF4ae7ZNwsqznmFTruEGnWI8WL6sUVtSyNb+CrfvK2VFQyXED0zh2YBrb8yu49MXvKSivpdbtaSjz8PkjuXR8HyprXSxan0daXCRpcU4GpMWSFh9JzyRr1v/x/VKY/4vjKdq7gwmjhhNht/GHD9awNsfqnBTjjCDG2fxXT0KUg4SopgeMKkDKAGpSBuDMiOfaM4dSUeNiVI2byAg7pA5k68ULKdi1iYiSnfzw0b857ah+xO8q5rJLL8W+dyUXbX+Ai3oARYC35e9X2xJZtao7o/uMZXHGxby1ooCy+AG4uo8gLnMYmYlR1oGZ44gYm8bg/EqG2wWbCBE2IT1hf9VnxqQsCsprsduk4ZEeH9Ww/7IJfVr83XR1IZeUql37/5OUVNWRGN21RjsbIUzVqjqU74MvHrYSUe6a/c1sUx+z7uvEdYf+J/gkHe9zfIZ1XOZY6+GHWpeHnYUVbNlXQVqck7F9UyiprGPyXz6npKqu4bgIm5AY7eDYgWkkxTiYNDCtIel0i48kLS6SI9LjARjeM5HF9zWf9BKjHSRGO1hXEkGE3erg+/uzhx/CD8w/UQ47qXH1H8RJ/2Hj6T9sPADHX/sQ5S+vwnXHHdb+vsfCjV/w9B9/zeTxRzL6xGnkuuIY8sVSrrzySu666y6u+vkshpaXs2DBAl555RW6devGE7NnN1zvqolZLcZz3pjMFvcf1cfcU2pJyCUl35rS5+vzOHdMryBGYxhNULVugOeu3p94clfDyItgyu/AEQWr34buw2DUdEgfDukjoLu3mSZ1AJz3XBsupxRU1FJe7SIrzWrVmTl3GWv3lLKzsBK3x6ptnTu6J2P7ppAQHcF5Y3rROyWG/t1iGZAWR6/kaOw2qyktKcbpd/NSqJg8eTLPP/88V199NYWOTB7/aDMXPziXHTU19OrTixuOGEdFZRXLli1j6tSpOJ1OLrjgAgYMGMA111wT7PC7lJBLSnXu/c0ZuaVdb7Sz0clUFu5PPJFxMOYKa/uLJ0FNKYgNUgdBr7FW8xxAZDzcvd2qNbWB26MNiePfP+xk6Y4ituaXsyWvnNJqFxP6pfDGzyc2HDs0I56fjcxgQPdY+qfF0b+blbBEhAfPab+aS2d03nnn8d133zFq1ChEhEcffZQePXrw6quv8pe//AWHw0FcXBxz5sxh9+7dzJgxA4/HapV5+OGHgxx91xJyHR3A+stw+O8XcMnRvdu1WcAwGrjrrHs8Sd77Af+90xoIWZaz/5h+k+Fqb3fgjQsgLh26DQZH21dYXptTysrsYrbklbM1v4It+8qxi/DZr08E4JqXf2RtTikDulnJZkC3OIZmJDBxQOphftDAaeomvdE2h9rRQUTOAJ7Cumv2D1V9pNH+ZGA2MACoBq5V1dUiMhh4w+fQ/sADqvqkiDwI3ADs8+67V1XnH/KHa0bI1ZTA+kuvR2KUqSkZh6Wq1s3u4kp2FVWRXVhJdlEVVx2bRa+kaJb8bxE/fTuffq6t9HVto7d7F2UST/Etq+nfLY7NpTZ21gxmW/QZbIvIYputH0VFSbxUUkVGYjRvlgxj9vxtwGLAatEDeOPnx5AU42T2N9t47cedDd0YVBWH3cb8XxyPzSb88/vtvPbjLiIjbPRLi2VEz0QGdo9DVRERXrxqHA57yE3IYnQAEbEDzwKnAtnAYhF5X1XX+hx2L7BCVc8TkSHe46eo6gZgtM95dgPzfMr9VVUfa8/4QzIpAWQkRjF/1V6q69xEOTpw8JwRMqrr3GQXVZFdVEl2YSV5+fmcPqY/wzNTWPzdZ3z23zfoLkWkSxHDpIiTpYjsAV/SKymTjJ0fMK7iZQrtaexy9OOT2PHscvZnqrfFbedRv2Fu7U7AaoWLBqKRhua1hOgI+qTENOwHEASbd39qnJNB6XEN270vKKtxkRjtYOZJA5l50kB6JkY3lPFlEpLRgvHAZlXdCiAirwPTAN+kNAx4GEBV14tIloikq2quzzFTgC2quqOD4gZCOClNHtSNbzcX8FN2CeP7pQQ7HCMIquvc5BRXsTu/mMK9Oyjft4uBIycwYWg/ti9byIp5T9BDiuhDEUdLEbFSw8Kk+QzPnMSgqp842vE6dRHxuGLTsSVm4Ew6il59rFaRXmfdC+c8QEpMCilA49v+Jw9J5+Qh6c3GdsaIDM4YkdHs/mmjezFtdPOddDKTY9ryozC6lggRWeLz/gVVfcHnfS9gl8/7bGBCo3OsBM4HvhGR8UBfIBPwTUrTgdcalbtVRK4ClgC/UtWiQ/8YTQvZpHTmiAwe/mg92/MrTFIKUzW1NeTm7CJ/zw7K9u2ipjCbyMGncMLEYyjb8CW7584kXYroL/vnNvsg8nkY2o/0iEpOit1BXUw6toT+kJKJJyWTU0cfAUDSpOth0vU4IuNoclBBXPeO+ZCG0XYuVR3Xwv6metA07jzwCPCUiKwAVgHLYf+IYRFxAucAv/UpMwv4k/dcfwIeB65ta/CtCdmklJEUhQjsLq4KdihGAKgquwqrKK2uY0SvRHTzZ0T883z6iOI71PA9iYaJxxCXmIqj2wAK43tQntyLuLRMErr35ezMowCIPnIa0Ue2MOllZFzz+wwjtGUDvX3eZwI5vgeoaikwA0CsNTK2eR/1zgSW+Tbn+b4WkReB/wY8ckI4KTnsNhKjHRRV1rZ+sNEpbcot47utBfy4rZDcbas4o+ojJCmTEb9+DOl3PKsG/BxPbHfi0jJJ7tGXlB59mRZv1WCkxwgG3GaWADCMJiwGBolIP6yOCtOBy3wPEJEkoFJVa4Hrga+8iarepTRquhORDFXd4317HnDwBIYBELJJCSA5xklhhUlKoaDW5WF1Tgnr95Q1TLPy2EersW38iGsjP+NoXYXbEUFp/2usAnYHo6/6c/ACNowQpaouEbkVWIDVJXy2qq4RkZu8+58DhgJzRMSN1QHiuvryIhKD1XPv541O/aiIjMZqvtvexP6ACPGk5KC4sq71A42g2LC3jPmr9vDjtkKW7yqius4ajHjmiB4kxzp5NHI2ic430YRMGPs77EddRbK5l2MESUtrL23fvp2zzjqL1avbpXIQcN7xQ/MbbXvO5/V3wKDG5bz7KoGDBryp6pUBDrNJIZ6UnOwpMWOVOoPiyloWby9i8fZCrjymL71TYli2s4i/fbaJoRkJXDoukzNj1jFq79tEVg+B2AEkTr4ZjroAGXRqx66JYxhGpxXaSSnWyWcb8oIdRpeVU1zF37/YzOJtRWzItSYVddptjM9KoXdKDGeP6slZgyKJX/sGLJkNRdushc8Kt1rzu/U6KsifwOgwH90De1cF9pw9RsKZjzS7++6776Zv377ccsstADz44IOICF999RVFRUXU1dXxf//3f0yb1rZVYKurq7n55ptZsmQJERERPPHEE5x00kmsWbOGGTNmUFtbi8fj4e2336Znz55cfPHFZGdn43a7+d3vfscll1xyWB873IV0UkqKdqAKq7JLGJnZtrViDP+pKtvyK/hxWyE/bi/k2AFpXDg2E4fdxrvLcziqbzLnjO7J0VkpHJmZ2DCYOc5WB88dDTUl0OdYOPl+a8mFCLPCmdH+pk+fzh133NGQlN58800+/vhj7rzzThISEsjPz+eYY47hnHPOQdowD+Gzz1qLHK5atYr169dz2mmnsXHjRp577jluv/12Lr/8cmpra3G73cyfP5+ePXvy4YcfAlBSUtLSqQ1CPCldMymLf3yzjR+2FZik1A48HuX2N1bw3ZZ88sutDiWpsc6GJQy6xUey8venNcxiQE05rJwDu5fCtGesOd/OfAQyRkP6sCB9CqNTaKFG017GjBlDXl4eOTk57Nu3j+TkZDIyMrjzzjv56quvsNls7N69m9zcXHr06OH3eb/55htuu+02AIYMGULfvn3ZuHEjEydO5KGHHiI7O5vzzz+fQYMGMXLkSH79619z9913c9ZZZ3H88ce318cNGyGdlDKTY4iPimBnYWWwQwl5Ho+yMruYz9fnUVxVxx+njcBmEzweZfKgbhzdL4Wjs1IY0C32gL8q7TaBvPWw5CVY+bo1M3b6CKguhagEawluwwiSCy+8kLfeeou9e/cyffp05s6dy759+1i6dCkOh4OsrCyqq9t2X7q5Sawvu+wyJkyYwIcffsjpp5/OP/7xD04++WSWLl3K/Pnz+e1vf8tpp53GAw88EIiPFrZaTUoi0huYA/QAPFhTWjzV6BjBmpF2KlAJXKOqywIf7sEyk2PYXWQG0B6q77YU8NbSbL7YkEdBRS02gQn9UvF4FJtNePbyVu77rH0f3rwS7E4Yfh6Muw56j2/zsgyG0R6mT5/ODTfcQH5+Pl9++SVvvvkm3bt3x+Fw8Pnnn7NjR9undZs8eTJz587l5JNPZuPGjezcuZPBgwezdetW+vfvzy9+8Qu2bt3KTz/9xJAhQ0hJSeGKK64gLi6OV155JfAfMsz4U1NyYc1xtExE4oGlIrKw0YyzZ2J1LxyENcfSLA6ea6lddI+PJK+spiMuFfJUlc155Xy2Po9LJ/QhIcrB8l1FLFqfywlHdOPkId2ZPKgbybHO5k9StAOWvgLdhsCoS2DASXDqH2H05RCb1mGfxTD8MXz4cMrKyujVqxcZGRlcfvnlnH322YwbN47Ro0czZMiQNp/zlltu4aabbmLkyJFERETwyiuvEBkZyRtvvMG//vUvHA4HPXr04IEHHmDx4sXcdddd2Gw2HA4Hs2bNaodPGV7avJ6SiLwHPKOqC322PQ98oaqved9vAE70Gf17kMNZT8nXb95ayZcb9/HDvc0vz9yVVde5+W5rAZ+vz+Oz9Xlke2uVL884mpMGd6eq1o3DLg3LVjfJ44bNi2DxP2DTJ1Yt6Jhb4PSHOuhTGKHIrKd0+A51PaVQ1qZ7SiKSBYwBfmi0q6lZaXsBzSalQOkeH8W+spoDVuXs6nKKq6hze+ibGsuuwkpmvLyYaIedSQNTufnEAZw0uDs9k6yF56KdfowPemsGrH0PYrvD8b+CsddAUu9WixmGYbSV30lJROKAt4E7Gs2RBP7NSouI3AjcCOB0ttBE1AbdEyLxKBRU1NA9Piog5ww1LreH5buKG2pD6/eWcf6YXjxxyWgGdo9j7vUTGNs3uW3rThXtgNhu4IyxktCwc2HIWRARmN+bYXRGq1at4sorD5y4IDIykh9+aPx3uNFe/EpKIuLASkhzVfWdJg5pdVZaAO+aHy+A1XzX5mibUJ+I8kq7VlLyXdzw/Fn/46fsEuw2YVzfZH575hCmDLXW+hERJg1s472edR/AuzNh1HSY+igMODnQ4RtGpzRy5EhWrFgR7DC6NH963wnwErBOVZ9o5rD3sRZ/eh2rg0NJS/eTAql7gjUQM7e0mhG9wneskqqyfm8Zn63P4/P1eWwvqOCHe0/BbhOundSPCLtw/KBuJEY3uTqQf1y18Onv4fu/Q8+jYOItgfsAhmEYfvCnpjQJuBJY5V0QCqz13ftAwyR/87G6g2/G6hI+I+CRNiMr1brft25PaUPtINy8uWQXf124sWGevxG9ErhsfB+q69zERkZw7pjmVzD1W/FO+M811sDXCTdbPepMU51hGB2s1aSkqt/Q9D0j32MUmBmooNoiJdZJz8QotuWH7wDaET0TGZaRwB2nDOLEwd1JT2iHZkp3HZTlwsVzYFjb5gIzDMMIlJCe0aFecqyT4jBb7K+suo7//rSHS8f3YVjPBF665ujAX8RdB6vesu4dpQ6AXyw3tSPDMIIqPJJSjJPCMEpKe0qqmPHyYjbllTO2b3LDXHMBVZIN/5kB2T9CYib0O94kJKNLa2k9JaPjhEdSinWyuzg8phpavbuE615dTEWNm5evObp9EtLGT2DejeB2wYWzrYRkGIbRCYRHUopxhMWy6J+vz2Pmv5eRFO3grZsnMqRHQuAv8vXjsOiP1lo0F71qNdsZRge45PnvDtp21pEZXDkxi6paN9e8/ONB+y8cm8lF43pTWFHLzf9aesC+N34+scXrBXI9pfLycqZNm9ZkuTlz5vDYY48hIhx55JH885//JDc3l5tuuomtW7cCMGvWLI499thWr2OETVJyUlpdh8vtaXm6nE6uus7NwO5xvHjVuPbpzACQMQrGzoAzHgFH1xnXZXQ9gVxPKSoqinnz5h1Ubu3atTz00EN8++23pKWlUVhYCMAvfvELTjjhBObNm4fb7e7wZkEROQNrkmw78A9VfaTR/mRgNjAAqAauVdXV3n3bgTLADbhUdZx3ewrwBpAFbAcuVtWigAevqkF5xMTEaKDM/mar9r37v1pQXhOwc3YUt9ujy3YUNrx3uT2Bv8jGharf/T3w5zWMFqxduzbYIeiQIUN09+7dumLFCj322GO1trZWZ86cqSNHjtRRo0ZpVFSU7tmzR1VVY2Njmz1Pc+Wefvppvffeew86Pi0tTaurqw87/qZ+hkCFtvDd6k1EW4D+gBNYCQxrdMxfgN97Xw8BFvns2w6kNXHeR4F7vK/vAf7cUhyH+gjdaoWP+gGjJVV1QY6kbapq3dw8dykXPfcdm/Osv6QCOn+f22U11c29AFbMtQbHGkYXUr+e0htvvHHQekorVqwgPT3dr/WUmiunqm1atbaDjAc2q+pWVa0FXgcat1EOAxYBqOp6IEtEWhvoOQ141fv6VeDcgEXswySlINlXVsP0F7/nk7W53Dt1KAO7xwX2AqV7YM406x7SmCvh2k9M7zqjy5k+fTqvv/46b731FhdeeCElJSWHtJ5Sc+WmTJnCm2++SUFBAUBD892UKVMalqlwu92UljaeLvSwRIjIEp/HjY32NzdBtq+VwPkAIjIe6Is1PRxY85Z+IiJLG507Xb0z9Xifuwfm4xwoLO4phVpS2pRbxoxXFlNQXsvzV4zltOH+L8Xsl9oKePEkqC6B8563xiEZRhcUqPWUmis3fPhw7rvvPk444QTsdjtjxozhlVde4amnnuLGG2/kpZdewm63M2vWLCZObLljRhs03Odphj8TZD8CPOWdpWcVsBxr7TyASaqaIyLdgYUisl5VvzrcoP1lklIQfLI2lxqXhzd+fgxHZiYF7sSq1lpHzliY8oA1f133ti9iZhjhZNWqVQ2v09LS+O67g3sBAi12Rmip3NVXX83VV199wLb09HTee++9Q4g2IFqdIFutlR5mQMP8ptu8D1Q1x/ucJyLzsJoDvwJyRSRDVfeISAaQ1x7Bm+a7DlTfbf2WEwfw0e3HBzYhleVazXUbPrLej77MJCTD6JoWA4NEpJ+IOIHpWJNmNxCRJO8+gOuBr1S1VERivSuMIyKxwGnAau9x7wP12fdqoF2ybljUlBK8Sam0kyYlVeWJhRuZ+8NO3r91EpnJMaTFRQbuAtu+greug5oy6/6RYRiHJBzWU1JVl4jcCizA6ok3W1XXiMhN3v3PAUOBOSLiBtYC13mLpwPzvJ03IoB/q+rH3n2PAG+KyHXATuCi9og/LJJSlMNOlMPWKee/q3G5ufutn3h3RQ4Xj8sM7Pgjj9vqyPDFw5A6EK56D9KHBe78htHFhMt6Sqo6H2v1Bt9tz/m8/g4Y1ES5rcCoZs5ZAEwJbKQHC4ukBJAS46SosnPVlIora7nxn0v5cVshd50+mFtOHBDY7qObPoHPH4IjL4GfPQGRAe7BZxiG0cHCJiklxTgp6mRTDT29aDMrdhbz1PTRTBsdgDWP6lUWQkwKHHEGXPU+9JtsdXAwDMMIcWHR0QGsdZWKOknzncdj9b686/TB/OemiYFLSB4PfPUXeGoU7NtoJaL+J5iEZBhG2AibpJQU4+gUzXcf/rSH8/7+LWXVdUQ77YzqnRSYE1fkw9wL4bP/g0GnQkJGYM5rGIbRiYRNUkqOCW5NSVWZ9cUWZv57GQ67DZe78Vi1w5CzAp47HrZ/A2f9FS54CSLbYUkLwwgjxcXF/P3vf29zualTp1JcXBz4gAy/hE9SinVSUlWH2xPAZOCnOreHe+et4s8fr+fsUT351/UTSI4N4JQ+S18GVxVcvxDGXWua6wzDD80lJbfb3WK5+fPnk5SU1E5RGa0Jm44OyTEOVK0BtCmBTAh+eOjDdbz24y5mnjSAX506GFsgJ1UFOPspOPMvZu46I7S9/LODtw0/F8bfALWVMLeJYS+jL4Mxl0NFAbx51YH7ZnzY4uXuuecetmzZwujRo3E4HMTFxZGRkcGKFStYu3Yt5557Lrt27aK6uprbb7+dG2+0pnnLyspiyZIllJeXc+aZZ3Lcccfxv//9j169evHee+8RHR3d5PVefPFFXnjhBWpraxk4cCD//Oc/iYmJaXZtpabWYTLCKCnVJ6KiytoOT0o3Tu7PkZmJnH9UZusHHyqTkAyjTR555BFWr17NihUr+OKLL/jZz37G6tWr6devHwCzZ88mJSWFqqoqjj76aC644AJSU1MPOMemTZt47bXXePHFF7n44ot5++23ueKKK5q83vnnn88NN9wAwP33389LL73Ebbfd1uTaSmvWrGlyHSYjjJJSUow3KVXUQrf2v97q3SW89uNO/jRtBD2TotsvIW38BBa/COf8DeIDPHGrYXSklmo2zpiW98emtlozas348eMbEhLA008/zbx58wDYtWsXmzZtOigp9evXj9GjRwMwduxYtm/f3uz5V69ezf33309xcTHl5eWcfvrpAHz22WfMmTMHALvdTmJiInPmzOHCCy8kLS0NgJSUlMP6bOEkbJJSSn1S6oAeeIvW5XLba8tJjnGSV1ZDj8R2XMF1x7ew5XOITm6/axhGFxAbG9vw+osvvuDTTz/lu+++IyYmhhNPPLHJdZUiI/dPB2a326mqqmr2/Ndccw3vvvsuo0aN4pVXXuGLL75o9thOug5TpxA2HR2SYqz579p7AO2c77Zzw5wlDOgWx7xbjm3fhASQsxzSh0NEAOfKM4wuID4+nrKysib3lZSUkJycTExMDOvXr+f7778/7OuVlZWRkZFBXV0dc+fObdje1NpKza3DZIRRUkr2uafUXp78dCMPvLeGk4ek88bPj6F7IOexa4rHY3UH73VU+17HMMJQamoqkyZNYsSIEdx1110H7DvjjDNwuVwceeSR/O53v+OYY4457Ov96U9/YsKECZx66qkHrNP01FNP8fnnnzNy5EjGjh3LmjVrDliHadSoUfzyl7887OuHC/Gut97hYmNjtaKiImDnU1UG3DufW04cyK9PHxyw8/pauqOQj1fv5Z4zhwZ22fLmFGyBvx1l3U866qrWjzeMTmTdunUMHTo02GGEtKZ+hiJSqaqxzRQJeWFzT0lEiIuMoKy6/e4pje2bwti+HXhDsrYc+k6CXi0tMmkYhhE+wiYpAcRHOSirdrV+YKjIGAUz5rd+nGEYHWbmzJl8++23B2y7/fbbmTFjRpAiCi9hlpQiKA2npOR2gT2sfkVGFxOOvcyeffbZDrlOsG6tBFvYdHQAawXa9my+61AeN/xlAHzzZLAjMYxDEhUVRUFBQZf9cj0cqkpBQQFRUe3cmaoTCqs/wxOiIsgpPnisQUjK3wjVxWbArBGyMjMzyc7OZt++fcEOJSRFRUWRmdmOs8R0UmGVlOKjHJTVND0uIeTsXmY99xwT3DgM4xA5HI4DZlAwOo6InAE8BdiBf6jqI432JwOzgQFANXCtqq4Wkd7AHKAH4AFeUNWnvGUeBG4A6v/KuNe77HpAhVVSSoiKoKQTrKkUEDnLwBkHqYOCHYlhGCFEROzAs8CpQDawWETeV9W1PofdC6xQ1fNEZIj3+CmAC/iVqi4TkXhgqYgs9Cn7V1V9rD3jD6t7SimxkZRWu6hze4IdyuHLWQ4Zo8EWVr8iwzDa33hgs6puVdVa4HVgWqNjhgGLAFR1PZAlIumqukdVl3m3lwHrgAAtne2fsPrGS4nzmZQ11I28CI66MthRGIYRenoBu3zeZ3NwYlkJnA8gIuOBvsABN7BEJAsYA/zgs/lWEflJRGZ7mwADrtWk5L14noisbmb/iSJSIiIrvI8HAh+mf1K9Uw0VhENSOuZmGDU92FEYhtH5RIjIEp/HjY32N9UHv3EXyEeAZBFZAdwGLMdqurNOIBIHvA3coaql3s2zsO5BjQb2AI8f7gdpij/3lF4BnsG6+dWcr1X1rIBEdBiSvTOFF4Z6UirJBrsT4roHOxLDMDofl6q2NM1LNtDb530mkON7gDfRzAAQayDZNu8DEXFgJaS5qvqOT5nc+tci8iLw38P7GE1rtaakql8BITGFbWpcmNSUvvwzPDsezPgOwzDabjEwSET6iYgTmA6873uAiCR59wFcD3ylqqXeBPUSsE5Vn2hUJsPn7XlAk61nhytQve8mishKrGz8a1Vd09RB3mrmjQBOZ+BXUq1fcbawvCbg5+5QOcutruBhNhLeMIz2p6ouEbkVWIDVJXy2qq4RkZu8+58DhgJzRMQNrAWu8xafBFwJrPI27cH+rt+PishorKbA7cDP2yP+QCSlZUBfVS0XkanAu0CT/ZhV9QXgBbBmCQ/AtQ+QHONEJMSb7+qqIG8dTDot2JEYhhGivElkfqNtz/m8/o4mvqdV9RuavieFqnZIz6vD7n2nqqWqWu59PR9wiEjaYUd2COw2ISnaQWE7rqnU7vauBo8Lepo1lAzDCE0iEisiNu/rI0TkHO+9qlYddlISkR7edsj6roU2oOBwz3uoUmKdoV1TylluPZuZHAzDCF1fAVEi0gtrPNQMrE5zrWq1+U5EXgNOBNJEJBv4PeCAhurghcDNIuICqoDpGsQZGFNjIykoD+GkNPhMiE6ChJ7BjsQwDONQiapWish1wN9U9VERWe5PwVaTkqpe2sr+Z7C6jHcKKbFOtuwrD3YYhy6pt/UwDMMIXSIiE4HL2d+Jwq8+DGE1owNAcqyTolC9p1RbAcvmQOmeYEdiGIZxOO4AfgvM8/b86w987k/B8EtKMQ6KK+tCcw2XnBXw/m2wd1WwIzEMwzhkqvqlqp6DtxXNOw/fL/wpG3ZJKSXWicujlNWE4Aq0OfXLVYwOahiGYRiHQ0QmisharAldEZFRIvJ3f8qGXVJKignhSVlzlkNCppleyDCMUPckcDrentiquhKY7E/BsEtKyTFWV/iiUFxXafcy6GW6ghuGEfpUdVejTW5/yoVfUooN0ZpSVTEUbTPjkwzDCAe7RORYQEXEKSK/xtuU15qwWnkW9i9fkR9q899FJ8FdW0DC7u8EwzC6npuwlmPvhTVr+SfATH8Khl1SSouLBCA/FAfQxgZldibDMIyAUtV8rDFKbRZ2f5bHRkYQ47SHXk3pmydh6SvBjsIwDOOwicijIpIgIg4RWSQi+SJyhT9lwy4pgVVbCrmk9OOLsO3rYEdhGIYRCKd5FxI8C6v57gjgLn8KhmlScoZWUirPg9Js08nBMIxwUT8j+FTgNVX1e6HYME1KkewrC6GkVD8zeC+zXIVhGGHhAxFZD4wDFolIN6Dan4LhmZTiI0Oro0POckCgx5HBjsQwDOOwqeo9wERgnKrWARXANH/KhmdSioukqLIWl9sT7FD8U1MGGUdCZFywIzEMwzhsInIR4FJVt4jcD/wL8Gs9nrBMSt3inKiG0LLopz8EN34Z7CgMwzAC5XeqWiYix2FNN/QqMMufgmGZlOrHKu0Lpc4O1uK9hmEYh01EzhCRDSKyWUTuaWJ/sojME5GfRORHERnRWlkRSRGRhSKyyfuc3EII9VMK/QyYparvAU5/Yg/PpBQfQgNoN3wML50GxY2niTIMw2g7EbEDzwJnAsOAS0VkWKPD7gVWqOqRwFVYsy+0VvYeYJGqDsJa4vygZOdjt4g8D1wMzBeRSPzMN+GZlOpndQiFHni7foDdSyG2W7AjMQwjPIwHNnvXMKoFXufgTgbDsBILqroeyBKR9FbKTsNqhsP7fG4LMVwMLADOUNViIIWuPk4JQmT+u5zl0H0YOKKCHYlhGKEhQkSW+DxubLS/F+Db9JLt3eZrJXA+gIiMB/oCma2UTVfVPQDe52bX2FHVSmALcLqI3Ap0V9VP/PlwYZmU4iIjiIywdf6kpGolJTNo1jAM/7lUdZzP44VG+5u6Qd14Ke5HgGQRWQHcBiwHXH6WbZWI3A7MxUpc3YF/icht/pQNuwlZAUTEO9VQJ7+nVLQNqovNoFnDMAIpG+jt8z4TyPE9wDsF0AwAERFgm/cR00LZXBHJUNU9IpIB5LUQw3XABFWt8F7jz8B3wN9aCz4sa0pQP4C2k9eUXLUweCpkjg92JIZhhI/FwCAR6SciTmA68L7vASKS5N0HcD3wlTdRtVT2feBq7+urgfdaiEE4cFE/N03Xwg4SljUlsMYqZRdVBTuMlnUfApe+FuwoDMMII6rq8t7HWQDYgdmqukZEbvLufw4YCswRETewFqtm02xZ76kfAd4UkeuAncBFLYTxMvCDiMzzvj8XeMmf+EW1zc2FAREbG6sVFRXtdv573v6JT9flseT+U9rtGoettgKcscGOwjCMECIilara6b84ROQo4DisGtJXqrrcn3JhW1NKi4uksKIGt0ex2zrhwFSPGx4bDMfcDCffF+xoDMMwDpuIpPi83e59NOzzZ7bwsE1K3eIj8SgUVdY2jFvqVAo2Q20ZpPQLdiSGYRiBshSrt159TaC+KU68r/u3doKwTUr7l0Wv6ZxJqX65CtMd3DCMMKGqh/1Xdvj2vqsfQFvWSbuF714GjlhIOyLYkRiGYXQa4ZuU4vfXlDqlnOWQMQps9mBHYhiG0WmEffNdp12BdtwMcEQHOwrDMIxOJWyTUkJUBE57J55qaPRlwY7AMAyjXYlId6BhYk9V3dlambBtvrOmGnJ2zjWVCrZA/iZr7jvDMIwwIyLniMgmrKmLvsTqGv6RP2XDNilB/VRDnbCjw/+ehhenBDsKwzCM9vIn4Bhgo7dH3hTgW38KhndSiovsnGsq5SyHnqPNarOGYYSrOlUtAGwiYlPVz4HR/hQM86Tk7Hz3lOqqIXeNmRncMIxwViwiccBXwFwReQpraYxWtZqURGS2iOSJyOpm9ouIPO1dz/0n73xHnUJaXCQFFbV4PJ3o3k3uGvC4zKBZwzDC2TSgErgT+Bhrwb+z/SnoT03pFeCMFvafCQzyPm4EZvlz4Y6QFheJ26MUV9UFO5T9cpZZzz07Te42DMMIGBGxA++pqkdVXar6qqo+7W3Oa1WrXcJV9SsRyWrhkGnAHLWmG//eu05HRv2yucHkO4A2JdbZytEdZPh5kJhpPQzDMMKMqrpFpFJEElW1pK3lAzFOqbk13Q9KSt615G8EcDrbP0nsn2qohiPS49v9en6JTYPBZwY7CsMwjPZUDawSkYVAwxpFqvqL1goGIin5vaa7dy35F8BaTykA125Rt/pZHTpLZ4faClj8Egw7B5Kzgh2NYRhGe/nQ+/Dl13d+IJJSq+vBB8v+mcI7yVilPT/Bwt9Zk7CapGQYRvhKUtWnfDeIyO3+FAxEl/D3gau8vfCOAUo6w/0kgMRoBw67dJ7578xyFYZhdA1XN7HtGn8K+tMl/DXgO2CwiGSLyHUiclP9eu/AfGArsBl4EbjFr5A7gM0mpCdEsaekKtihWHKWQUIviE8PdiSGYYQxETlDRDZ4h+rc08T+RBH5QERWisgaEZnh3T5YRFb4PEpF5A7vvgdFZLfPvqlNnPdSEfkA6Cci7/s8PgcC1vvu0lb2KzDTn4sFQ2ZyNNlFnSUpLTe1JMMw2pW3S/azwKlYt1cWi8j7qrrW57CZwFpVPVtEugEbRGSuqm7AO/OC9zy7gXk+5f6qqo+1cPn/YXVySwMe99leBvzkT/xhO0t4vd7JMXy5cV+ww7A6ORTtgFHTgx2JYRjhbTywWVW3AojI61hDd3yTkgLxIiJAHFDIwTMuTAG2qOoOfy/sPXaHiFzXKAkiIicCX7R2jrCeZgigd0oMeWU1VNe5gxuIMxZ+uwvG/zy4cRiGEeoiRGSJz+PGRvubG6bj6xlgKFantFXA7arqaXTMdOC1Rttu9c7cM1tEkluI8U0R+Y23r0G0iPwNeNifD9cFkpK1kF6naMJzRENUQrCjMAwjtLlUdZzP44VG+/0ZpnM6sALoidVc94yINHw5iYgTOAf4j0+ZWcAA7/F7OLB5rrEJQB+s5rzFWMlvUoufyiv8k1JyDAC7iiqDG8gXj8C3Twc3BsMwugJ/hunMAN5Ry2asdY+G+Ow/E1imqrn1G1Q1V1Xd3hrVi1jNhM2pA6qAaKxF/rY1URNrUtgnpUxvUgp6TWn53P1dwg3DMNrPYmCQiPTz1nimYw3d8bUT654RIpIODMbqRV3vUho13YlIhs/b84AmJ+n2iaEKOBo4DrhURN7yJ/iw7+jQPT4SZ4SN7MIg1pQq8qFkJ4y/IXgxGIbRJaiqS0RuBRYAdmC2qq6pH8ajqs9hLcL3ioiswmruu1tV8wFEJAar517jG+CPishorKbA7U3s93Wdqi7xvt4LTBORK/2JP+yTks0mZCZFB7f5rr6GZNZQMgyjA6jqfKwxpL7bnvN5nQOc1kzZSiC1ie1+JRWvpSJyBdBfVf8oIn2ADf4UDPvmO4DMlBh2FQax+S5nOSCQMSp4MRiGYXScvwMTsZoBwRqn9Kw/BbtEUuqdHE12MGtKHjf0OQYiO8lM5YZhGO1rgqrOxJotHFUtAvxaGqJLJKXM5BiKKusor/FrNd7AO+m3cO3Hwbm2YRhGx6vzzgihAN5ZI0zvu3p9U60eeNv2VbRypGEYhhEAT2NNT9RdRB4CvgH+nz8Fu0RSGtMnCYAlOwo7/uIbPoJnxkPh1taPNQzDCAOqOhf4DdYsDnuAc1X1Py2XsoR97zuAHglRJEY72JxX3vEX370UCjZDXI+Ov7ZhGEaQqOp6YH1by3WJmpKI0Dslmt3FQeiBt3sZdB8KzpiOv7ZhGEaI6RJJCSA9Poq80g5e7E/VLFdhGIbRBl0mKXVPiCSvrLpjL1q8E6oKTVIyDMPwU9dJSvFR5JfXUuf2q1diYKgbRl0KfSZ23DUNwzBCWJfo6ADQIzEKgL0l1fRO6aD7Oyn94bznWj/OMAzDALpQTemIdGs2hXV7SjvuouV51n0lwzAMwy9dJikNy0jAJrB6d0nHXNDjgb+NhQX3dcz1DMMwwkCXSUrRTjuDeySwbGdxx1ywcAvUlEL6sI65nmEYRhjoMkkJYHjPBDbllXXMxXYvs55NzzvDMAy/damklJkcTW5pDZW1HTAxa85ycMRA2uD2v5ZhGEaY6FJJ6eisFAC+2ZTf/hfLWQ49jgR7l+ngaBiGcdi61Dfm+H4pRDlsfL+1kNOGt/NcdMfeBtKlcr5hGMZh61Lfmg67jazU2I65rzT0LBgytf2vYxiG0YiInCEiG0Rks4jc08T+RBH5QERWisgaEZnhs2+7iKwSkRUissRne4qILBSRTd7n5PaIvUslJYDjBqbxvy0FVNW62+8i+zbArsVWt3DDMIwO5F1c71ngTGAYcKmINO4GPBNYq6qjgBOBx0XEd2XYk1R1tKqO89l2D7BIVQcBi7zvA67LJaUJ/VNxe5S1e9pxvNIPz8O/zm+/8xuGYTRvPLBZVbeqai3wOjCt0TEKxIuIAHFAIdBaD7BpwKve168C5wYsYh9dLikdmZkIwE/Z7ZiUcpZBxiiwdbkfr2EYwdcL2OXzPtu7zdczwFAgB1gF3K6q9U07CnwiIktF5EafMumqugfA+9y9PYLvct+a6QlRdI+P5LP1ee1zAVcN7F1txicZhtFeIkRkic/jxkb7pYkyjec7Ox1YAfQERgPPiEiCd98kVT0Kq/lvpohMDlzoretySQlg+tG9+XpTPjntsehf7hrw1EGvowJ/bsMwDHCp6jifxwuN9mcDvX3eZ2LViHzNAN5Ry2ZgGzAEQFVzvM95wDys5kCAXBHJAPA+t8tf9l0yKZ0yLB2AD3/aE/iT55iZHAzDCKrFwCAR6eftvDAdeL/RMTuBKQAikg4MBraKSKyIxHu3xwKnAau9Zd4Hrva+vhp4rz2C71LjlOqN6JnIkB7xPDR/HfFREUwf3ydwJz9yOnQbCkl9A3dOwzAMP6mqS0RuBRYAdmC2qq4RkZu8+58D/gS8IiKrsJr77lbVfBHpD8yz+j8QAfxbVT/2nvoR4E0RuQ4rqV3UHvGLBmlphdjYWK2oqAjKtQGe+3ILj3y0HoBtD0/F+0swDMPo1ESkUlVjgx1He+mSzXcApwxNb3i9eHtRYE5aWwmL/gR56wNzPsMwjC7Gr6Tkx+jgE0WkxDsCeIWIPBD4UANrQLdYzh3dE4A5320PzEn3roKvH4PCrYE5n2EYRhfT6j0ln9HBp2L16lgsIu+r6tpGh36tqme1Q4ztQkR4cvoYYiMjeH3xLurcHhz2w6w45iy3nk0nB8MwjEPiz7ewP6ODQ9a4rGTcHuXzQIxbylkG8RmQkHH45zIMw+iC/ElK/owOBpjondzvIxEZ3tSJROTG+gFfLlcHrGnkh9OG9SA+KoK73vqJOvdhzlWXs9zUkgzDMA6DP0nJn9HBy4C+3sn9/ga829SJVPWF+gFfERGdozd6bGQEP5/cn5KqOj5Zk3voJ6qrhvJc6GkGzRqGYRwqf5JSq6ODVbVUVcu9r+cDDhFJC1iU7ez64/sjAv9ZuouiitpDO4kjCn6zHY69NaCxGYZhdCX+JKVWRweLSA/vbLOIyHjveQsCHWx7iXLYOXVoOl9s2MfUp7+mouYQmxZtNnBEBzY4wzCMLqTVpKSqLqB+dPA64M360cH1I4SBC4HVIrISeBqYrsEalXuInrnsKKaN7smekmrW7Slt+wk+/YM1RskwDMM4ZH7d2PE2yc1vtO05n9fPYE2FHrKcETZ+PnkA763IYWdhJeOyUtp2gjXvWMtVGIZhGIesy87o0JQeiVEAPL1oEx5PGyp6lYVQtN10cjAMwzhMJin5SIl1cv1x/dheUMmTn270v6CZGdwwDCMgOke/7E7kvp8NZW9pNU9/tpmeSdH+zSDeMJPD6HaNzTAMI9yZmlIjIsLdZwwB4J53VvHVxn2tF4qIggEnQ1RiO0dnGIYR3kxSakLvlJiG1yt3Fbde4Njb4Mp57ReQYRhGF2GSUjOW3H8KAD9uL2z5wNDq+W4YhtGpmaTUjLS4SO46fTBfb8on654P+evCZjo+bPwYHh8C+zZ0bICGYRhhyCSlFsyYlNXw+qlFm5o+KGe5NeddYmbHBGUYhhHGTFJqQYwzglUPntbwftQfPuHn/1xyYOeH3cug2xBwhu3qxIZhGB3GJKVWxEc5WPmAlZhKqupYsCaXGa8sprCi1rqflLPcDJo1DKNT8WO18EQR+cC73NAaEZnh3d5bRD4XkXXe7bf7lHlQRHb7rDA+tT1iN0nJD4kxDhbfZ3V8iHHacXuUo/60kDkLvoHKfDM+yTCMTsNntfAzgWHApSIyrNFhM4G13uWGTgQe90647QJ+papDgWOAmY3K/lVVR3sf82kHJin5qVt8JOv/dAZr/nA6k4/oBsBzX2wlf/gMyDqu4bjXf9zJBbP+R3ZRJXtLqnn9x52UVNUFK2zDMLoef1YLVyDeu7pDHFAIuFR1j6ouA1DVMqxJuJta1LXdmBkd/OXxELXod3DEacy55nh+3FHCxc9/x/V5FzGv25CGlRCf/HQTe0urOe7PnzcUXbg2l5euOTo4cRuGEW4iRGSJz/sXVPUFn/dNrRY+odE5nsFagigHiAcuUdUDlt4WkSxgDPCDz+ZbReQqYAlWjarocD5IU0xNyV+FW2DZqzBnGjw+hPHrHuaJEVtZsyuPS57/ntW7S3B71LrX1Mii9XnUuNxBCNowjDDkql/B2/t4odF+f1YLPx1YAfQERgPPiEhCwwlE4oC3gTtUtX4tn1nAAO/xe4DHD/NzNMkkJX+lDYK7NsPFc6DPMbD0Vc7ffD/LI2/i5OxnuPuZf/LUpxupdXu49aSBDcUum2DNnbd1X0WwIjcMo2tpdbVwYAbwjlo2A9uAIQAi4sBKSHNV9Z36Aqqaq6pub43qRaxmwoCTYK3FFxsbqxUVIfxFnbMSXphMTmR/ulXvwCFutnp68E3kZC697k7yo/uRHONk/d4yzn32W04dls6LV40LdtSGYYQ4EalU1WbHoIhIBLARmALsxlo9/DJVXeNzzCwgV1UfFJF0YBkwCmvF8FeBQlW9o9F5M1R1j/f1ncAEVZ0e0A+HqSkdunxrhoeeM+ZQe+cG7q67gRxN5Yq6t3A8N5GMf59C1PdPMiLamqZo4dpcjnxwAaXVbe/08Oznm5n2zDeHvky7YRhdhp+rhf8JOFZEVgGLgLtVNR+YBFwJnNxE1+9HRWSViPwEnATc2R7xm5rSofr4XljyEvw2G+wOVmWXcPYz3/DxdUcwpPAzWP027LLuD1Z2G83jOSP4r/sYcknhd2cN47rj+jWcqsblJjLC3uRl1uaUMvXprwE468gMnp4+huyiKvqkxjR5vGEY4a21mlKoM0npUM0+EzwuuH5hwyZVxeph6VW8E9bMsxLUnpV4EBZ7BvO++1g+co/nqlPGUlnr5oWvtvLXS0Zx3piDpyp6YuFGnm5iiqNPfzmZ3ikx/PGDtfx88gCTpAyjizBJqZ2EfFLa9hW462DgFP+Oz9+Ee9Xb5H77L3q6duFSG996RvCBZyIL3EdTbY9l00MHDpB+8P01vPK/7RydlcxVE7O47bXlDfuuO64fkwamcu0rSxiVmch7tx7X+IqGYYQhk5TaScgnpUOUV1rFhws/ZXD+QgbkLSDdvReXOFjkGoV72PlMveAacMayq7CS4x/9nISoCD658wRSYp2c9NgXRNiFmjoPe0urG87ZLT6SH++dcmAtzTCMsGSSUjsJ6aS0dxWU7YX+J4H90Mcfu90eXLsWo6vepmTJG6RLMVVEsiHxOHIyp3LH0jQW/ea0hkUHq2rd2Gzww9ZCrpr94wHn6p0SjccDZdV1zL7maMZlpRzWR2ys1uXh3z/soEdiFBMHpJEY7Qjo+Q3D8I9JSu0kpJPSh7+Gla/BPbvAFpgOjJ+tzeH5f/6bs+3/Y6r9B1KknAqJJXbUeTDifOh3wgEJcN7ybHomRjMkI4FRf/jkgHON7p3EuzMnBSSueu8sy+aXb64EICs1hi/uOimg5zcMwz8mKbWTkE5KL06BiCiY8WFAT/uXBevZXVTFf1fsZJJtDb/vt47++V9ATSnYIiA+w3okZEBCL+/rnuTbUvk+38kXu+1sKHCxancJD503gtOH9yAtLvKwYtpbUs2/f9jB059tPmD7ol+dwIBucYd1bsMw2s4kpXYSsknJVQsPZ8KEG+G0/wv46avr3Az53ccArHzgNBIdbtj8KexeCmV7oHQ3lO6B0hyoO/jnV+1MZmt1Ans1mb2aAgk9mT5lPLaEXpDQk/e3KjEJqZwyvEersVTVuhn6gBVLaqyTmScNpNbt4ZGP1hMZYePTX57Q0LRoGEbHCPekZCZkbat968BdAz3HtMvpoxx27ps6lO4JkSTGOAAHDD3LevhStWpQpXugLMdKUqV7cJTsJm7nFrrn7WSkbSvdKkvhg381FDsHqFInO22pxKb1Jjm9L7ZEK2FZNa9ekJCBxnbjypcWA3DG8B7833kjGmpdtS4PTyzcyJTHv+Svl4zmZ0dmtMvPYm9JNbUuD89+vplzRvdk0sC0Qz7Xeyt2c/vrK7j5xAHcfcaQAEZpGEYgmZpSWy19BT64HX6xAlL6tXZ0ULncHiY/vABbRS7pFJIhhQyLqyDFk09cTR7pUkgPisiwFRHBgbNFqNjI9SSyV5MZdUR/JCoRopIgKhGik9hTE8kfP82hlBhmnjmWY4cP2L/fZg0EXrBmL6t3lzBtdE+25VdywxxrYuO/X34Upw5LxyaC3dZ0j8G80mpOfvxLyhvNYjG8ZwJl1S7+dO4Isosq2ZRbzoPnDG/x57BhbxmnP/lVw/sbju/Hb88ciq2ZaxtGZxbuNSWTlNrK7YJ96yF9OIRIF+yNuWV8smYvKbGRnH9UL5x2G19u2seMl62akOAhhTJ6SBE9pIAeUkS6FJJBIWf1E6LdZVBdDNUlUFUM2vKM56UaTaUtnkJ3NCUaSykxjZ5jKdFYunXrxn0XHGslsvqE5oimxu3hgXfX8MaSXUQ77EwamMqn6/Kavd6vTzuCW08e1OS+3NJqJvy/RUTYhFtPHsiTn1oDkW84vh/3/azxumfB9fBH6/hpVwkjeiXwq9MGE+VoepYPo2szSamdhGxSCiO1Lg8uj4e/LNjAy99uB2BU7yRW7ioG4IQjuvHqtY0mAlaF2gqoLiZ7zx5++eqXJEoFCVRaz1JBIhUkSCWJVHBEkofKkgISpII0exVOT1WLMdVoBKXEUKqxaGQ8A3ulgyMaT0QMVUTy9fYKMtPT+HRzGZXqpJIoqjSS608eTt+MNCKj47E5YxBnLHOX7+Pxz7OpJJJZV0/ipKHpFFfWMvqPCxs+3+xrjm62ttaYqrKjoJLF2wuZOjKD2MjWW79/yi6mzq2M7Zvc7DEllXX86cO1vLU0u2HbL04eyC9PG+xXXF3Zx6v30iclhkHpcaiCMyL8p/M0SamdhGRSqquChQ/A6MvDbgn0nOIqusdHEmG3sWFvGZ+uy+XGyf1x2Fv+T15aXcdN/1zKrqJKyqpdHJEez+/PHobTbiPKYad3Sgyrd5fw14Ub+eO5I+gVHwHVJazduoPf/vubAxNaw3MFybZKpgyIwemutjp01FVBbeX+167qFuNqkiMGHDGUayQ5FVBJJD3SUkmIT6CoLoKMbqnYnDHgiGb/kjT7/398uXEfG/aWAdA3JYbTh6cfeH7v/yWXx8OCNXuprnNTXFmHoGSlxpDqvSc3KjMRsMaU2QTeW5FDjcuD0y6MH9iDtQUeftrnxhGdwDUnjaB7ahq2qHiIjKPUE4krIpaU5FQrTp/a+h8+WENFjYs7TjmCnknRbf/5YHVuWbqjiJ5JUfRvx96VJZV1lFbX0Ssputlm1LLqOnJLq/l49V5EhL6pMZx1ZE/Aapr+cVshl/1j//pz0Q47MyZlMWVod8b2Dew4vc7EJKV2EpJJaddieOkUuGTuwR0PjDZRVe54YwWfrs3lzJEZvLU0m4SoCMb3S+HTdXk8esGRXHx07+ZP4HFDXSXUVbF0czYvLlqNu6aS8vJSYqgmmlqipYY/nJFFjNR6E5r1cNdUsGFXLgVFxTi1uuH4eHst3aPcUFd9wCppboUa1/5FOX3/x9hsNqIibA2zaShCjctNnVsbraq2/4w2gQi7vWHhR0VweJO43VNrxekPsaPOWDyOOLIr7RS6IinXKGpsMRw3PIuo2ESIjANnHETGe5+bee+MY2dRDZMf/7Ih1vioCB48ezjnH9WrxdlC6twe3B5tsblx3Z5SPt+Qx6Mfbzho35Qh3Vm3p5SckmoibMLMkwby47ZCvttacNCxL109jr6pMVz+jx/ILa0BoH+32APWK3NG2Fhwx2T6pR3e93ZJZR1r95Ty3JdbWJldzMPnjeTMkYfXqef7rQX8+j8refTCIzl2wKF13DFJqZ2EZFL64Xn46Ddw51pI7NBl67uUgya29ZPHo9w8dym9k63mnHPH9Gp29nWwut/f+u9ldE+I4ssN+9hdvL9pMS3OydmjevLu8t0UVVrLjUTYhEW/OoGkaCej/rh/wPKIXgncMeUIVu0uYfXuEhatz6NPSgy/mDKIaaN78s3mfP7w/hoKK2qpqHXj9hz4f+6gtbbcLnLzC/jspy289Nkq4qgmVqqsZ6qIlWr6xnk4uX8MX67aRpzPtl4xbvbuyyfRXk2SvQanuxK7tm3JE7cKHrHhVhtubDgiHDgdEVYHFrE3PFd7oLjKTUUdeLARERFBj6QYop2OA471YGPJzhJqPYIHG3HRUdQplFR7cGPDg6CNHvu3gWIjNjKCsmr3QcdmpsRwwuB0PAib8sqpdilLdxSjCOmJ0XRPiGJcViorskspqKhlQr9UthVW0S8tjoRoJxW1Lkoqa8lIiGZTXhm9kqOJddjIK6vhraXWiuLi/fNCgKEZ8ZwwqD6Z+PweG3+PNrzXhvcbc8v4dvM+ANJHn87UC69r0++lnklK7SQkk9K8m2DLZ/CrDSHTycHwj8vt4fhHP2dPycHNgt3jI/n75UcdNHVTTnEVjy3YwDvLdx+w/fhBacy5dvwBidXjUVwexWEXcktreOC91VTUuvjjtBH0SYlpspm0us7NyAcXMKZPMvdNHUp2URU1LjcL1+by0eq9Bxz7wpVjOc079uwfX2/l/z5c17DPSV1DMqtPbHGyP8HFU0U0NdjxMD4rkUn9k8HjpqCsii/W76G0soapw7uDWtuSo+1s31fKnuIK7Hiw0o4HO4oNDyN7xpESHUGEzcO63cWUVdZgEw+JkTayUiKJsgMeNxU1tUSgVFTX4lEPqbFOqmvrKK+uIzE6AqcNQK3vdfVQXlNnNXniISrCRnykHZs3baEebyJQal1u6lxuVBXbQelOEcAm1j5PwyW8vysR7CLUef9wUASn3QYi1Lk9eBQi7DZsNsGGNHwPeKNoKGMTQbzn9GDVJmtc1hFRzgick2bCifcc/A/RDyYptZOQTErPToDkLLjsjWBHYrSDwopa3B6lW3wkZdV1/LC1kAn9U4iLjGix5vbLN1awJqeUB88ZjjPCxqjMRCJauRd3OJbuKOSa2YuJdNh58aqxjOlzcCeKOreH1xfv4ov1eVxxTF9SYp3ERkZQ6/I0rM91ytDufLouj6QYB+nxUYzMTOTh80cekCA355VxyhNfHXT+ek9NH83yncXcctIACitqOePJr5s8bkiPeN6dOemwexSWVteREOXfvIvb8ys48bEvAGvqrQvHZnL/u6sPOMZhF9ITosguOrgDzlPTRzNt9P4WkYoaFxfM+h/rvfcVn7xkNIN7xHP/u6tZuqPooPKjMhNBpKHjEMBdpw9m5kkD/Yq/OSYptZOQS0quWvjbWBhzBZx4d7CjMYxDlldajc0mpMVFUlRRS0K0o8UeiN9uzufhj9aRHOPk1GHpFFbUMmlgGkc3Menvsp1FbNtXwd1v/0RsZARTR2bw4DnDcNhsQRkXtq+shiXbCzljRI8D/rD49w87WbQul+uO68eIzETKql30Sorm9teXs3JXMQ+ffyQTB6QedL6y6jo+XZfLve+spqpu/9CIaaN7clSfZGpcbnKKq3nlf9sPKHf7lEEcnZXCcYMOfQB4PZOUABE5A3gKsAP/UNVHGu0X7/6pQCVwjaoua+mcIZeU6nncDYNDDcNoWll1Xas1zM7Kn3uaX2/ax+fr9zGiVwLJsU5OPKLbAWXqO3+4PUqM0x7Qn0OXT0oiYgc2AqcC2cBi4FJVXetzzFTgNqykNAF4SlUntHTekE1KhmEYQeRPUvKjIpEI/AvogzXd3GOq+nJLZUUkBXgDyAK2Axer6sHtlofJn4bv8cBmVd2qqrXA68C0RsdMA+ao5XsgSUTaZ0I0wzAMo1neisSzwJnAMOBSEWk8fclMYK2qjgJOBB4XEWcrZe8BFqnqIGCR933A+ZOUegG7fN5ne7e19RhE5EYRWSIiS1yutnVTNQzDMPziT0VCgXjvrZc4oBBwtVJ2GvCq9/WrwLntEbw/SampxtDGbX7+HIOqvqCq41R1XESEmaDcMAzjEETU/3HvfdzYaL8/lYRngKFADrAKuF1VPa2UTVfVPQDe5+4B+TSN+JMZsgHfofWZWB+krccYhmEYh8+lquNa2O9PJeF0YAVwMjAAWCgiX/tZtl35U1NaDAwSkX4i4gSmA+83OuZ94CqxHAOU1GdUwzAMo0P5U0mYAbzj7QewGdgGDGmlbG59XwHvc/NT9x+GVpOSqrqAW4EFwDrgTVVdIyI3ichN3sPmA1uBzcCLwC3tEaxhGIbRKn8qEjuBKQAikg4MxvoOb6ns+8DV3tdXA++1R/Bm8KxhGEYI8bNL+FTgSaxu3bNV9aH6SoSqPiciPYFXgAysJrtHVPVfzZX1bk8F3sTqRr4TuEhVCwP++UxSMgzDCB1dfvBsu11YxAO0vOJb8yKAUO1THqqxh2rcELqxh2rcELqxh0Lc0aoatqsZBi0pHQ4RWdJK75NOK1RjD9W4IXRjD9W4IXRjD9W4w0nYZlvDMAwj9JikZBiGYXQaoZqUXgh2AIchVGMP1bghdGMP1bghdGMP1bjDRkjeUzIMwzDCU6jWlAzDMIwwZJKSYRiG0WmEXFISkTNEZIOIbBaRdlnP41CJSG8R+VxE1onIGhG53bs9RUQWisgm73OyT5nfej/LBhE5PXjRW+uwiMhyEfmv932oxJ0kIm+JyHrvz35iKMQuInd6/52sFpHXRCSqs8YtIrNFJE9EVvtsa3OsIjJWRFZ59z3tXTohGLH/xfvv5ScRmSciSZ0x9i5JVUPmgTXtxRagP+AEVgLDgh2XT3wZwFHe1/FYK/YOAx4F7vFuvwf4s/f1MO9niAT6eT+bPYjx/xL4N/Bf7/tQiftV4HrvayeQ1Nljx1oOYBvWQEiwpm+5prPGDUwGjgJW+2xrc6zAj8BErKltPgLODFLspwER3td/7qyxd8VHqNWU/Fm8KmhUdY+qLvO+LsOawLYXzS+ONQ14XVVrVHUb1oS24zs0aC8RyQR+BvzDZ3MoxJ2A9aXzEoCq1qpqMSEQO9bsAdEiEgHEYM3G3CnjVtWvsBaC89WmWL0zSyeo6ndqfcvPoZ0WimstdlX9RK3JpgG+x5oNu9PF3hWFWlLya4XbzkBEsoAxwA80vzhWZ/o8TwK/ATw+20Ih7v7APuBlb9PjP0Qklk4eu6ruBh7DmthyD9ZyL5/QyeNupK2x9vK+brw92K7FqvlA6MUedkItKQV9ASp/iEgc8DZwh6qWtnRoE9s6/POIyFlAnqou9bdIE9uC9XuIwGqamaWqY4AKrKak5nSK2L33X6ZhNRH1BGJF5IqWijSxrdP92/dqLtZO9xlE5D6sue7m1m9q4rBOGXu4CrWk1OlXuBURB1ZCmquq73g3N7c4Vmf5PJOAc0RkO1aT6Mki8i86f9z1sWSr6g/e929hJanOHvspwDZV3aeqdcA7wLF0/rh9tTXWbPY3k/luDwoRuRo4C7jc2yQHIRJ7OAu1pOTP4lVB4+2N8xKwTlWf8NnV3OJY7wPTRSRSRPoBg7BupnYoVf2tqmaqahbWz/QzVb2CTh43gKruBXaJyGDvpinAWjp/7DuBY0QkxvvvZgrWPcjOHrevNsXqbeIrE5FjvJ/5KtppobjWiMgZwN3AOapa6bOr08ce9oLd06KtD2AqVq+2LcB9wY6nUWzHYVXpfwJWeB9TgVRgEbDJ+5ziU+Y+72fZQCfozQOcyP7edyERNzAaWOL9ub8LJIdC7MAfgPXAauCfWD2+OmXcwGtY977qsGoN1x1KrMA47+fdAjyDd1aZIMS+GeveUf3/0+c6Y+xd8WGmGTIMwzA6jVBrvjMMwzDCmElKhmEYRqdhkpJhGIbRaZikZBiGYXQaJikZhmEYnYZJSoZhGEanYZKSYRiG0Wn8fwOm7ts4Xb4KAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# static plot after training, useful when coming back after training finished\n",
    "plotter.static_plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "15fe9b7682874d52b5d604f1dee1cfe5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Testing:   0%|          | 0/27 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[{'test_loss': 0.05571528896689415, 'test_acc': 0.982699990272522}]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.test(verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "756dc39967c94b2bb0b3bd036e0e18e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/313 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model.eval()\n",
    "all_targets = []\n",
    "all_scores = []\n",
    "for images, targets in tqdm(mnist_datamodule.test_dataloader()):\n",
    "    with torch.no_grad():\n",
    "        scores = model(images).cpu()\n",
    "        all_scores.append(scores)\n",
    "        all_targets.append(targets)\n",
    "        \n",
    "scores = torch.cat(all_scores)\n",
    "targets = torch.cat(all_targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.9827)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy = (scores.max(1)[1] == targets).sum() / float(len(targets))\n",
    "accuracy"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:dev_pytorch17]",
   "language": "python",
   "name": "conda-env-dev_pytorch17-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}