ISIC2k19_inceptionresnetv2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ISIC2k19_inceptionresnetv2.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/talhaanwarch/ISIC2K19/blob/master/ISIC2k19_inceptionresnetv2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZCq6S218a683",
        "colab_type": "text"
      },
      "source": [
        "#data download"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6FV6Y73WWApL",
        "colab_type": "code",
        "outputId": "c1cd7a77-8040-459b-ed3d-c7703dbf560c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 217
        }
      },
      "source": [
        "!wget https://s3.amazonaws.com/isic-challenge-2019/ISIC_2019_Training_Input.zip"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-04-29 01:44:56--  https://s3.amazonaws.com/isic-challenge-2019/ISIC_2019_Training_Input.zip\n",
            "Resolving s3.amazonaws.com (s3.amazonaws.com)... 52.216.133.61\n",
            "Connecting to s3.amazonaws.com (s3.amazonaws.com)|52.216.133.61|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 9771618190 (9.1G) [application/zip]\n",
            "Saving to: ‘ISIC_2019_Training_Input.zip’\n",
            "\n",
            "ISIC_2019_Training_ 100%[===================>]   9.10G  36.3MB/s    in 3m 54s  \n",
            "\n",
            "2020-04-29 01:48:51 (39.9 MB/s) - ‘ISIC_2019_Training_Input.zip’ saved [9771618190/9771618190]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jRbzfyYiXEKN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%capture\n",
        "!unzip ISIC_2019_Training_Input.zip;"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NrBW0Zn0XYIc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "rm ISIC_2019_Training_Input.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8Nx7lSdnWGyV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 217
        },
        "outputId": "b6695888-b257-4e01-e857-c0bae3f2b207"
      },
      "source": [
        "!wget https://s3.amazonaws.com/isic-challenge-2019/ISIC_2019_Test_Input.zip"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-04-29 01:53:48--  https://s3.amazonaws.com/isic-challenge-2019/ISIC_2019_Test_Input.zip\n",
            "Resolving s3.amazonaws.com (s3.amazonaws.com)... 52.216.129.181\n",
            "Connecting to s3.amazonaws.com (s3.amazonaws.com)|52.216.129.181|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 3823247963 (3.6G) [application/zip]\n",
            "Saving to: ‘ISIC_2019_Test_Input.zip’\n",
            "\n",
            "ISIC_2019_Test_Inpu 100%[===================>]   3.56G  45.1MB/s    in 89s     \n",
            "\n",
            "2020-04-29 01:55:18 (40.8 MB/s) - ‘ISIC_2019_Test_Input.zip’ saved [3823247963/3823247963]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8vOHFeJNZDRe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%capture\n",
        "!unzip ISIC_2019_Test_Input.zip;"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZS4q_z79Zkn-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "rm ISIC_2019_Test_Input.zip"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IsJ3fOIfZxl3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "768e16a5-e289-4956-d91e-ae0d577ccf27"
      },
      "source": [
        "import pandas as pd\n",
        "train_df=pd.read_csv('https://s3.amazonaws.com/isic-challenge-2019/ISIC_2019_Training_GroundTruth.csv')\n",
        "train_df.drop('UNK',inplace=True,axis=1)\n",
        "train_df['image']=train_df['image']+'.jpg'\n",
        "train_df.head()"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>image</th>\n",
              "      <th>MEL</th>\n",
              "      <th>NV</th>\n",
              "      <th>BCC</th>\n",
              "      <th>AK</th>\n",
              "      <th>BKL</th>\n",
              "      <th>DF</th>\n",
              "      <th>VASC</th>\n",
              "      <th>SCC</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ISIC_0000000.jpg</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ISIC_0000001.jpg</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ISIC_0000002.jpg</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ISIC_0000003.jpg</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ISIC_0000004.jpg</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              image  MEL   NV  BCC   AK  BKL   DF  VASC  SCC\n",
              "0  ISIC_0000000.jpg  0.0  1.0  0.0  0.0  0.0  0.0   0.0  0.0\n",
              "1  ISIC_0000001.jpg  0.0  1.0  0.0  0.0  0.0  0.0   0.0  0.0\n",
              "2  ISIC_0000002.jpg  1.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0\n",
              "3  ISIC_0000003.jpg  0.0  1.0  0.0  0.0  0.0  0.0   0.0  0.0\n",
              "4  ISIC_0000004.jpg  1.0  0.0  0.0  0.0  0.0  0.0   0.0  0.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YXsQ-Ox8aDOz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "0d7147ac-7c73-49cf-bf98-4371d400c900"
      },
      "source": [
        "test_df=pd.read_csv('https://s3.amazonaws.com/isic-challenge-2019/ISIC_2019_Test_Metadata.csv')\n",
        "test_df['image']=test_df['image']+'.jpg'\n",
        "test_df.head()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>image</th>\n",
              "      <th>age_approx</th>\n",
              "      <th>anatom_site_general</th>\n",
              "      <th>sex</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ISIC_0034321.jpg</td>\n",
              "      <td>60.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>female</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ISIC_0034322.jpg</td>\n",
              "      <td>70.0</td>\n",
              "      <td>anterior torso</td>\n",
              "      <td>male</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ISIC_0034323.jpg</td>\n",
              "      <td>70.0</td>\n",
              "      <td>lower extremity</td>\n",
              "      <td>male</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ISIC_0034324.jpg</td>\n",
              "      <td>70.0</td>\n",
              "      <td>lower extremity</td>\n",
              "      <td>male</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ISIC_0034325.jpg</td>\n",
              "      <td>30.0</td>\n",
              "      <td>upper extremity</td>\n",
              "      <td>female</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              image  age_approx anatom_site_general     sex\n",
              "0  ISIC_0034321.jpg        60.0                 NaN  female\n",
              "1  ISIC_0034322.jpg        70.0      anterior torso    male\n",
              "2  ISIC_0034323.jpg        70.0     lower extremity    male\n",
              "3  ISIC_0034324.jpg        70.0     lower extremity    male\n",
              "4  ISIC_0034325.jpg        30.0     upper extremity  female"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CxlHye-IZyWh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "3081571c-10b9-4a03-fb99-ffd6f50c54a7"
      },
      "source": [
        "import cv2\n",
        "img=cv2.imread('ISIC_2019_Test_Input/ISIC_0073120.jpg')\n",
        "img.shape"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1024, 1024, 3)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HrcifLOBa_W0",
        "colab_type": "text"
      },
      "source": [
        "#Model download"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RZadX4RdbAuC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%capture\n",
        "!pip install image-classifiers==1.0.0b1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dPJ9-sh-rsd7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "9ee1146e-fdcb-4d43-80f7-5a521484a6b2"
      },
      "source": [
        "from classification_models.keras import Classifiers\n",
        "inceptionresnetv2, preprocess_input = Classifiers.get('inceptionresnetv2')"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pvHP5ty6r2S9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "train, val = train_test_split(train_df, test_size = 0.2)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J4PbS3tlr3tn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras_preprocessing.image import ImageDataGenerator\n",
        "\n",
        "train_data_gen= ImageDataGenerator(\n",
        "    horizontal_flip=True,\n",
        "    vertical_flip=True,\n",
        "    rotation_range=90,\n",
        "    width_shift_range=0.2,\n",
        "    height_shift_range=0.2,\n",
        "    zoom_range=.2,\n",
        "    rescale=1/255,\n",
        "    fill_mode='nearest',\n",
        "    shear_range=0.2,\n",
        "    brightness_range=[0.5, 1.5])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LYOEqrHUr44w",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "da860acf-9239-4f28-c08d-a0c114f45369"
      },
      "source": [
        "img_shape=300\n",
        "batch_size=20\n",
        "col=train_df.columns[1::].to_list()\n",
        "col"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['MEL', 'NV', 'BCC', 'AK', 'BKL', 'DF', 'VASC', 'SCC']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "La7WeKO7r5cF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "266e0f7e-619c-426e-8958-bdf600a61885"
      },
      "source": [
        "train_generator=train_data_gen.flow_from_dataframe(train,directory='/content/ISIC_2019_Training_Input',\n",
        "                                                      target_size=(img_shape,img_shape),\n",
        "                                                      x_col=\"image\",\n",
        "                                                      y_col=col,\n",
        "                                                      class_mode='raw',\n",
        "                                                      shuffle=False,\n",
        "                                                      batch_size=batch_size)\n",
        "\n",
        "val_generator=train_data_gen.flow_from_dataframe(val,directory='/content/ISIC_2019_Training_Input',\n",
        "                                                      target_size=(img_shape,img_shape),\n",
        "                                                      x_col=\"image\",\n",
        "                                                      y_col=col,\n",
        "                                                      class_mode='raw',\n",
        "                                                      shuffle=False,\n",
        "                                                      batch_size=batch_size)\n",
        "test_generator=train_data_gen.flow_from_dataframe(test_df,directory='/content/ISIC_2019_Test_Input',\n",
        "                                                      target_size=(img_shape,img_shape),\n",
        "                                                      x_col=\"image\",\n",
        "                                                      class_mode=None,\n",
        "                                                      shuffle=False,\n",
        "                                                      batch_size=batch_size)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 20264 validated image filenames.\n",
            "Found 5067 validated image filenames.\n",
            "Found 8238 validated image filenames.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7WBDR4JwsBa4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 92
        },
        "outputId": "bd0fd980-c69b-41f1-fbcb-c4582e864a6d"
      },
      "source": [
        "base_model = inceptionresnetv2(input_shape=(img_shape,img_shape,3), weights='imagenet', include_top=False)\n",
        "train_generator.next()[0].shape,train_generator.next()[1].shape\n"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://github.com/fchollet/deep-learning-models/releases/download/v0.7/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels_notop.h5\n",
            "219062272/219055592 [==============================] - 5s 0us/step\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((20, 300, 300, 3), (20, 8))"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BckIN4qKsE1V",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import keras\n",
        "\n",
        "\n",
        "x = keras.layers.GlobalAveragePooling2D()(base_model.output)\n",
        "output = keras.layers.Dense(len(col), activation='sigmoid')(x)\n",
        "model = keras.models.Model(inputs=[base_model.input], outputs=[output])\n",
        "\n",
        "\n",
        "model.compile(keras.optimizers.rmsprop(lr=0.0001, decay=1e-6),loss=\"binary_crossentropy\",metrics=[\"accuracy\"])\n",
        "\n",
        "\n",
        "from sklearn.metrics import accuracy_score,f1_score,roc_auc_score\n",
        "from keras.callbacks import ModelCheckpoint,ReduceLROnPlateau"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bsv8NCntrmET",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 764
        },
        "outputId": "3cafca0d-9d30-4536-94ab-253a20fbf5fb"
      },
      "source": [
        "\n",
        "results = model.fit_generator(train_generator,epochs=20,\n",
        "                              steps_per_epoch=train_generator.n/batch_size,\n",
        "                              validation_data=val_generator,\n",
        "                             validation_steps=val_generator.n/batch_size,\n",
        "                              callbacks=[\n",
        "                                  ReduceLROnPlateau(monitor='val_loss', factor=0.3,patience=3, min_lr=0.000001)])"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "1014/1013 [==============================] - 1245s 1s/step - loss: 0.1870 - accuracy: 0.9269 - val_loss: 0.1484 - val_accuracy: 0.9319\n",
            "Epoch 2/20\n",
            "1014/1013 [==============================] - 1167s 1s/step - loss: 0.1521 - accuracy: 0.9393 - val_loss: 0.0728 - val_accuracy: 0.9367\n",
            "Epoch 3/20\n",
            "1014/1013 [==============================] - 1189s 1s/step - loss: 0.1368 - accuracy: 0.9451 - val_loss: 0.1304 - val_accuracy: 0.9399\n",
            "Epoch 4/20\n",
            "1014/1013 [==============================] - 1198s 1s/step - loss: 0.1269 - accuracy: 0.9498 - val_loss: 0.0569 - val_accuracy: 0.9427\n",
            "Epoch 5/20\n",
            "1014/1013 [==============================] - 1201s 1s/step - loss: 0.1196 - accuracy: 0.9535 - val_loss: 0.1325 - val_accuracy: 0.9452\n",
            "Epoch 6/20\n",
            "1014/1013 [==============================] - 1199s 1s/step - loss: 0.1121 - accuracy: 0.9563 - val_loss: 0.0934 - val_accuracy: 0.9467\n",
            "Epoch 7/20\n",
            "1014/1013 [==============================] - 1177s 1s/step - loss: 0.1059 - accuracy: 0.9590 - val_loss: 0.0491 - val_accuracy: 0.9462\n",
            "Epoch 8/20\n",
            "1014/1013 [==============================] - 1178s 1s/step - loss: 0.1004 - accuracy: 0.9612 - val_loss: 0.2115 - val_accuracy: 0.9347\n",
            "Epoch 9/20\n",
            "1014/1013 [==============================] - 1163s 1s/step - loss: 0.0950 - accuracy: 0.9634 - val_loss: 0.0561 - val_accuracy: 0.9467\n",
            "Epoch 10/20\n",
            "1014/1013 [==============================] - 1163s 1s/step - loss: 0.0904 - accuracy: 0.9653 - val_loss: 0.1037 - val_accuracy: 0.9471\n",
            "Epoch 11/20\n",
            "1014/1013 [==============================] - 1174s 1s/step - loss: 0.0651 - accuracy: 0.9747 - val_loss: 0.0378 - val_accuracy: 0.9542\n",
            "Epoch 12/20\n",
            "1014/1013 [==============================] - 1169s 1s/step - loss: 0.0572 - accuracy: 0.9781 - val_loss: 0.0461 - val_accuracy: 0.9534\n",
            "Epoch 13/20\n",
            "1014/1013 [==============================] - 1163s 1s/step - loss: 0.0524 - accuracy: 0.9800 - val_loss: 0.0153 - val_accuracy: 0.9530\n",
            "Epoch 14/20\n",
            "1014/1013 [==============================] - 1169s 1s/step - loss: 0.0470 - accuracy: 0.9822 - val_loss: 0.0950 - val_accuracy: 0.9550\n",
            "Epoch 15/20\n",
            "1014/1013 [==============================] - 1166s 1s/step - loss: 0.0453 - accuracy: 0.9834 - val_loss: 0.1038 - val_accuracy: 0.9503\n",
            "Epoch 16/20\n",
            "1014/1013 [==============================] - 1164s 1s/step - loss: 0.0420 - accuracy: 0.9840 - val_loss: 0.1142 - val_accuracy: 0.9552\n",
            "Epoch 17/20\n",
            "1014/1013 [==============================] - 1171s 1s/step - loss: 0.0342 - accuracy: 0.9872 - val_loss: 0.2430 - val_accuracy: 0.9558\n",
            "Epoch 18/20\n",
            "1014/1013 [==============================] - 1165s 1s/step - loss: 0.0314 - accuracy: 0.9883 - val_loss: 8.2599e-04 - val_accuracy: 0.9570\n",
            "Epoch 19/20\n",
            "1014/1013 [==============================] - 1163s 1s/step - loss: 0.0286 - accuracy: 0.9891 - val_loss: 0.0215 - val_accuracy: 0.9566\n",
            "Epoch 20/20\n",
            "1014/1013 [==============================] - 1171s 1s/step - loss: 0.0284 - accuracy: 0.9896 - val_loss: 0.3767 - val_accuracy: 0.9578\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DYTVei5GcqYw",
        "colab_type": "code",
        "outputId": "07a11eb0-cbef-43b4-c14f-bb542b089707",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.plot(results.history['loss'])\n",
        "plt.plot(results.history['val_loss'])\n",
        "plt.title('model loss')\n",
        "plt.ylabel('loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'test'], loc='upper left')\n",
        "plt.show()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOydd3yV5fn/31c22SEJEEYIe8sKKCLugQuse1apFhdt7fCnba1t7bfW2tZWq1VpBa0Lt0WLFVFcDCEM2SPsAIEkQBbZuX9/3CdwCCf7PGck1/v1Oq9znnE/z5WT5HzOfV9LjDEoiqIoSn1C/G2AoiiKEpioQCiKoigeUYFQFEVRPKICoSiKonhEBUJRFEXxiAqEoiiK4hEVCEXxAiLyooj8XzPP3Ski57f1OoriNCoQiqIoikdUIBRFURSPqEAoHQbX0s79IrJGREpF5AUR6SoiH4lIsYgsEJEkt/OniMh6ETkiIp+LyBC3Y6NFZKVr3BtAVL17XSYiq11jF4vIKa20+fsiki0ih0Rkroh0d+0XEfmriBwUkSIRWSsiw13HLhGRDS7b9orIz1r1hikdHhUIpaNxFXABMBC4HPgI+AWQiv1/+CGAiAwEXgfucx2bB3wgIhEiEgG8D7wMdAbecl0X19jRwCzgTiAZeB6YKyKRLTFURM4F/gBcC6QBu4A5rsMXAme6fo4E1zkFrmMvAHcaY+KA4cBnLbmvotShAqF0NP5ujDlgjNkLfAV8Y4xZZYwpB94DRrvOuw74rzHmE2NMFfBnoBNwOnAaEA78zRhTZYx5G1judo/pwPPGmG+MMTXGmJeACte4lnATMMsYs9IYUwH8HJggIhlAFRAHDAbEGLPRGLPfNa4KGCoi8caYw8aYlS28r6IAKhBKx+OA2+syD9uxrtfdsd/YATDG1AJ7gB6uY3vNiZUud7m97g381LW8dEREjgC9XONaQn0bSrCzhB7GmM+Ap4FngIMiMlNE4l2nXgVcAuwSkS9EZEIL76sogAqEojTEPuwHPWDX/LEf8nuB/UAP17460t1e7wF+b4xJdHtEG2Neb6MNMdglq70AxpinjDFjgaHYpab7XfuXG2OmAl2wS2FvtvC+igKoQChKQ7wJXCoi54lIOPBT7DLRYmAJUA38UETCReRKYLzb2H8Cd4nIqS5ncoyIXCoicS204XVgmoiMcvkvHsUuie0UkXGu64cDpUA5UOvykdwkIgmupbEioLYN74PSgVGBUBQPGGM2AzcDfwfysQ7ty40xlcaYSuBK4DbgENZf8a7b2Czg+9gloMNAtuvcltqwAPgV8A521tIPuN51OB4rRIexy1AFwJ9cx24BdopIEXAX1pehKC1GtGGQoiiK4gmdQSiKoigeUYFQFEVRPKICoSiKonhEBUJRFEXxSJi/DfAWKSkpJiMjw99mKIqiBBUrVqzIN8akejrWbgQiIyODrKwsf5uhKIoSVIjIroaO6RKToiiK4hEVCEVRFMUjKhCKoiiKR9qND8ITVVVV5OTkUF5e7m9THCcqKoqePXsSHh7ub1MURWkntGuByMnJIS4ujoyMDE4svNm+MMZQUFBATk4Offr08bc5iqK0E9r1ElN5eTnJycntWhwARITk5OQOMVNSFMV3tGuBANq9ONTRUX5ORVF8R7sXCEVRlHbN6tcga7Yjl1aBcJgjR47wj3/8o8XjLrnkEo4cOeKARYqitCtWvwZr3nDk0ioQDtOQQFRXVzc6bt68eSQmJjpllqIo7YXSPIhJceTSKhAO8+CDD7Jt2zZGjRrFuHHjmDRpElOmTGHo0KEAXHHFFYwdO5Zhw4Yxc+bMY+MyMjLIz89n586dDBkyhO9///sMGzaMCy+8kLKyMn/9OIqiBBqleRDjsZRSm2nXYa7u/PaD9WzYV+TVaw7tHs+vLx/W6DmPPfYY69atY/Xq1Xz++edceumlrFu37lg46qxZs+jcuTNlZWWMGzeOq666iuTk5BOusXXrVl5//XX++c9/cu211/LOO+9w8803e/VnURQlCKmphqOHVCDaC+PHjz8hV+Gpp57ivffeA2DPnj1s3br1JIHo06cPo0aNAmDs2LHs3LnTZ/YqihLAlB0CjApEW2nqm76viImJOfb6888/Z8GCBSxZsoTo6GjOPvtsj7kMkZGRx16HhobqEpOiKJbSPPusPojgJC4ujuLiYo/HCgsLSUpKIjo6mk2bNrF06VIfW6coSlBzTCB0BhGUJCcnM3HiRIYPH06nTp3o2rXrsWOTJ0/mueeeY8iQIQwaNIjTTjvNj5YqihJ0lObbZxWI4OW1117zuD8yMpKPPvrI47E6P0NKSgrr1q07tv9nP/uZ1+1TFCVIcXgGoUtMiqIowUppPkgoRDmTM6UCoSiKEqzUJcmFOPNR7qhAiMhkEdksItki8qCH43eJyFoRWS0iX4vIUNf+DBEpc+1fLSLPOWmnoihKUFKa79jyEjjogxCRUOAZ4AIgB1guInONMRvcTnvNGPOc6/wpwBPAZNexbcaYUU7ZpyiKEvQ4WGYDnJ1BjAeyjTHbjTGVwBxgqvsJxhj31OYYwDhoj6IoSvvCwTIb4KxA9AD2uG3nuPadgIjcKyLbgMeBH7od6iMiq0TkCxGZ5OkGIjJdRLJEJCsvL8+btiuKogQ+pfkQHZwziGZhjHnGGNMPeAB4yLV7P5BujBkN/AR4TUTiPYydaYzJNMZkpqY6p6JtobXlvgH+9re/cfToUS9bpChKu6CqDCqLg3aJaS/Qy227p2tfQ8wBrgAwxlQYYwpcr1cA24CBDtnpKCoQiqI4gsNJcuBsotxyYICI9MEKw/XAje4niMgAY8xW1+alwFbX/lTgkDGmRkT6AgOA7Q7a6hju5b4vuOACunTpwptvvklFRQXf+c53+O1vf0tpaSnXXnstOTk51NTU8Ktf/YoDBw6wb98+zjnnHFJSUli4cKG/fxRFUQIJh5PkwEGBMMZUi8gM4GMgFJhljFkvIo8AWcaYucAMETkfqAIOA7e6hp8JPCIiVUAtcJcx5lCbDProQchd26ZLnES3EXDxY42e4l7ue/78+bz99tssW7YMYwxTpkzhyy+/JC8vj+7du/Pf//4XsDWaEhISeOKJJ1i4cCEpKc5NIRVFCVKCfAaBMWYeMK/evofdXv+ogXHvAO84aZs/mD9/PvPnz2f06NEAlJSUsHXrViZNmsRPf/pTHnjgAS677DImTfLok1cURTmOw5VcoSPVYmrim74vMMbw85//nDvvvPOkYytXrmTevHk89NBDnHfeeTz88MMerqAoiuLCB0tMfo9iau+4l/u+6KKLmDVrFiUlJQDs3buXgwcPsm/fPqKjo7n55pu5//77Wbly5UljFUVRTqA0D8I6QURM0+e2ko4zg/AT7uW+L774Ym688UYmTJgAQGxsLK+88grZ2dncf//9hISEEB4ezrPPPgvA9OnTmTx5Mt27d1cntaIoJ1JXZkPEsVuIMe0jeTkzM9NkZWWdsG/jxo0MGTLETxb5no728ypKh+blK6HsMExv25dHEVlhjMn0dEyXmBRFUYIRh8tsgAqEoihKcOJwJVfoAALRXpbQmqKj/JyKogDGOF7JFdq5QERFRVFQUNDuPzyNMRQUFBAVFeVvUxRF8QXlhVBb5fgMol1HMfXs2ZOcnBw6QqXXqKgoevbs6W8zFEXxBT7IooZ2LhDh4eH06dPH32YoiqJ4Fx9kUUM7X2JSFEVplxz1zQxCBUJRFCXY8EGZDVCBUBRFCT7qfBDRyY7eRgVCURQl2CjNg6hECItw9DYqEIqiKMGGD3IgQAVCURQl+PBBFjWoQCiKogQf7WEGISKTRWSziGSLyIMejt8lImtFZLWIfC0iQ92O/dw1brOIXOSknYqiKEGFDwr1gYMCISKhwDPAxcBQ4AZ3AXDxmjFmhDFmFPA48IRr7FDgemAYMBn4h+t6iqIoHZuaajh6KLgFAhgPZBtjthtjKoE5wFT3E4wxRW6bMUBd0aSpwBxjTIUxZgeQ7bqeoihKx6bsEGB8IhBOltroAexx284BTq1/kojcC/wEiADOdRu7tN7YHs6YqSiKEkT4qMwGBICT2hjzjDGmH/AA8FBLxorIdBHJEpGsjlCQT1EUxVdZ1OCsQOwFerlt93Tta4g5wBUtGWuMmWmMyTTGZKamOv9mKYqi+B0fVXIFZwViOTBARPqISATW6TzX/QQRGeC2eSmw1fV6LnC9iESKSB9gALDMQVsVRVGCAx/OIBzzQRhjqkVkBvAxEArMMsasF5FHgCxjzFxghoicD1QBh4FbXWPXi8ibwAagGrjXGFPjlK2KoihBQ2keSKgtteEwjvaDMMbMA+bV2/ew2+sfNTL298DvnbNOURQlCKlLkgtx3oXsdye1oiiK0gJ8VGYDVCAURVGCCx+V2QAVCEVRlODCR2U2QAVCURQluNAlJkVRFOUkqsqgskSXmBRFUZR6+DBJDlQgFEVRggcfJsmBCoSiKErwoDMIRVEUxSM+rOQKKhCKoijBQ51ARKtAKIqiKO6U5kFYJ4iI8cntVCAURVGChbocCBGf3E4FQlEUJVjwYZkNUIFQFEUJHnxYZgNUIBRFUYIHH5bZABUIRVGU4MAYXWJSFEVRPFBeCLVVOoNQFEVR6uHjLGpwWCBEZLKIbBaRbBF50MPxn4jIBhFZIyKfikhvt2M1IrLa9ZjrpJ2KoigBj4+zqMHBntQiEgo8A1wA5ADLRWSuMWaD22mrgExjzFERuRt4HLjOdazMGDPKKfsURVGCCh8X6gNnZxDjgWxjzHZjTCUwB5jqfoIxZqEx5qhrcynQ00F7FEVRgpd2JhA9gD1u2zmufQ1xO/CR23aUiGSJyFIRucLTABGZ7jonKy8vr+0WK4qiBCp1PojoZJ/d0rElppYgIjcDmcBZbrt7G2P2ikhf4DMRWWuM2eY+zhgzE5gJkJmZaXxmsKIoiq8pzYOoRAiL8NktnZxB7AV6uW33dO07ARE5H/glMMUYU1G33xiz1/W8HfgcGO2grYqidFQqj0JVub+taBofZ1GDswKxHBggIn1EJAK4HjghGklERgPPY8XhoNv+JBGJdL1OASYC7s5tRVEU7zDnRnj/bn9b0TRHC3wuEI4tMRljqkVkBvAxEArMMsasF5FHgCxjzFzgT0As8JbY6oS7jTFTgCHA8yJSixWxx+pFPymKorSdyqOw8ytI6uNvS5qmNA9SBvr0lo76IIwx84B59fY97Pb6/AbGLQZGOGmboigKe1dAbTUU7rGlLHxURrtVlOZB74k+vaVmUiuK0nHZvcQ+V5dDyQH/2tIYNdVw9FC78kEoiqIENnUCAXBkt//saIqyQ4DxaRY1qEAoitJRqamGPcsgY5LdDmSB8EOZDVCBUBSlo3JgLVSWwIhr7PaRXf61pzH8kEUNKhCKonRUdi+1z/3Ph+iUAJ9B+L6SK6hAKIrSUdm1GBLSIaEHJPWGwzqDqI8KhKIoHQ9j7Ayi9wS7nZge4DOIPJBQW2rDh6hAKIrS8Ti0HUoPQrqbQBTugdpa/9rVEHWtRkN8+5GtAqEoSsejLrzVXSBqKgM3F6I03+fLS6ACoShKR2TXEujUGVIH2e1EVzPLQF1mqptB+BgVCEVROh67l0D6acdLaxwTiAB1VPuhkiuoQCiK0tEoPgCHth1fXgJIdHUmCFiB0CUmRVEU59njyn9wF4jwThDTJTCXmCqP2oQ+XWJSlFaQNQv2r/G3FUqwsGsJhHWCtJEn7g/UUNej/kmSAxUIJdipLIUPfwJLnva3JUqwsHsx9Mw8uXVnYnpgJsv5KUkOVCCUYOfABsDAvtX+tkQJBiqKIXftictLdST1hsIcqK3xvV2N4acyG6ACoQQ7ua6lpfwtdjahKI2xZxmY2uMZ1O4kpkNtFRTn+t6uxvBTJVdwWCBEZLKIbBaRbBF50MPxn4jIBhFZIyKfikhvt2O3ishW1+NWJ+1Ugpjcta4XBnLX+dUUJQjYvRQkBHqOO/lYYrp9DjQ/RHtcYhKRUOAZ4GJgKHCDiAytd9oqINMYcwrwNvC4a2xn4NfAqcB44NcikuSUrUoQk7sWOvezr/d/619blMBn9xLodgpExp18LFCT5UrzITwaImJ8fmsnZxDjgWxjzHZjTCUwB5jqfoIxZqEx5qhrcynQ0/X6IuATY8whY8xh4BNgsoO2KsFIbQ0cWA8DLrTfrlQglMaoroScLM/+B4CEAM2FKM33y/ISNFMgRORHIhIvlhdEZKWIXNjEsB7AHrftHNe+hrgd+KglY0VkuohkiUhWXl5e0z+I0r4o2AbVZZB2ig1Z3K+OaqUR9n9r/148+R8AwqMgtlsACoR/sqih+TOI7xljioALgSTgFuAxbxkhIjcDmcCfWjLOGDPTGJNpjMlMTfXPG6j4kToHdbcRkDYKDm6EqnL/2qQELvUL9HkiEHMhgkAgXAVLuAR42Riz3m1fQ+wFerlt93TtO/HCIucDvwSmGGMqWjJW6eDkroWQcEgZZGcQpgYOrve3VUqgsnuJ9VfFdmn4nIAUiABfYgJWiMh8rEB8LCJxQFOF05cDA0Skj4hEANcDc91PEJHRwPNYcTjoduhj4EIRSXI5py907VOU4+SuhS6DbcJTXVas+iEUT9TWugr0NTJ7AFdfiADKhTDGziCi/SMQYc0873ZgFLDdGHPUFWU0rbEBxphqEZmB/WAPBWYZY9aLyCNAljFmLnZJKRZ4S2xVxd3GmCnGmEMi8jusyAA8Yow51OKfTmnf5K6FARfY14npttuWCoTiifwtUHa4Yf9DHUm9obYaivYdL+DnT8oLbW6Gn5aYmisQE4DVxphSl79gDPBkU4OMMfOAefX2Pez2+vxGxs4CZjXTPqWjUXzAdgTrNsJui7gc1SoQigd2L7bPzZlBgF1mCgSB8GMWNTR/ielZ4KiIjAR+CmwD/u2YVT6kuLyKe15dwYZ9Rf42RWkJdQlydQIBViAOrLfhjIrizu6ltlpr576NnxdouRB+zKKG5gtEtTHGYPMYnjbGPAN4yDQJPkorali1+wjfe3E5+wvL/G2O0lzqIpi6Dj++r/so2zYyb5N/bFICl11L7PKSNBFbk+BKxQo4gQjsGUSxiPwcG976XxEJAcKdM8t3dEuIYtZt4yipqGba7OUUl1f52ySlOeSutcsBnRKP70sbZZ91mUlxpzAHCnc3vbwEEBYJcWkqEC6aKxDXARXYfIhcbNhpi3IWApkhafE8e/MYsg+WcM+rK6mqaSpAS/E7uWttyQR3kvpARJwKhHIiuz00CGqMxN6BkyxX54OITvbL7ZslEC5ReBVIEJHLgHJjTLvwQdQxaUAqf7hyBF9tzecX767FrqgpAUllKRRkn+h/AAgJsVnVKhCKO7uXQETsicuRjZGYHkACkWej8+r3rvARzS21cS2wDLgGuBb4RkSudtIwf3BNZi9+dN4A3lqRw1OfZvvbHKUh6npA1BcIsI7q3LWBE8eu+J9dS6DXeAhtZtBmYjoU7oWaamftag5+zKKG5oe5/hIYV5fMJiKpwAJsBdZ2xX3nDyDncBl/XbCFHkmduHpsz6YHKb7FvcRGfdJG2no7+VugyxDf2qUEHmWH4eAGGHZF88ckptus/OJ9x8Ne/UVpvl8Fork+iJB6mc4FLRgbVIgIf7hyBBP7J/PgO2tYlJ3vb5OU+uSuhaiE49U33dGMasWdPcsA03z/AwRWX4jSPL+FuELzP+T/JyIfi8htInIb8F/qJcC1JyLCQnj25rH0S43lrpdXsClXcyQCijoHtaeQxZSBtiG9CoQCsGuxrdfVY2zzxyS5ciECoT+1n5eYmuukvh+YCZziesw0xjzgpGH+Jj4qnNnTxhEdGcq02cvJLdQqoQFBXQ8IT8tLACGh9pgKhAI2gqn7KIiIbv6Y+J6A+H8GUVMNZYcCXyAAjDHvGGN+4nq856RRgUL3xE7Mum0cRWVVTHtxOSUVAeC06ujU9YBoSCDAVXJjjS3QpnRcqsph38qWLS+BjRiK7+5/gThaYJ8DdYlJRIpFpMjDo1hEOsS6y7DuCfzj5rFsOVCsORKBQGMO6jrSRkJlMRze4RublMBk30qbWd9SgYDAKPvt5yQ5aEIgjDFxxph4D484Y0y8r4z0N2cNTOXR7wznyy15PPTeOs2R8CfuPSAaos5RvW+Vb2xSApNddQX6Tmv52EAQiKP+LdQH7TQSyQmuG5fOD87tzxtZe3hmoeZI+A33HhANkToYQiPUD9HR2b3U/i1Ed2752MTeUJQDNX4svePnSq6gAtEifnLBQK4c3YM/z9/Ce6ty/G1Ox8RTiY36hEVA12EqEB2Z2hrY803rlpfAlQtRC0V+bGTp50quoALRIkSEx646hQl9k/l/b69hseZI+Jb6PSAao643hC4HdkwOboCKorYJBPh3mak0D0LCbKkNP6EC0UIiwkJ47paxZCTHcOcrK9hyoNjfJnUcjvWAaGIGAVYgyo/4fx1Z8Q+7ltjnpjrINUSgCER0iq0x5iccvbOITBaRzSKSLSIPejh+poisFJHq+rWdRKRGRFa7HnPrj/UnCZ1sjkRUeCi3zVrGgSLNkfAJxyKYmlF0TTOqOza7F9t8htaWykjoCRLi32Q5P5fZAAcFQkRCgWeAi4GhwA0iMrTeabuB24DXPFyizBgzyvWY4pSdraVnUjSzbxvHkbIqvvficgrLtI+E4+Sutc7DqISmz+0yDCQU9q923i4lsDDGOqhbE71UR2g4xPfw/wwixj9lvutwcgYxHsg2xmw3xlQCc7Ad6Y5hjNlpjFkDBGVywfAeCTxz0xg25RZz5uMLefqzrdpwyEly1zbP/wAQHmWL9ekMouNxeCcU72/98lId/g519XOZDXBWIHoAe9y2c1z7mkuUiGSJyFIR8ViKUUSmu87JysvLa52V5UWw5BmbodsKzhnUhf/cO5FxGUn8ef4WzvjjQv7+qQqF1znWA6IZ/oc60kbBvtXqqO5otLRBUEP4XSDa8RKTF+htjMkEbgT+JiL96p9gjJlpjMk0xmSmprbyjawuh08ehuUvtNrQ4T0S+Net4/hgxhmMy0jiL5+oUHidxnpANETaSJtsVLzfMbOUAGT3YrsMmdrGcu+J6bbkd3Wld+xqCZVHobLEryGu4KxA7AXc6zH3dO1rFsaYva7n7cDnwGhvGneM2C4w+DL49jVbu6UNjOjpLhSdTxCKIhWKttGcEhv1UUd1x2T3Uuh1Wtujf47lQvgh5ykAsqjBWYFYDgwQkT4iEgFcDzQrGklEkkQk0vU6BZgIbHDM0sxptrHIhv945XJWKDJPEIpJKhRtI3etjQdPaEEDp27DAbHLTErHoDTfNotqq/8BbEAE+GeZKQDqMIGDAmGMqQZmAB8DG4E3jTHrReQREZkCICLjRCQH28r0eRFZ7xo+BMgSkW+BhcBjxhjnBCLjTOjcF1bM9uplPQnFGY99xlMqFC2nzkHtqQdEQ0TE2P4QOoPoOHjL/wD+zYUIgDIb0PyWo63CGDOPeo2FjDEPu71ejl16qj9uMdCCtYQ2EhICY6fBJ7+Cgxu93qqyTijW5hTy5KdbeeKTLfzrq+3cMakvt03MID4q3Kv3a3fU9YDI/F7Lx6aNhJ1fe98mJTDZvQRCI6G7F1ak43vYUGm/ziDarw8iuBh1ky3wtuJFx25RJxQf/uAMxvdJ5gnXjOLJBVvJK65w7L5BT3N6QDRE91HW0VhysOlzleBn12LbPS4ssu3XCg3zXy5Ee19iCjpikmHIFPj2dagqc/RWNurpuFD8dcEWxj+6gGufW8Ksr3ew74iz9w86WuOgruOYo3qN9+xRApPKUruc6A3/Qx2J6f7Jpi7Nh/Bou0zqR1Qg3MmcBuWFsN43DfPqhOKTH5/JD88dQGFZFY98uIHTH/uMqc8s4rkvtrGroNR5Q2prnL9HW8hda2d3KQNbPrZOVDSjuv2TsxxMDaSf7r1rJvX23wzCz8tL4LAPIujoPdF+CGXNglE3+uy2A7rG8eML4vjxBQPZnlfCR+ty+Xh9Lo99tInHPtrEkLR4Lh7ejYuHd2NA1zjv3ryiGP4xAYZfCRc84t1re4vctbauf2M9IBoiKsEGIKhAtH92LwUEeo3z3jUT020eTXWFd5atmksAZFGDziBORATG3ma/ieSu8809yw7D0+Ng7dsA9E2N5d5z+jN3xhl8/cA5PHTpEGIiQvnrgi1c8NcvOe8vn/Pnjzezbm+hdzrbrX0LCvfAoieP2RBwNKcHRGPUlf5W2je7FtvQ5ubU6mouiemAgUIf50KoQAQoI2+wURBeDnltkK//ZuO2v/7rSSUheiZFc8ekvrx99+l88/Pz+N3UYXSNj+LZL7Zx2d+/5sw/LeTReRtZseswtbWtEAtjIGu2LWyXPgHm/sBGcQUSLekB0RBpI+0ywdFD3rNLCSxqqiAnyzvhre74K9S1NF+XmAKS6M4w7ApY86ZdcnHSSVScC988D7Hd4MA6VwcszxUou8RHccuEDG6ZkMGh0koWbDjAR+v2M3vRDmZ+uZ34qDAyMzozvo99jOiRQHhoE/q/b6V1AF/6F5tN/twkeONm+P5CiAqQluPHekC0RSBGua61Bvqe3VaLlEAkdw1UlTooED50VBsTEHWYQAXCM2OnwZo3YN07MOa7zt3ni8ehtgpueRdmXQzL/9WsEsWdYyK4dlwvrh3Xi6LyKhZuOsjS7YdYtqOAzzbZcM5O4aGMTk88JhijeyXRKSL0xAtlzYbwGBhxrRWEa16Ely6H/9wD177csqQ0p2hJD4iGcC+50ffstlqkBCLeTJBzJ6677ermyxlEeaH9XFCBCFDST7NO0azZzgnEoe2w8iUYc6vtnzzqRisQF/0BYpv/hxEfFc7UUT2YOsoWys0vqSBr5yG+2XGIZTsO8eSnWzEGwkOFET0SGN8nmVP7dGZst1Di170DI64+PlvImAgX/BbmPwSLn4KJP3LiJ28ZLekB0RDRnSEhXf0Q7ZldiyEpA+LTvHtdf+RCBEgWNahAeEbEziL+94D9UKn7BupNFj4KIeFw1v+z2+Nuh2+ehVX/hkk/bfVlU2IjmTw8jcnD7T9KUXkVK3YdZjeHW2UAACAASURBVJlLMF74ejvPfbGNW0I/4XfhR5lZehY91+5nXEZnUuMiYcIM66Rf8BvoPgb6TPLCD9sGWtIDojHSTtGaTO2VugZBAy5w5vq+LvsdIFnUoALRMCOvgwW/trOIy//m3WvnrrMRQ2fcB3Hd7L6UAdDnLHu/ifdBSGjj12gm8VHhnDOoC+cM6gJAWWUNq3YfYuC7D7Ojqj9PbIih/NuVAPRLjWF8n2QmZjzE5NwNhL09De78EuK7e8WWFlPXA2LENW2/Vtoo2PSh7f8RKP4VxTsUZNvqp95eXqojsTds+9SZa3siQLKoQaOYGqZTEgy70oaBVhR799qf/c5+SNVfwhl3hw053Trfu/dzo1NEKKdH7iDlaDZ9LprBml9fxLv3nM6DFw+md3IMH367jxnvbuWi/dM5WlrMzmev5u1l29lVUOqdsNqW0JoeEA1RNwusc3or7Yddi+1zby8myLlTlwvRxnYAzSaABEJnEI2R+T3bJ2Lt2zbL2hvsXgpb/gfnPWxFyJ1Bl0BcmvVFDLrYO/fzRNZsiIiFEVcTERbCmPQkxqQncddZ/aipNWzcX8SyHYd4dW0x3z/wCAvnPshZ1bfSNT7ymA/j1D6d6d8lFnHSkd2WEhv16e6KZNr/rfW1KO2DqjL7hSo6BZL7O3OPJFfZ78IcSHHoHu7U+SCi/duPGlQgGqdnJnQdbjOrx97W9qgeY2DBbyG2K5x618nHQ8Os7+PzR22BuuSTmui1nbLDsP5dm+8ReXJWdmiIMLxHAsN7JMAZP8X8L59pS//BgDHn8EbFaXyzvYAPvt0H2Giq8a7Q2tHpiQxJiycq3DtLY0DrekA0RGwXK77qqA5+qisg+1P7d7z5I9t5bew056Lu3ENdfSIQefbvPtT/VZ5VIBqjLrN63s9szkCPsW27XvYC2w7xkj83nF8x5rvw5eNWlC76fdvu54lv37BtVps5I5ILHoF9qzhj4yOcccenmOvPY/eho3yz4xDfbD/Esp0F/G99LgBhIcLArnGM6JHAiJ4JnNIzgUHd4ogMa6VotKYHRGNoRnXwUlMF2z+Hde/Cpv9CRaGdgQ+/ypaJ6X2Gc/f2dbJcgGRRgwpE05xyre1ZnTW7bQJRWwuf/tY6vMbc2vB58Wk2aW3VK3DuQxDeqfX3rI8xNkO8+5jmR2aFhtv8iOfPhDduRqYvpHdyAr2TY7g203aU3V9Yxrd7Clm79whrcgr5eEMub2TtAWx47aBucYzokcgpPRMY0SOBgV3jiAhrwv3Vlh4QDZE20i5HVJb6vUqm0gxqqmHnV3amsPEDO/uNTIDBl1ph6HuWb75lx6X5NhciQJLkQAWiaaIS7B/junfsN/rWxuNveM9+I/7OzKaLzo27Aza8b78tjb6pdffzxO6lkLcJpjzdsnFx3axIvHgZvH8PXPfKCd/q0xI6kZbQicnDbUSWMYacw2Ws3VvImhwrHB+u2cfry+w/WERoCEPS4uwso0ciw3skMKBr7ImZ323pAdEQaSNtj+ED66HXeO9dV/EetTW26c+6d2HjXPttOiLW+uSGXQn9z/Nt0TywEYUJPX2XTV2aB6mDfHOvJlCBaA6Z02DVy7b8xvjvt3x8TRV89ntb82jE1U2fn3EGpAyyzmpvCsSK2RAZb6fkLaX36XDh7+DjX9jCfmfc1+CpIkKvztH06hzNJSNsPoYxhl0FR1m7t9AlHEd4f9U+Xll6XDT6d4llSFo8Q9LimFSxiEHgfYEAu8ykAhE41Nba3Jv178L696EkF8I6wcCL7N/qgAu9O5NuDYk+LPtdmmc/AwIARwVCRCYDTwKhwL+MMY/VO34m8DfgFOB6Y8zbbsduBR5ybf6fMeYlJ21tlO5jbDXRFS/ab/ctXRNf9Qoc2gY3zGlefoOIvc9H98PeFW33fYAtVLf+fevjaO3yymn32H/kT39rWzr2PavZQ0WEjJQYMlJiuHykzauorTXsKChl3d5CNuwrYsP+Ir7Yksc7K3N4IOxTMkLDOO+FHAZ0P8rgtHiGpMUzNC2OPimxhIa0wi8R38NGu3iz9HdNFbx1m/0QG9vI0qFyIhXF1qewdT5s/cSGkYZG2mS3Yd+BgZMhMtbfVh4nMd3R8PNj1FRD2aH2v8QkIqHAM8AFQA6wXETmGmM2uJ22G7gN+Fm9sZ2BXwOZgAFWuMYedsreRhGxs4gPf2wrRrak3nxVGXzxR+g53v7RN5eR19ls5uWzvCMQ374ONRVtC9cVgSl/t/kJb3/PJtEl9Gj15UJChH6psfRLjT1WKgQgr7iC0Neepai4P+N6dWPj/iK+2ppPtatibWRYCIO6xTGkm51tDEmLZ3C3eBKim1iPFvG+o3rRkzYBb9ciOztU34ZnjIH8rS5BmG9zF2qr7Iy23zk2xHvQJYGbxJjYG0oO2P9nJ2czRwvscwBkUYOzM4jxQLYxZjuAiMwBpgLHBMIYs9N1rLbe2IuAT4wxh1zHPwEmA687aG/jjLgG5v/KLtO0RCCW/dN+O7rqXy2beUQlWJFY/Zpd2onu3HKb66gr691zvK371BYi46wP4p/nwFu3wm3zWtfIpxFS4yKhaDMMuJC/XmHzFyqqa8g+WMLG/cVs3F/Exv1FzHdzhgOkJUQxsGscg7vFMcj16N8l9sQoqrSRts5UVTmER7XN0PxsW3CxTnRWvgyneQhf7qhUlcGOr46LQt0afuoQOO1uO+tKPy0gwjmb5Fgk0x5IbUVnw+YSQEly4KxA9AD2uG3nAKe2YexJX1VFZDowHSA9Pb11VjaXyDj7DfHbN+CiR6FTYtNjygvh6yeg//mtW1PMvN2Gu65+DU6f0fLxdez8Ggq2whXPtv4a7qQOhKnPWIH4+Bdw6Z+9c906PPSAiAwLZVj3BIZ1Px4kYIzhYHEFG/YXsTm3mM25xWzKLWbJtgIqa+x3jtAQoU9KDIO6xTG4axxn1vRmZG01tQc2ENJzTOttrK2FD35kRebGt+wy05KnbU2tYPjAc4rDO+2S0db5sONLG1IdHm3LyEz8kV1CSnT4f9UJ6pLljuxWgQgWjDEzgZkAmZmZzteBGDvN+iHWvAGn3tn0+Yv/bkPzznu4dffrNtzWl8l6wa7/h7SyMsqK2XZGMuw7rRvviWFXQM4M+6HYa7wNB/YWzewBISJ0jY+ia3zUsVpTAFU1tezML2WTm2isyTnCf9fs502p5atI+O3M1/i2axmDutqZxoCusfRNjSUtPoqQ5vg3Vv0bdn1tl9ziutoPv9evs/3MvfleBAN5m2Hlv60o5G+x+5L62ByiARfYHIW2ztb8ja/6QgRQJVdwViD2Ar3ctnu69jV37Nn1xn7uFavaQvdR1jmbNRvGT298yajkICz5h/1Qbks12HF3wDu3w/aFNsSvpZTmw4a59putt9dOz/8N7FsFc39ol67aunxVRxt7QISHhjCgaxwDusZxudtbX1JRzZbcIipffpjLE/PYEh7KJxsPnLBMFRUeQkZyDP1SY+mbGmMfKfZ1XJRrZlC0H+Y/DBmTYPQtdt+AC+3SyaIn7XJkIPTS8AU1VbaHSNlhO0vO/J59L5yoAuBPYrvZ6stORzIFUCVXcFYglgMDRKQP9gP/euDGZo79GHhUROqKFV0I/Nz7JraCsdPggx822v0NgK/+YqfX5zzU8DnNYcjl9tvE8hdaJxCrX7XOwLFeqiXlTmg4XD0bnp8E79wB0z/3Toy6N3pAeCA2MowxvTtDr9FkVuzm9emnYYwhr6SCbQdL2Z5fwva8UrbnlbBuXyEfrduPeyfXlNhI+qbG8HDpHxhcVcayob8ireAovZI6ERYaAhN/CO/fbctADDjfq7YHLFvnW+ft9a/D4Ev8bY1zhIRAYi/fCERImC21EQA4JhDGmGoRmYH9sA8FZhlj1ovII0CWMWauiIwD3gOSgMtF5LfGmGHGmEMi8jusyAA8Uuew9jvDr7INdbJmNSwQh3fZ46NvbnvtlrBIG5r69V+tgyyxV9Nj6qittUti6ROgy+C22dEQcV1t4t1r18DC39s2rW3FWz0gGiJtJHwzE2qqkNBwusRF0SUuign9TiyOVlFdw55DR9mWV3pMOFJz5jO86Av+WHU9z76bD3xOeKhwbWYvfn3JlUR89n+w6G8dRyBWvWJriw240N+WOE9iug+WmPJsKHZrl5O9jKM+CGPMPGBevX0Pu71ejl0+8jR2FjDLSftaRWSsXWNe+TJMfsxzdNHnjwECZz3gnXuOvc0KxIoX4bxfNX/czi9t57qzHvSOHQ0x8EJr46KnYODF0LsNdfm92QOiIdJG2ZDfvE2NClFkWCj9u8TRv4urqGHZEXjmX9B1BNNveoLzD1eyPa+EFbsO8+o3u9l6oITZY6YT8/mvIWcF9PRCeHIgU3wAtnxsAyhCg9qd2TwSe8PmeU2f1xYCqMwGaD+I1jF2mv2A+dZD1O3BTbBmjs24bkOOwAkkptscipUvQXVl88dlzbYFzYZO9Y4djXHh722kx3t3tq1/hjd7QDSEe0Z1S1jwGxtdNeUpkuJjGNs7iWsye/HYVafw1A2j+TbnCFcsGUBNRAIsftLrZgcca94AUwOjbva3Jb4hMd1+w6886tw9juYHjP8BVCBaR7fh0HOc/QCu30Tns99BeAyc8RPv3nPc7faPc+Pc5p1fctAmcI26yTcRJJGxNoz2yG67BNdavNkDoiE697P1fVoiEDsX2Wiw0+6BHieHx04Z2Z0375xAYW0k/6o4F7Nhrq0n1V4xxi4v9TrV2bDPQCKxri/EnsbPawsBVMkVVCBaz9hpNrdg16Lj+3JW2A/liT+EGC83++h7rg0dXP5C885f9TLUVtulH1/R+3Q4/Qd2KWzrJ627hjd7QDRESIgVoOYKRFW5DUxI7A3n/KLB00b2SuSDH5zBouSrqDRhrHv7977vwucrcrIgf7P1s3UUfFH2W5eY2gnDvmNLD2fNPr7v099YB9Npd3v/fiEhdhaxe7GtRtoYtbWw4iUbhpkywPu2NMY5v4QuQ+E/M2z9p5bi7R4QDZE2yt6rtqbpc7/8k/WLXP63JktpdI2PYubdl7A8cTID9s3l4Vc/pbyqGfcINla9bBPgvJlbE+jUCcThnc5cv/KobX6kS0ztgIhoGHm9qyRxAWxbaDNHz/yZx05tXmHUTRAW1fQsYvtnNtrCl7OHOsKj4DvP25oy//1py8bW9YDodooztrmTNhKqjtoP/sbIXWejkkbeAP3Obdalo8JDmXjLb4iQarpu+jfXzVzKwSIf9TP2BZWlthz30Cuc+1sPRGK72oKCTs0gjgZWkhyoQLSNzGlQUwmrX4FPH4GEXt5tcFOf6M42zHbNG1Be1PB5WbNtP9shlztnS2OknQJnP2jLN699u+nz63CiB0RD1Dmq9zVS2bW2Bub+wC55XfRoiy4vKf2RIZdzZ6eF7D1wkClPL2JtTmEbDA4gNn4AlcUda3kJnM+FCLAyG6AC0Ta6DIFep9mw1n0r4eyfO9/MZNztdhq65g3Px4v22z69o27yfWMVdybeZx35//0pFO1r3hhfOKjrSBloZ2ON+SGWzbS/14v/2LpiiWfcR3hVER+evo3QEOGa5xfz4ZpmvheBzKpXoHNf63PqaCSmOygQdTMIXWJqP2ROs0sVKYPskpPT9Bhry30sf+HkCCqw/7ymxj/LS+6EhtmlpppK649ojrM2dy2ERtgPb1/Y13V4wwJxeBd8+jubADb8qtbdo8dYyJhEt/Uv8J+7xzG8ewIzXlvFE/M3U1sbpM7rQ9ttG9BRN3WcciLuOCoQgVVmA1Qg2s7QK+yHyCWPN68ZkDcYdwfkbbQ19d2prbG5En3OCoxaOMn9bGb1tk9tZnlT5K6B1MFeLx/eIGkj7T1r61WbN8b2/gC49Im2fRBOvA+K95Gy4wNe/f6pXDO2J099ls09r67kaGV166/rL1a/BhJifTIdkcR06yuoKPH+tXWJqR0SHgU3vQV9z/bdPYddadfFl//rxP3Zn9oY7bY0BfI24+6AvufY3IjG8gKMgf1rfOOgrqP7KKgogsM7Tty/9i0rauf/umWlTTzR/zw7U1n0FJEhwuNXn8JDlw5h/oZcrn52CXuPlLXt+r6ktsYKRL9zvZcEGmw4mQtRmm8jwwKo6ZQKRDASEW0dhBvnQnHu8f1ZsyCmCwy61H+21UfE9o4IDYf37mo4rLTkgP1m5gv/Qx2eMqpL8+GjB6z/ZNwdbb+HiC0FnrcRts5HRLhjUl9euG0cew4dZerTi1ixyz+NElvM9oVQtLfjOafdqRMIJ5aZSvMCankJVCCCl8zv2US4lf+224U5sPVj+8/rqyWa5pLQAy75C+Qss+WwPdHMHhBeJXWILeHs3qP641/YUiGXP+W9JcNh37ERbm4/+zmDuvDevacTExnKDTOX8ui8jXy1NS+wcyZWvWpLtwxqx1Vbm8LJZLkAy6IGFYjgJbmfnepnzbaNzle+bJdpxt7qb8s8M+Jq669Z+OhxMXCnjT0gWkVYBHQdenwGkb3ARoed8WO731uEhsOEe22S455lx3b37xLHf+6dyDmDU5m9aAe3vLCMU34znxtmLuXpz7aycvdhqmvqd+P1E0cP2SoBp1zn3+g4fxPbxUa/OVHVNQAFogOUYGzHjLsD5twImz6wM4l+50JShr+t8oyIdfjuWgzv3gnTF574QeNQD4gmSRtp4/orSuCDH9sIqjN/5v37jPkufPFHO4u4/tVjuxOjI3j+lkyOVlazbMchFmXnsyi7gD/P3wLztxAXGcapfZOZ2D+Zif1TGNAlFvFH9NDat21EWkdeXgL7d5zQy0a5eZvS/LY1F3MAFYhgZsBFEN8T/vszu35/yeP+tqhxYpJti87Xr4PP/2A70tXhdA+IhkgbacX1/buhcDdM+58z35AjYmDc923ZjrwtJxW4i44I4+xBXTjb1Tq1oKSCJdsLWJRdwOJt+SzYeACA1LhIJvZL5vT+KUzsn0KPRC93CWyIVS/b98ofv6NAI6m395eYjNEZhOJlQsMg8zb47P9sS8SBk/1tUdMMmmy/TS960tqbfpr99l6wDUb4oZdz2mj7vHEuZN7etl4WTXHqnbD4KfuY+nSjpybHRnLZKd257JTuAOw5dJTF2+zs4uvsfN5fbRPuMpKjOb1/CqN6JdIv1bZKTYz2sg9q/xq7BHjxn7x73WAlMR32rvTuNcuPWJ+iCoTiVcbcCl89YZ3WoeH+tqZ5XPQobP/cRjXd9TUc9EEPiIboOhQk1NbZOf/Xzt4rJsUu0az8ty1qGJ/W7KG9OkdzXed0rhuXjjGGzQeK7ewiO5+5q/fx2jfHv9F2jok4JhZ9jz3HHm+N2lJWv2oTGEdc3fKx7ZHEdCg7ZIMZvFWLqjTw6jCBwwIhIpOBJ7EtR/9ljHms3vFI4N/AWKAAuM4Ys1NEMoCNwGbXqUuNMXc5aWvQEtsFfvStrb0ULETGwRXPwYuXwie/gq7D7H5/CER4J7s0122kb/wfE2bYcORvnm11e1YRYXC3eAZ3i+f2M/pQXVNLzuEytueXHOutve1gKQs2HiB/+fEGU+GhQu/kGPqlxtA3Nfa4gKTEkhDdwJeL6grruB98WevKjbRHjkUy7fFeMEMAZlGDgwIhIqHAM8AFQA6wXETmGmM2uJ12O3DYGNNfRK4H/ghc5zq2zRgzyin72hWxXfxtQcvJmGhbVS7+uy0P7nQPiMbwRr5Dc+ncx0ZzZc2GST/1iiiFhYaQkRJDRkoM59ZrPX7kaKWrp3bJsefsgyV8uvEg1W7lPrrERTI4LZ7B3eIY3C2OQd3i6N8llsjN86DssDqn3TmWC7HLAYHoODOI8UC2MWY7gIjMAaYC7gIxFfiN6/XbwNPilxANxS+c8xBsXWCXmDImdZzaPhN/ZCvdZs2GM+5z9FaJ0RGM7R3B2N5JJ+yvqqllz6GjbM8rJTuvhC0HitmcW8yLiwqodIXWhoYIc6Kfol9YF+bs7smgqgMMToune0KUfyKpAgUnkuU6oED0ANzz0XOAUxs6xxhTLSKFQN1aSR8RWQUUAQ8ZY76qfwMRmQ5MB0hPT/eu9YrzhEfBlc/DP8/12Maz3dJ9lC3NsvRZ21zKD3kF4aEh9HX5Js6n67H91TW17MgvZVNuMXt3bWXsylW8FHo1j88/3jcjLirs2CzDLnXF0T2xE3FRYcREhBES0s7FIyYFwjp5WSAK7HOALRUHqpN6P5BujCkQkbHA+yIyzBhzQhMEY8xMYCZAZmZmkJbH7OCkjYS7F0Nc8x227YKJP4KXv2PX98d8t3XXqCixLW93LYI+Z0L/89tsVlhoCAO6xjGgaxwUvgoYpt37S66K7smW3GI25RazKbeIzbnF/GfVPl6pOPFDUgRiI8OIjwonLiqM2Mgw4qLCiHNtH392PSLtdnJsBF3jo4iLCoJACxFXVVcv5kKU5tks9QALNHFSIPYC7pXOerr2eTonR0TCgASgwNhGvhUAxpgVIrINGAhkOWiv4i9SB/nbAt/T9xxbmHDRUzDqZtuMpilqa2zW97bPbBTY7qVQW2WPLX4arn7Bey1AjbGlNTImQec+xAOZGZ3JzOjsdophX2E5m/YXkVdcQXF5NcXlVRSVVx97XVxeTX5JJTvyS137qo8tYXkiNjKMrvGRdEuIolt8J7olRNItoRPd4qPsIyGK5JgI/89SEtO9mywXgDkQ4KxALAcGiEgfrBBcD9xY75y5wK3AEuBq4DNjjBGRVOCQMaZGRPoCA4DtDtqqKL6lrojfO7fD5nkw5DLP5x3Z4xKEhVYUylyF/bqNgAn3HBeaOTfC27fbkN2hU9pu367Ftsrt2Q828iMIPRI7tThZr7yq5piAlFRUU1RWTUFpBbmF5ewvLOdAUTm5ReUs3pbPweIKaur1zggPFbrERblExD4P7BrL6f1S6NU5ulU/botJTIec5d67Xml+xxIIl09hBvAxNsx1ljFmvYg8AmQZY+YCLwAvi0g2cAgrIgBnAo+ISBVQC9xljDnklK2K4heGXmFb1S76Gwy+1IpGRTHs/NqKwraFULDVnhvbDQZebMup9D3r5Mi1m9+Gl6+Et6fBNS81LDjNZdUrEBEHQ7wgNvWICg8lKjyU1LimfS81tYaCkgr2F1rROFDkEhHX9sb9RXy26SBlriKHPRI7cXq/ZE7vn8yEvil0S4jyuv2AzaYuPwLlhd4Jjy7Ngy6Dmz7PxzjqgzDGzAPm1dv3sNvrcuAaD+PeAd5x0jZF8TuhYXD6D2Dez2De/XBgva14W1ttnaAZZ9jeHv3OtY2UGoscioyDm9+xfo23boPrXoZBF7fOrvIi2PA+nHKtLS3vR0JDhC7xUXSJj6KhKkXGGLbllbB4WwGLswv4ZOMB3lqRA0DflBgm9EtmQr9kTuubTEqslwIC3HMhunlJIGImtf06XiZQndSK0jEYdRN88Tgs/6d12J/+A7tslH5ay6ObouLhlnetSLxxiy0KOPCiltu0/j3bRnf0LS0f6wdEhP5d4ujfJY7vTsigttawMbeIJdsKWLKtgP+s3serrkzzwd3iOK1vMqf3S+bUvskkdGqlU9i97HdbKxDXVNvM7OjASpIDFQhF8S8R0XD3ItvG0xtZtFEJcPO78PIV8MbNcP1rMOCCll1j9au2x3qPsW23xw+EhAjDuicwrHsCd0zqS3VNLWv3FrJkuxWMOct38+LinYQIDOuewOn9khnaPZ7U2EhS4iJJjY0koVN4445w92S5tnLUFeIaYFnUoAKhKP7H25nwnRLhlvfg31Nhzk1ww2vND4HN2wJ7voELftduEhfDQkMYnZ7E6PQk7jm7PxXVNXy7p5DF2/JZsq2A2Yt2nhRZFRYiJMdGkBoXSUps5DHxSImNtPtiwjktrBOVeTuINKZtiYMBmiQHKhCK0j7plAS3vA//nuISiTnQ75ymx61+xUZCjby+6XODlMiwUMb36cz4Pp2573wbVZVz+Ch5xZXklVSQX1xBfkkFeXXPJRVs2l9MfknFCeVJPo5IZueyFcxY+tEx4UiNPS4i7o+6fTERoSeLiQqEoig+J7ozfHcuvHQ5vH493PimjYBqiJoqWP26LcMejPW9WklUeKjLh9H4ebW1hsKyqmPikfBJPzJLc7l9WN9j+/YXlrNmbyEFJRXUekjd7eSK3jomJnERnHF0HZOBd7dUULzPLn0hQohAiOtZRBBc2yH22X1fUnQ4p/f3/hKVCoSitGeiO8N3/2NF4rXr4Ka3oE8D0TLZC6D0oBbma4CQECEpJoKkmAibab55IKxdzYMXnxyeWlNrOHy0krziiuMP1+wkzyUm2/JKWLqjgoiKzUwOh99+lkchZa2ybVSvRN5XgVAUpcXEpLhmEpfBa9fCTW/barr1WfUKxHRpuVO7o5KYbvMgyo5Yv48boSFCimu5aUgTVWRq5i/GLA3ji19NpZYQao2h1hgwUGs4tm2MTXCv2641NsTXAJFhrejz0QxUIBSlIxCbCrd+AC9eBq9eY3Mm3LvnleTBlv/Z4oEBVg8oYHEPda0nEC0htCwfolNIjHEoqa8NOCM7iqIEHrFdrEjEd4dXr4bd3xw/tuYNm6A3SpeXmk2SF8p+V1dC/paAdFCDCoSidCziulqRiO0Kr1wFe5a7CvO9DD3HBWS5h4ClrX0hctfCP8+xYcXDpnrPLi+iAqEoHY34NLjtQ7vs9MqVsOyfkLdJndMtpVMSRMS2XCBqquHLP8HMc2yI6w1vwJn3O2NjG1EfhKJ0ROK7w60fwouXwEf329pPw670t1XBRWv6QuRthvfugn0rYfhVcMmfA7rXtwqEonRUEnpYkXjlSlsQMCre3xYFH4npzZtB1NbYDoKfPgIRMXDNi97r3eEgKhCK0pFJ7AX3LrN+CKXlJPa2vTOMabg0yaHt8P69sHsxDLoELn8yaBIRVSAUpaMj0m7qLvmcxHSoFttoiAAACTVJREFUKLK9ITolnXjMGMiaBfN/BSFhcMVztoRJEL3XKhCKoiit5YRcCDeBKMyB/8ywnQD7nQtTnrZLekGGCoSiKEprqROIw7tsPw9j4NvX4aMHrN/hsr/C2GlBNWtwx9EwVxGZLCKbRSRbRE5qbisikSLyhuv4NyKS4Xbs5679m0WkFV1PFEVRHMY9Wa74ALx+A7x/t+0ZfvciyPxe0IoDODiDEJFQ4BngAiAHWC4ic40xG9xOux04bIzpLyLXA38ErhORodj+1MOA7sACERlojKlxyl5FUZQWE5UIkfGw/l346s9QVQYX/QFOvQtCgj/NzMmfYDyQbYzZboypBOYA9dMFpwIvuV6/DZwntlj6VGCOMabCGLMDyHZdT1EUJXCoy4XYuwI694U7v4IJ97QLcQBnfRA9gD1u2znAqQ2dY4ypFpFCINm1f2m9sSd5eERkOjAdID093WuGK4qiNJuzHoDi/ZB5O4S2L7duUP80xpiZwEyAzMxMDeRWFMX3DJ3ibwscw8l50F6gl9t2T9c+j+eISBiQABQ0c6yiKIriIE4KxHJggIj0EZEIrNN5br1z5gK3ul5fDXxmjDGu/de7opz6AAOAZQ7aqiiKotTDsSUml09hBvAxEArMMsasF5FHgCxjzFzgBeBlEckGDmFFBNd5bwIbgGrgXo1gUhRF8S1i2kkNlszMTJOVleVvMxRFUYIKEVlhjMn0dKx9xGIpiqIoXkcFQlEURfGICoSiKIriERUIRVEUxSPtxkktInlAC3r/nUQKkO8lc5xA7Wsbal/bUPvaRiDb19sYk+rpQLsRiLYiIlkNefIDAbWvbah9bUPtaxuBbl9D6BKToiiK4hEVCEVRFMUjKhDHmelvA5pA7Wsbal/bUPvaRqDb5xH1QSiKoige0RmEoiiK4hEVCEVRFMUjHUogRGSyiGwWkWwRedDD8UgRecN1/BsRyfChbb1EZKGIbBCR9SLyIw/nnC0ihSKy2vV42Ff2udmwU0TWuu5/UnVEsTzleg/XiMgYH9o2yO29WS0iRSJyX71zfPoeisgsETkoIuvc9nUWkU9EZKvrOamBsbe6ztkqIrd6Osch+/4kIptcv7/3RCSxgbGN/i04aN9vRGSv2+/wkgbGNvr/7qB9b7jZtlNEVjcw1vH3r80YYzrEA1tyfBvQF4gAvgWG1jvnHuA51+vrgTd8aF8aMMb1Og7Y4sG+s4EP/fw+7gRSGjl+CfARIMBpwDd+/H3nYpOA/PYeAmcCY4B1bvseBx50vX4Q+KOHcZ2B7a7nJNfrJB/ZdyEQ5nr9R0/2NedvwUH7fgP8rBm//0b/352yr97xvwAP++v9a+ujI80gxgPZxpjtxphKYA4wtd45U4GXXK/fBs4TEfGFccaY/caYla7XxcBGPPThDgKmAv82lqVAooik+cGO84Btxpi2ZNe3GWPMl9heJ+64/529BFzhYehFwCfGmEPGmMPAJ8BkX9hnjJlvjKl2bS7FdnT0Cw28f82hOf/vbaYx+1yfHdcCr3v7vr6iIwlED2CP23YOJ38AHzvH9Q9SCCT7xDo3XEtbo4FvPByeICLfishHIjLMp4ZZDDBfRFaIyHQPx5vzPvuC62n4H9Pf72FXY8x+1+tcoKuHcwLlffwedkboiab+FpxkhmsJbFYDS3SB8P5NAg4YY7Y2cNyf71+z6EgCERSISCzwDnCfMaao3uGV2CWTkcDfgfd9bR9whjFmDHAxcK+InOkHGxpFbIvbKcBbHg4Hwnt4DGPXGgIy1lxEfont6PhqA6f462/hWaAfMIr/397dvcZRhXEc//40xZdWYkXFV9SqF1WQgqVIq1dKURFRifhSq63eFPTCK0VUhP4BelWwqGDVXIjSYpCC0AiBXpRUitZXNHjVUhIQqURRJH28OM/quJ0NG9udieT3gSW7Z87OnDk5s8/MmZkzcIzSjbMYPcL8Rw+LfltaSgHiKHBl5fMVmVabR9IQMAz81EjpyjKXUYLDaETs7p4eEb9ExGy+3wssk3RhU+XL5R7NvzPAHsqhfFU/9TxodwGHImK6e8JiqENgutPtln9navK0Wo+StgD3AJsyiJ2kj7YwEBExHRFzEXECeKPHctuuvyHgAeD9Xnnaqr+FWEoB4iBwvaRrcg/zYWCsK88Y0LlaZAT4tNfGcbplf+VbwLcR8WqPPJd0zolIWkf5/zUZwJZLOq/znnIy86uubGPA43k10y3A8Up3SlN67rm1XYep2s6eAD6qyfMJsFHSyuxC2ZhpAyfpTuA54N6I+K1Hnn7awqDKVz2ndX+P5fazvQ/SHcB3EXGkbmKb9bcgbZ8lb/JFucLme8rVDS9m2nbKhgBwNqVbYgqYBFY1WLZbKV0Nh4HP83U3sA3YlnmeAb6mXJFxAFjfcP2tymV/keXo1GG1jAJ2ZB1/CaxtuIzLKT/4w5W01uqQEqiOAX9S+sGfopzXGgd+APYBF2TetcCble8+mW1xCtjaYPmmKP33nXbYubLvMmDvfG2hofK9m23rMOVH/9Lu8uXnk7b3JsqX6W932lwlb+P1d6ovD7VhZma1llIXk5mZLYADhJmZ1XKAMDOzWg4QZmZWywHCzMxqOUCYLQI5yuzHbZfDrMoBwszMajlAmC2ApMckTeYY/jslnSlpVtJrKs/xGJd0UeZdI+lA5bkKKzP9Okn7csDAQ5KuzdmvkPRhPothtKmRhM16cYAw65Ok1cBDwIaIWAPMAZsod29/FhE3AhPAK/mVd4DnI+Imyp2/nfRRYEeUAQPXU+7EhTKC77PADZQ7bTcMfKXM5jHUdgHM/kduB24GDubO/TmUgfZO8M+gbO8BuyUNA+dHxESm7wI+yPF3Lo+IPQAR8TtAzm8ycuyefArZ1cD+wa+WWT0HCLP+CdgVES/8K1F6uSvffx2/5o/K+zm8fVrL3MVk1r9xYETSxfD3s6WvomxHI5nnUWB/RBwHfpZ0W6ZvBiaiPC3wiKT7ch5nSTq30bUw65P3UMz6FBHfSHqJ8hSwMygjeD4N/Aqsy2kzlPMUUIbyfj0DwI/A1kzfDOyUtD3n8WCDq2HWN4/manaKJM1GxIq2y2F2urmLyczMavkIwszMavkIwszMajlAmJlZLQcIMzOr5QBhZma1HCDMzKzWX16sXqqeltOmAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dP37qPGyealY",
        "colab_type": "code",
        "outputId": "6609bc1c-6733-4d8a-baec-890f78e6f831",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "y_pred = model.predict_generator(val_generator,steps=val_generator.n/batch_size)\n",
        "y_pred=y_pred.round().astype(int)\n",
        "y_true=val.iloc[:,1::].values\n",
        "print(accuracy_score(y_true,y_pred))\n",
        "print(f1_score(y_true,y_pred,average='macro'))\n"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.7985000986777185\n",
            "0.7661361539656808\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mmjme-j6evHK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "y_test = model.predict_generator(test_generator,steps=test_generator.n/batch_size)\n",
        "#y_test=y_test.round().astype(int)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BLVFaRfRfmhw",
        "colab_type": "code",
        "outputId": "fce86404-c942-4010-ef1a-d72929591a79",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        }
      },
      "source": [
        "import numpy as np\n",
        "d = pd.DataFrame(0, index=np.arange(len(y_test)), columns=col)\n",
        "sub=pd.concat([test_df['image'],d],axis=1)\n",
        "sub['image']=sub['image'].str.replace(r'.jpg', '')\n",
        "sub.head()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>image</th>\n",
              "      <th>MEL</th>\n",
              "      <th>NV</th>\n",
              "      <th>BCC</th>\n",
              "      <th>AK</th>\n",
              "      <th>BKL</th>\n",
              "      <th>DF</th>\n",
              "      <th>VASC</th>\n",
              "      <th>SCC</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ISIC_0034321</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ISIC_0034322</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ISIC_0034323</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ISIC_0034324</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ISIC_0034325</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          image  MEL  NV  BCC  AK  BKL  DF  VASC  SCC\n",
              "0  ISIC_0034321    0   0    0   0    0   0     0    0\n",
              "1  ISIC_0034322    0   0    0   0    0   0     0    0\n",
              "2  ISIC_0034323    0   0    0   0    0   0     0    0\n",
              "3  ISIC_0034324    0   0    0   0    0   0     0    0\n",
              "4  ISIC_0034325    0   0    0   0    0   0     0    0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N53LWkrofqHI",
        "colab_type": "code",
        "outputId": "f3e35af4-306f-4931-9b2e-b30b08178ce6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 226
        }
      },
      "source": [
        "for i,j in enumerate(col):\n",
        "    sub[j]=y_test[:,i]\n",
        "sub.head()"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>image</th>\n",
              "      <th>MEL</th>\n",
              "      <th>NV</th>\n",
              "      <th>BCC</th>\n",
              "      <th>AK</th>\n",
              "      <th>BKL</th>\n",
              "      <th>DF</th>\n",
              "      <th>VASC</th>\n",
              "      <th>SCC</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ISIC_0034321</td>\n",
              "      <td>2.411615e-03</td>\n",
              "      <td>9.950216e-03</td>\n",
              "      <td>9.951366e-01</td>\n",
              "      <td>1.865926e-06</td>\n",
              "      <td>1.149814e-03</td>\n",
              "      <td>5.015582e-07</td>\n",
              "      <td>1.408054e-07</td>\n",
              "      <td>2.692782e-05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>ISIC_0034322</td>\n",
              "      <td>1.339413e-10</td>\n",
              "      <td>9.999999e-01</td>\n",
              "      <td>2.802629e-18</td>\n",
              "      <td>1.268978e-24</td>\n",
              "      <td>2.317892e-11</td>\n",
              "      <td>3.138214e-30</td>\n",
              "      <td>4.118626e-30</td>\n",
              "      <td>5.309550e-28</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>ISIC_0034323</td>\n",
              "      <td>4.133657e-07</td>\n",
              "      <td>1.456745e-07</td>\n",
              "      <td>9.999576e-01</td>\n",
              "      <td>2.059041e-07</td>\n",
              "      <td>4.149686e-07</td>\n",
              "      <td>1.425497e-11</td>\n",
              "      <td>5.634845e-15</td>\n",
              "      <td>2.874657e-04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ISIC_0034324</td>\n",
              "      <td>3.775400e-07</td>\n",
              "      <td>9.999977e-01</td>\n",
              "      <td>1.218146e-12</td>\n",
              "      <td>2.038537e-19</td>\n",
              "      <td>5.121590e-11</td>\n",
              "      <td>4.035054e-22</td>\n",
              "      <td>8.844212e-16</td>\n",
              "      <td>7.612656e-22</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>ISIC_0034325</td>\n",
              "      <td>8.950728e-10</td>\n",
              "      <td>9.999996e-01</td>\n",
              "      <td>1.957036e-14</td>\n",
              "      <td>2.389760e-23</td>\n",
              "      <td>2.163081e-12</td>\n",
              "      <td>3.957146e-29</td>\n",
              "      <td>1.102805e-25</td>\n",
              "      <td>2.668486e-28</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "          image           MEL  ...          VASC           SCC\n",
              "0  ISIC_0034321  2.411615e-03  ...  1.408054e-07  2.692782e-05\n",
              "1  ISIC_0034322  1.339413e-10  ...  4.118626e-30  5.309550e-28\n",
              "2  ISIC_0034323  4.133657e-07  ...  5.634845e-15  2.874657e-04\n",
              "3  ISIC_0034324  3.775400e-07  ...  8.844212e-16  7.612656e-22\n",
              "4  ISIC_0034325  8.950728e-10  ...  1.102805e-25  2.668486e-28\n",
              "\n",
              "[5 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HY35Yr50ir1O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sub.to_csv('inceptionresnetv2.csv', index=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jJsz3WmRbVvI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}