IQF-UseCase.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment Setup\n",
    "\n",
    "1. Define name of IQF experiment\n",
    "1. Define path of the original(reference) dataset\n",
    "1. Define path of the training script\n",
    "1. DS_wrapper is the class that encapsulate a dataset\n",
    "1. Define the list of modifications that will be applyed to the original dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES']='1'\n",
    "os.chdir('yolov5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import shutil\n",
    "\n",
    "from iquaflow.datasets import DSModifier, DSWrapper, DSModifier_jpg\n",
    "from iquaflow.experiments import ExperimentInfo, ExperimentSetup\n",
    "from iquaflow.experiments.experiment_visual import ExperimentVisual\n",
    "from iquaflow.experiments.task_execution import PythonScriptTaskExecution\n",
    "from iquaflow.metrics import BBDetectionMetrics\n",
    "\n",
    "from custom_iqf import DSModifierJPG, gen_dataset_partitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define name of IQF experiment\n",
    "experiment_name = \"airportV2\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " * DSModifier refers to a modifier that simply does nothing - Reference training\n",
    " * DSModifier_jpg apply JPG compresion to the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "#List of modifications that will be applied to the original dataset:\n",
    "ds_modifiers_list = [\n",
    "    DSModifier_jpg(params={'quality': i})\n",
    "    for i in [10,20, 30,50,70,90]\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Task execution executes the training loop\n",
    "task = PythonScriptTaskExecution(model_script_path = 'train.py')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Execution"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Experiment definition, pass as arguments all the components defined beforehand:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for random_seed in [\n",
    "    47625, \n",
    "    29884, \n",
    "    51915, \n",
    "    89294, \n",
    "    71825, \n",
    "    57277, \n",
    "    39673, \n",
    "    80043, \n",
    "    14516, \n",
    "    90692\n",
    "]:\n",
    "    \n",
    "    ref_dsw_train, ref_dsw_val = gen_dataset_partitions(\n",
    "        random_seed,\n",
    "        dsdir = 'datasets/alldata',\n",
    "        ratio_train_val=0.2\n",
    "    )\n",
    "\n",
    "    #Experiment definition, pass as arguments all the components defined beforehand\n",
    "    experiment = ExperimentSetup(\n",
    "        experiment_name   = experiment_name,\n",
    "        task_instance     = task,\n",
    "        ref_dsw_train     = ref_dsw_train,\n",
    "        ref_dsw_val       = ref_dsw_val,\n",
    "        ds_modifiers_list = ds_modifiers_list,\n",
    "        repetitions=1,\n",
    "        cloud_options={'tracking_uri': 'https://mlflow.ml.analytics-dev.satellogic.team/'},\n",
    "        extra_train_params={\"weights\":[\n",
    "            'yolov5n.pt',\n",
    "            'yolov5s.pt'\n",
    "        ]}\n",
    "    )\n",
    "\n",
    "    #Execute the experiment\n",
    "    experiment.execute()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pick-up results / metrics and plot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " * Define name of IQF experiment\n",
    " * Define path of the original(reference) dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#Define name of IQF experiment\n",
    "experiment_name = \"airport\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load experiment info\n",
    " - ExperimentInfo is used to retrieve all the information of the whole experiment. \n",
    " - It contains built in operations but also it can be used to retrieve raw data for futher analysis\n",
    " - The get_df method will sort data in a dataframe suitable for the visualizations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_info = ExperimentInfo(experiment_name)\n",
    "\n",
    "df = experiment_info.get_df(\n",
    "    ds_params=[\"quality\",'modelSize'],\n",
    "    metrics=['P', 'R', 'mAP_0.5', 'mAP_0.5_0.95'],\n",
    "    dropna = True,\n",
    "    fields_to_float_lst = ['P', 'R', 'mAP_0.5', 'mAP_0.5_0.95'],\n",
    "    fields_to_int_lst = ['ds_quality']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>ds_quality</th>\n",
       "      <th>ds_modelSize</th>\n",
       "      <th>P</th>\n",
       "      <th>R</th>\n",
       "      <th>mAP_0.5</th>\n",
       "      <th>mAP_0.5_0.95</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train#jpg50_modifier</td>\n",
       "      <td>50</td>\n",
       "      <td>S</td>\n",
       "      <td>0.974136</td>\n",
       "      <td>0.916649</td>\n",
       "      <td>0.964987</td>\n",
       "      <td>0.708531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train#jpg50_modifier_0</td>\n",
       "      <td>50</td>\n",
       "      <td>S</td>\n",
       "      <td>0.975428</td>\n",
       "      <td>0.921442</td>\n",
       "      <td>0.962092</td>\n",
       "      <td>0.707859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train#jpg50_modifier_1</td>\n",
       "      <td>50</td>\n",
       "      <td>S</td>\n",
       "      <td>0.975556</td>\n",
       "      <td>0.919901</td>\n",
       "      <td>0.963683</td>\n",
       "      <td>0.708527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train#jpg90_modifier</td>\n",
       "      <td>90</td>\n",
       "      <td>N</td>\n",
       "      <td>0.968384</td>\n",
       "      <td>0.896416</td>\n",
       "      <td>0.951973</td>\n",
       "      <td>0.670115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train#jpg50_modifier_2</td>\n",
       "      <td>50</td>\n",
       "      <td>S</td>\n",
       "      <td>0.972752</td>\n",
       "      <td>0.916821</td>\n",
       "      <td>0.958584</td>\n",
       "      <td>0.706375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>train#jpg90_modifier_0</td>\n",
       "      <td>90</td>\n",
       "      <td>N</td>\n",
       "      <td>0.959255</td>\n",
       "      <td>0.903266</td>\n",
       "      <td>0.948764</td>\n",
       "      <td>0.668735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>train#jpg50_modifier_3</td>\n",
       "      <td>50</td>\n",
       "      <td>S</td>\n",
       "      <td>0.971952</td>\n",
       "      <td>0.913432</td>\n",
       "      <td>0.958990</td>\n",
       "      <td>0.703889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>train#jpg90_modifier_1</td>\n",
       "      <td>90</td>\n",
       "      <td>N</td>\n",
       "      <td>0.972253</td>\n",
       "      <td>0.898336</td>\n",
       "      <td>0.954132</td>\n",
       "      <td>0.669647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>train#jpg30_modifier</td>\n",
       "      <td>30</td>\n",
       "      <td>S</td>\n",
       "      <td>0.969467</td>\n",
       "      <td>0.920826</td>\n",
       "      <td>0.958161</td>\n",
       "      <td>0.700475</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>train#jpg90_modifier_2</td>\n",
       "      <td>90</td>\n",
       "      <td>N</td>\n",
       "      <td>0.966624</td>\n",
       "      <td>0.896488</td>\n",
       "      <td>0.954160</td>\n",
       "      <td>0.669108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>train#jpg30_modifier_0</td>\n",
       "      <td>30</td>\n",
       "      <td>S</td>\n",
       "      <td>0.974037</td>\n",
       "      <td>0.913056</td>\n",
       "      <td>0.960612</td>\n",
       "      <td>0.703326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>train#jpg90_modifier_3</td>\n",
       "      <td>90</td>\n",
       "      <td>N</td>\n",
       "      <td>0.959651</td>\n",
       "      <td>0.906962</td>\n",
       "      <td>0.956425</td>\n",
       "      <td>0.669292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>train#jpg30_modifier_1</td>\n",
       "      <td>30</td>\n",
       "      <td>S</td>\n",
       "      <td>0.972869</td>\n",
       "      <td>0.916892</td>\n",
       "      <td>0.962368</td>\n",
       "      <td>0.705903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>train#jpg70_modifier</td>\n",
       "      <td>70</td>\n",
       "      <td>N</td>\n",
       "      <td>0.965725</td>\n",
       "      <td>0.894066</td>\n",
       "      <td>0.949841</td>\n",
       "      <td>0.664586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>train#jpg30_modifier_2</td>\n",
       "      <td>30</td>\n",
       "      <td>S</td>\n",
       "      <td>0.973403</td>\n",
       "      <td>0.915588</td>\n",
       "      <td>0.962901</td>\n",
       "      <td>0.703875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>train#jpg70_modifier_0</td>\n",
       "      <td>70</td>\n",
       "      <td>N</td>\n",
       "      <td>0.963230</td>\n",
       "      <td>0.903880</td>\n",
       "      <td>0.951647</td>\n",
       "      <td>0.666226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>train#jpg30_modifier_3</td>\n",
       "      <td>30</td>\n",
       "      <td>S</td>\n",
       "      <td>0.968680</td>\n",
       "      <td>0.914716</td>\n",
       "      <td>0.956908</td>\n",
       "      <td>0.701782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>train#jpg70_modifier_1</td>\n",
       "      <td>70</td>\n",
       "      <td>N</td>\n",
       "      <td>0.971566</td>\n",
       "      <td>0.898028</td>\n",
       "      <td>0.952500</td>\n",
       "      <td>0.666669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>train#jpg10_modifier</td>\n",
       "      <td>10</td>\n",
       "      <td>S</td>\n",
       "      <td>0.964393</td>\n",
       "      <td>0.892790</td>\n",
       "      <td>0.942311</td>\n",
       "      <td>0.671416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>train#jpg70_modifier_2</td>\n",
       "      <td>70</td>\n",
       "      <td>N</td>\n",
       "      <td>0.958538</td>\n",
       "      <td>0.907270</td>\n",
       "      <td>0.952223</td>\n",
       "      <td>0.666466</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>train#jpg10_modifier_0</td>\n",
       "      <td>10</td>\n",
       "      <td>S</td>\n",
       "      <td>0.976354</td>\n",
       "      <td>0.890418</td>\n",
       "      <td>0.948040</td>\n",
       "      <td>0.677843</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>train#jpg70_modifier_3</td>\n",
       "      <td>70</td>\n",
       "      <td>N</td>\n",
       "      <td>0.959245</td>\n",
       "      <td>0.899126</td>\n",
       "      <td>0.944716</td>\n",
       "      <td>0.662611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>train#jpg10_modifier_1</td>\n",
       "      <td>10</td>\n",
       "      <td>S</td>\n",
       "      <td>0.968111</td>\n",
       "      <td>0.893715</td>\n",
       "      <td>0.945405</td>\n",
       "      <td>0.674751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>train#jpg50_modifier_4</td>\n",
       "      <td>50</td>\n",
       "      <td>N</td>\n",
       "      <td>0.963408</td>\n",
       "      <td>0.892218</td>\n",
       "      <td>0.947997</td>\n",
       "      <td>0.663420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>train#jpg10_modifier_2</td>\n",
       "      <td>10</td>\n",
       "      <td>S</td>\n",
       "      <td>0.976500</td>\n",
       "      <td>0.886630</td>\n",
       "      <td>0.945223</td>\n",
       "      <td>0.676497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>train#jpg50_modifier_5</td>\n",
       "      <td>50</td>\n",
       "      <td>N</td>\n",
       "      <td>0.962122</td>\n",
       "      <td>0.899904</td>\n",
       "      <td>0.949347</td>\n",
       "      <td>0.662803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>train#jpg10_modifier_3</td>\n",
       "      <td>10</td>\n",
       "      <td>S</td>\n",
       "      <td>0.972696</td>\n",
       "      <td>0.890943</td>\n",
       "      <td>0.946140</td>\n",
       "      <td>0.672785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>train#jpg50_modifier_6</td>\n",
       "      <td>50</td>\n",
       "      <td>N</td>\n",
       "      <td>0.963256</td>\n",
       "      <td>0.888376</td>\n",
       "      <td>0.943721</td>\n",
       "      <td>0.661054</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>train#jpg50_modifier_7</td>\n",
       "      <td>50</td>\n",
       "      <td>N</td>\n",
       "      <td>0.966971</td>\n",
       "      <td>0.886938</td>\n",
       "      <td>0.944628</td>\n",
       "      <td>0.663686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>train#jpg90_modifier_4</td>\n",
       "      <td>90</td>\n",
       "      <td>S</td>\n",
       "      <td>0.972941</td>\n",
       "      <td>0.921750</td>\n",
       "      <td>0.966593</td>\n",
       "      <td>0.714393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>train#jpg50_modifier_8</td>\n",
       "      <td>50</td>\n",
       "      <td>N</td>\n",
       "      <td>0.958834</td>\n",
       "      <td>0.896954</td>\n",
       "      <td>0.949093</td>\n",
       "      <td>0.662754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>train#jpg90_modifier_5</td>\n",
       "      <td>90</td>\n",
       "      <td>S</td>\n",
       "      <td>0.976915</td>\n",
       "      <td>0.918053</td>\n",
       "      <td>0.961937</td>\n",
       "      <td>0.713436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>train#jpg30_modifier_4</td>\n",
       "      <td>30</td>\n",
       "      <td>N</td>\n",
       "      <td>0.967160</td>\n",
       "      <td>0.885397</td>\n",
       "      <td>0.947060</td>\n",
       "      <td>0.656489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>train#jpg90_modifier_6</td>\n",
       "      <td>90</td>\n",
       "      <td>S</td>\n",
       "      <td>0.972247</td>\n",
       "      <td>0.919901</td>\n",
       "      <td>0.963972</td>\n",
       "      <td>0.710945</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>train#jpg30_modifier_5</td>\n",
       "      <td>30</td>\n",
       "      <td>N</td>\n",
       "      <td>0.966243</td>\n",
       "      <td>0.881810</td>\n",
       "      <td>0.948280</td>\n",
       "      <td>0.658483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>train#jpg90_modifier_7</td>\n",
       "      <td>90</td>\n",
       "      <td>S</td>\n",
       "      <td>0.969068</td>\n",
       "      <td>0.930376</td>\n",
       "      <td>0.968941</td>\n",
       "      <td>0.713545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>train#jpg30_modifier_6</td>\n",
       "      <td>30</td>\n",
       "      <td>N</td>\n",
       "      <td>0.962275</td>\n",
       "      <td>0.887974</td>\n",
       "      <td>0.944304</td>\n",
       "      <td>0.656072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>train#jpg90_modifier_8</td>\n",
       "      <td>90</td>\n",
       "      <td>S</td>\n",
       "      <td>0.976293</td>\n",
       "      <td>0.922366</td>\n",
       "      <td>0.967920</td>\n",
       "      <td>0.714896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>train#jpg30_modifier_7</td>\n",
       "      <td>30</td>\n",
       "      <td>N</td>\n",
       "      <td>0.959494</td>\n",
       "      <td>0.896180</td>\n",
       "      <td>0.946317</td>\n",
       "      <td>0.657495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>train#jpg70_modifier_4</td>\n",
       "      <td>70</td>\n",
       "      <td>S</td>\n",
       "      <td>0.974043</td>\n",
       "      <td>0.913275</td>\n",
       "      <td>0.957322</td>\n",
       "      <td>0.706717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>train#jpg30_modifier_8</td>\n",
       "      <td>30</td>\n",
       "      <td>N</td>\n",
       "      <td>0.963595</td>\n",
       "      <td>0.888806</td>\n",
       "      <td>0.945534</td>\n",
       "      <td>0.657670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>train#jpg10_modifier_4</td>\n",
       "      <td>10</td>\n",
       "      <td>N</td>\n",
       "      <td>0.965622</td>\n",
       "      <td>0.873980</td>\n",
       "      <td>0.935358</td>\n",
       "      <td>0.635832</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>train#jpg70_modifier_5</td>\n",
       "      <td>70</td>\n",
       "      <td>S</td>\n",
       "      <td>0.974466</td>\n",
       "      <td>0.917040</td>\n",
       "      <td>0.963296</td>\n",
       "      <td>0.708726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>train#jpg10_modifier_5</td>\n",
       "      <td>10</td>\n",
       "      <td>N</td>\n",
       "      <td>0.975521</td>\n",
       "      <td>0.871654</td>\n",
       "      <td>0.936181</td>\n",
       "      <td>0.635952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>train#jpg70_modifier_6</td>\n",
       "      <td>70</td>\n",
       "      <td>S</td>\n",
       "      <td>0.973897</td>\n",
       "      <td>0.919528</td>\n",
       "      <td>0.964793</td>\n",
       "      <td>0.711364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>train#jpg10_modifier_6</td>\n",
       "      <td>10</td>\n",
       "      <td>N</td>\n",
       "      <td>0.966550</td>\n",
       "      <td>0.872370</td>\n",
       "      <td>0.934420</td>\n",
       "      <td>0.636233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>train#jpg70_modifier_7</td>\n",
       "      <td>70</td>\n",
       "      <td>S</td>\n",
       "      <td>0.972301</td>\n",
       "      <td>0.919202</td>\n",
       "      <td>0.963217</td>\n",
       "      <td>0.708689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>train#jpg10_modifier_7</td>\n",
       "      <td>10</td>\n",
       "      <td>N</td>\n",
       "      <td>0.967757</td>\n",
       "      <td>0.873691</td>\n",
       "      <td>0.942091</td>\n",
       "      <td>0.636890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>train#jpg70_modifier_8</td>\n",
       "      <td>70</td>\n",
       "      <td>S</td>\n",
       "      <td>0.972895</td>\n",
       "      <td>0.921750</td>\n",
       "      <td>0.968295</td>\n",
       "      <td>0.709997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>train#jpg10_modifier_8</td>\n",
       "      <td>10</td>\n",
       "      <td>N</td>\n",
       "      <td>0.968493</td>\n",
       "      <td>0.871232</td>\n",
       "      <td>0.933772</td>\n",
       "      <td>0.637196</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      name  ds_quality ds_modelSize         P         R  \\\n",
       "0     train#jpg50_modifier          50            S  0.974136  0.916649   \n",
       "1   train#jpg50_modifier_0          50            S  0.975428  0.921442   \n",
       "2   train#jpg50_modifier_1          50            S  0.975556  0.919901   \n",
       "3     train#jpg90_modifier          90            N  0.968384  0.896416   \n",
       "4   train#jpg50_modifier_2          50            S  0.972752  0.916821   \n",
       "5   train#jpg90_modifier_0          90            N  0.959255  0.903266   \n",
       "6   train#jpg50_modifier_3          50            S  0.971952  0.913432   \n",
       "7   train#jpg90_modifier_1          90            N  0.972253  0.898336   \n",
       "8     train#jpg30_modifier          30            S  0.969467  0.920826   \n",
       "9   train#jpg90_modifier_2          90            N  0.966624  0.896488   \n",
       "10  train#jpg30_modifier_0          30            S  0.974037  0.913056   \n",
       "11  train#jpg90_modifier_3          90            N  0.959651  0.906962   \n",
       "12  train#jpg30_modifier_1          30            S  0.972869  0.916892   \n",
       "13    train#jpg70_modifier          70            N  0.965725  0.894066   \n",
       "14  train#jpg30_modifier_2          30            S  0.973403  0.915588   \n",
       "15  train#jpg70_modifier_0          70            N  0.963230  0.903880   \n",
       "16  train#jpg30_modifier_3          30            S  0.968680  0.914716   \n",
       "17  train#jpg70_modifier_1          70            N  0.971566  0.898028   \n",
       "18    train#jpg10_modifier          10            S  0.964393  0.892790   \n",
       "19  train#jpg70_modifier_2          70            N  0.958538  0.907270   \n",
       "20  train#jpg10_modifier_0          10            S  0.976354  0.890418   \n",
       "21  train#jpg70_modifier_3          70            N  0.959245  0.899126   \n",
       "22  train#jpg10_modifier_1          10            S  0.968111  0.893715   \n",
       "23  train#jpg50_modifier_4          50            N  0.963408  0.892218   \n",
       "24  train#jpg10_modifier_2          10            S  0.976500  0.886630   \n",
       "25  train#jpg50_modifier_5          50            N  0.962122  0.899904   \n",
       "26  train#jpg10_modifier_3          10            S  0.972696  0.890943   \n",
       "27  train#jpg50_modifier_6          50            N  0.963256  0.888376   \n",
       "28  train#jpg50_modifier_7          50            N  0.966971  0.886938   \n",
       "29  train#jpg90_modifier_4          90            S  0.972941  0.921750   \n",
       "30  train#jpg50_modifier_8          50            N  0.958834  0.896954   \n",
       "31  train#jpg90_modifier_5          90            S  0.976915  0.918053   \n",
       "32  train#jpg30_modifier_4          30            N  0.967160  0.885397   \n",
       "33  train#jpg90_modifier_6          90            S  0.972247  0.919901   \n",
       "34  train#jpg30_modifier_5          30            N  0.966243  0.881810   \n",
       "35  train#jpg90_modifier_7          90            S  0.969068  0.930376   \n",
       "36  train#jpg30_modifier_6          30            N  0.962275  0.887974   \n",
       "37  train#jpg90_modifier_8          90            S  0.976293  0.922366   \n",
       "38  train#jpg30_modifier_7          30            N  0.959494  0.896180   \n",
       "39  train#jpg70_modifier_4          70            S  0.974043  0.913275   \n",
       "40  train#jpg30_modifier_8          30            N  0.963595  0.888806   \n",
       "41  train#jpg10_modifier_4          10            N  0.965622  0.873980   \n",
       "42  train#jpg70_modifier_5          70            S  0.974466  0.917040   \n",
       "43  train#jpg10_modifier_5          10            N  0.975521  0.871654   \n",
       "44  train#jpg70_modifier_6          70            S  0.973897  0.919528   \n",
       "45  train#jpg10_modifier_6          10            N  0.966550  0.872370   \n",
       "46  train#jpg70_modifier_7          70            S  0.972301  0.919202   \n",
       "47  train#jpg10_modifier_7          10            N  0.967757  0.873691   \n",
       "48  train#jpg70_modifier_8          70            S  0.972895  0.921750   \n",
       "49  train#jpg10_modifier_8          10            N  0.968493  0.871232   \n",
       "\n",
       "     mAP_0.5  mAP_0.5_0.95  \n",
       "0   0.964987      0.708531  \n",
       "1   0.962092      0.707859  \n",
       "2   0.963683      0.708527  \n",
       "3   0.951973      0.670115  \n",
       "4   0.958584      0.706375  \n",
       "5   0.948764      0.668735  \n",
       "6   0.958990      0.703889  \n",
       "7   0.954132      0.669647  \n",
       "8   0.958161      0.700475  \n",
       "9   0.954160      0.669108  \n",
       "10  0.960612      0.703326  \n",
       "11  0.956425      0.669292  \n",
       "12  0.962368      0.705903  \n",
       "13  0.949841      0.664586  \n",
       "14  0.962901      0.703875  \n",
       "15  0.951647      0.666226  \n",
       "16  0.956908      0.701782  \n",
       "17  0.952500      0.666669  \n",
       "18  0.942311      0.671416  \n",
       "19  0.952223      0.666466  \n",
       "20  0.948040      0.677843  \n",
       "21  0.944716      0.662611  \n",
       "22  0.945405      0.674751  \n",
       "23  0.947997      0.663420  \n",
       "24  0.945223      0.676497  \n",
       "25  0.949347      0.662803  \n",
       "26  0.946140      0.672785  \n",
       "27  0.943721      0.661054  \n",
       "28  0.944628      0.663686  \n",
       "29  0.966593      0.714393  \n",
       "30  0.949093      0.662754  \n",
       "31  0.961937      0.713436  \n",
       "32  0.947060      0.656489  \n",
       "33  0.963972      0.710945  \n",
       "34  0.948280      0.658483  \n",
       "35  0.968941      0.713545  \n",
       "36  0.944304      0.656072  \n",
       "37  0.967920      0.714896  \n",
       "38  0.946317      0.657495  \n",
       "39  0.957322      0.706717  \n",
       "40  0.945534      0.657670  \n",
       "41  0.935358      0.635832  \n",
       "42  0.963296      0.708726  \n",
       "43  0.936181      0.635952  \n",
       "44  0.964793      0.711364  \n",
       "45  0.934420      0.636233  \n",
       "46  0.963217      0.708689  \n",
       "47  0.942091      0.636890  \n",
       "48  0.968295      0.709997  \n",
       "49  0.933772      0.637196  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualize\n",
    " - Plotting utilities are available in it_tool_box"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEaCAYAAAAcz1CnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAdDklEQVR4nO3deZgV5Zn+8e/NImhg3AA1NlsSUDGgkFZjXEYTjGAMauICSBRc0InEZBKT4G+MgvMzrmOiCRMHF3ANLiEKAZVEZTBxAwRBJSgqSoMKoqKAiMgzf5xqumloOL3UOd1d9+e6+vJU1VtVT5fa96m3qt5SRGBmZtnVrNgFmJlZcTkIzMwyzkFgZpZxDgIzs4xzEJiZZZyDwMws4xwEZg2MpPGS/n/y+QhJC4tdkzVtDgJrciRNl/SBpFZV5o+XtF7SaknvS/qrpH2LVWc+IuLJiNinfFrSYkl9i1mTNT0OAmtSJHUBjgACGLCVJtdERBugBFgOjC9YcWYNlIPAmpozgGfI/YE/s7pGEbEWuAf4am12Iqm5pOskvSfpdUkXSApJLZLlm31zlzRK0l2Vpu+X9I6kVZJmSNq/mv0cJaks+Xwn0AmYnJzV/ELSFEk/qrLOPEkn1eb3smxyEFhTcwZwd/JzrKQ9ttZIUhvgdGBOLfdzLnA80BsoBU6u4foPA92ADsDzSb3bFBE/AN4CvhsRbSLiGuB2YEh5G0kHAHsDU2pYj2WYg8CaDEmHA52B+yJiNvAaMLhKs4skfQgsAtoAQ2u5u1OB30bEkoh4H7iyJitHxG0R8XFEfAqMAg6QtHMt6pgEdJfULZn+AXBvRKyvxbYsoxwE1pScCUyLiPeS6XvYsnvouojYJSL2jIgBEfFa1Y1I6pR0vayWtLqafX0RWFJp+s18i0y6la6S9Jqkj4DFyaJ2+W6jXESsA+4FhkhqBgwC7qzpdizbWhS7ALP6IGlHct/Sm0t6J5ndCthF0gER8UK+24qIt8idLWzL20DHStOdqixfA+xUaXrPSp8HAycAfcmFwM7AB4DyKW8r824n98f/78DaiHg6j+2YbeIzAmsqTgQ+B3oAByY/+wFPkrtuUN/uAy6UVCJpV2BkleVzgYGSWkqqeg2hLfApsJJcWPy6Bvt9F/hS5RnJH/6NwH/hswGrBQeBNRVnAuMi4q2IeKf8B/g9cHr53Tz16GbgUeAFchd7J1ZZ/ivgy+S+6Y8m101V7g5yXUlLgZfJ3eWUryuBSyR9KOmiKtvsCdy19dXMqie/mMas7pLnF94AWkbEhiLs/wxgeEQcXuh9W+PnMwKzRk7STsAPgbHFrsUaJweBWSMm6VhgBblrB/dsp7nZVrlryMws43xGYGaWcakGgaTbJC2X9GI1yyXpRkmLkvFR+qRZj5mZbSntB8rGk7t9745qlvcnN95KN+AQ4A/JP7epXbt20aVLl/qp0MwsI2bPnv1eRLSvOj/VIIiIGcltddU5AbgjchcqnpG0i6S9IuLtbW23S5cuzJo1qz5LNTNr8iRtdSiUYl8j2JvNx2spS+aZmVmBFDsI8iZpuKRZkmatWLGi2OWYmTUZxQ6CpWw+cFdJMm8LETE2IkojorR9+y26uMzMrJaKPfroJGCEpAnkLhKv2t71ATPLps8++4yysjLWrVtX7FIavNatW1NSUkLLli3zap9qEEj6I3AU0C553d5lQEuAiLgJmAocR+4lIWuBYWnWY2aNV1lZGW3btqVLly5I+YzYnU0RwcqVKykrK6Nr1655rZP2XUODtrM8gAvSrMHMmoZ169Y5BPIgid13352aXEst9jUCM7O8OQTyU9Pj5CAwM8s4B4GZNQmjRo3iuuuuK8q+p0+fzvHHH199g2VzmP7AzRzf9whYNod3X/gbx/c9ggN6dKdH9y9x3LcOh2VzWPb8NE7+Tl9YNmfLnxQV+64hawpG7VwP21hV9200BD4WFXwsqnXptX/gmCO/zo/PGQzAvJdfAeCLe7bngZuvLXg9PiMws0briiuuoHv37hx++OEsXLgQgBtvvJEePXrQq1cvBg4cWO26o0aN4swzz+SII46gc+fOTJw4kV/84hf07NmTfv368dlnnwHw2GOP0bt3b3r27MlZZ53Fp59+CsAjjzzCvvvuS58+fZg4seJNpWvWrOGss87i4IMPpnfv3jz00ENb7Pvt5e9Rstcem6Z79egOwOIly/jqN08B4JyLLufAYwZy4DEDad/zm4wePRqAa6+9loMOOohevXpx2WWX1eXwbeIgMLNGafbs2UyYMIG5c+cydepUZs6cCcBVV13FnDlzmDdvHjfddNM2t/Haa6/x+OOPM2nSJIYMGcLRRx/N/Pnz2XHHHZkyZQrr1q1j6NChjL7hZu5++ElWfLSWX115PTMXvcPQs87m2pvvZtxDj7HwjSV8tO4z5pV9yI9/+Su+cuAh3DJxGr+760Eu/PefsWbtJ5vt94Khp3L2RaM5+uThXHHDLSx7Z8s7fG657lLm/nUCD427nna77cLQoUOZNm0ar776Ks899xxz585l9uzZzJgxo87H0l1DRpeRU+q0/uLW9VRIA+BjUaGhH4snn3ySk046iZ122gmAAQMGANCrVy9OP/10TjzxRE488cRtbqN///60bNmSnj178vnnn9OvXz8AevbsyeLFi1m4cCFdu3aly5e+ktvHyYOYcPstlB56OHt37Eznrl8G4DsnncID99wOwNMznmD6Xx/mjv/5PQDrP13HW0s3f0722KO+wetPTeaR6U/x8OP/oPexg3jx8fu3qG/duk855bxf8rv//CWdO3fmd7/7HdOmTaN3794ArF69mldffZUjjzyyNodwEwdBbbn/06xBmjJlCjNmzGDy5MlcccUVzJ8/nxYttv6nrlWrVgA0a9aMli1bbrrtslmzZmzYsKFW+w+C68feQZcvd9s0b79mb/Duivc3a7fbrjsz+KT+DD6pP8efcSEznnmer/Xab7M254/8Nd/r/036HpkbnT8iuPjiiznvvPNqVVt13DVkZo3SkUceyYMPPsgnn3zCxx9/zOTJk9m4cSNLlizh6KOP5uqrr2bVqlWsXr261vvYZ599WLx4MW+98ToAf/nTvZR+/TC6frkby8reYsniNwB4+KE/bVrnG0d+k3vGjaX8NcALXpy3xXYf//tzrP0k11308eo1vPZmGZ323nOzNmPG38vHa9YwckTFgAvHHnsst91226bfaenSpSxfvrzWv1+5zJ4RNPTTXjPbtj59+nDaaadxwAEH0KFDBw466CAkMWTIEFatWkVEcOGFF7LLLrvUeh+tW7dm3LhxXPBvQ/l8wwb2P6APpwwZxg6tWnHpVb9lxNDTaL3jjvQ5+FDWrMn9cR7+459zzeiLOfmYw9gYwd4dO3HaHVdttt3Z8xcw4pKradGiORs3BucMOomDDtyfxUuWbWpz3U130rJFCw48JnfB+/wf/Tvnn38+CxYs4NBDDwWgTZs23HXXXXTo0KHWvyM00pfXl5aWRl1fTFP3IBhcp/WBBtM15GNRwceiQkM7FgsWLGC//fbbRuP0zCv7sE7r92r2Rt2L+GLvGjXf2vGSNDsiSqu2ddeQmVnGZbZryMyyYdy4cdxwww2bzTvssMMYM2ZMkSpqeBwEZtakDRs2jGHDPML9trhryMws4xwEZmYZ5yAwM8s4B4GZWYFo7z78bPT1m6avu+kORv3XtsdDKgRfLDazTKrrcxJVTRpx2HbbtGq1AxMffpyLfzSMdrvtWq/7rwufEZiZFUiL5s0Zfvr3+M3Yu4tdymYcBGZmBXTB0FO5+88Ps+qjj4tdyiYOAjOzAvqXtm044+TjufHWCcUuZRMHgZlZgf3knMHcOuHBLV5YUywOAjOzAttt15059bvHcOsft3yNZTE4CMzMiuBn5/2A997/sNhlAL591MwyavFV38m7bV2HoS63+tV/bPq8R/vdWfvaU/Wy3bryGYGZWcY5CMzMMs5BYGaWcQ4CM7OMcxCYmWWcg8DMLOMcBGZmBXTFDbew/9En06vvqRx4zECefX5+sUvycwRmllGjds67aa882sw7583ttnl61gv85W9P8vwj99Cq1Q689/4HrF+/Ie860pL6GYGkfpIWSlokaeRWlneS9ISkOZLmSTou7ZrMzIrh7eXv0W63XWjVagcA2u22K1/cs32Rq0o5CCQ1B8YA/YEewCBJPao0uwS4LyJ6AwOB/06zJjOzYvn2vx7KkmXv0v3wE/nhxVfyv0/PLnZJQPpnBAcDiyLi9YhYD0wATqjSJoB/ST7vDCxLuSYzs6Jo84WdmP3I3Yy95hLa774Lp/3bSMbfO6nYZaV+jWBvYEml6TLgkCptRgHTJP0I+ALQN+WazMyKpnnz5hz1jVKO+kYpPfftxu33T2boaQOKWlNDuGtoEDA+IkqA44A7JW1Rl6ThkmZJmrVixYqCF2lmVlcLFy3m1dff2jQ996WFdC7Zq4gV5aR9RrAU6FhpuiSZV9nZQD+AiHhaUmugHbC8cqOIGAuMBSgtLY20CjYzS8vqtWv50SXX8OFHH9OiRXO+0qUjY6+5pNhlpR4EM4FukrqSC4CBwOAqbd4CvgWMl7Qf0BrwV34zS9eoVXk3ra9hqL/WqwdPTRpfL9uqT6l2DUXEBmAE8CiwgNzdQS9JulxSeafYz4BzJb0A/BEYGhH+xm9mViCpP1AWEVOBqVXmXVrp88vAYWnXYWZmW9cQLhabmVkROQjMrNFwr3F+anqcHARm1ii0bt2alStXOgy2IyJYuXIlrVu3znsdDzpnZo1CSUkJZWVlFOM5onc/+KRO6y9QPdS8akHeTVu3bk1JSUne7R0EZtYotGzZkq5duxZl3/1HTqnT+otbV71rvhZqcLtrTblryMws4xwEZmYZ5yAwM8s4B4GZWcY5CMzMMs5BYGaWcQ4CM7OMcxCYmWWcg8DMLOMcBGZmGecgMDPLOAeBmVnGOQjMzDLOQWBmlnEOAjOzjHMQmJllnIPAzCzjHARmZhnnIDAzyzgHgZlZxjkIzMwyzkFgZpZxDgIzs4xzEJiZZZyDwMws4xwEZmYZ5yAwM8s4B4GZWcY5CMzMMi71IJDUT9JCSYskjaymzamSXpb0kqR70q7JzMwqtEhz45KaA2OAY4AyYKakSRHxcqU23YCLgcMi4gNJHdKsyczMNpf2GcHBwKKIeD0i1gMTgBOqtDkXGBMRHwBExPKUazIzs0rSDoK9gSWVpsuSeZV1B7pL+oekZyT1S7kmMzOrJNWuoTy1ALoBRwElwAxJPSPiw8qNJA0HhgN06tSpwCWamTVdNT4jkHS4pJ9K+nYezZcCHStNlyTzKisDJkXEZxHxBvAKuWDYTESMjYjSiCht3759Tcs2M7NqbDcIJD1X6fO5wO+BtsBl1d0FVMlMoJukrpJ2AAYCk6q0eZDc2QCS2pHrKno9z/rNzKyO8jkjaFnp83DgmIgYDXwbOH1bK0bEBmAE8CiwALgvIl6SdLmkAUmzR4GVkl4GngB+HhEra/h7mJlZLeVzjaCZpF3JhYYiYgVARKyRtGF7K0fEVGBqlXmXVvocwE+THzMzK7B8gmBnYDYgICTtFRFvS2qTzDMzs0Zsu0EQEV2qWbQROKleqzEzs4Kr9e2jEbEWeKMeazEzsyKo0wNlkv5SX4WYmVlx1PXJ4nPrpQozMyuaGgWBpN0k7VY+HRFv139JZmZWSPk8UNZJ0gRJK4BngeckLU/mdUm9QjMzS1U+ZwT3An8G9oyIbhHxFWAvck8ET0ixNjMzK4B8gqBdRNwbEZ+Xz4iIzyNiArB7eqWZmVkh5HP76GxJ/w3cTsWQ0h2BM4E5aRVmZmaFkU8QnAGcDYym4l0CZcBk4NaU6jIzswLJ58ni9cAfkp9qSbo4Iq6sr8LMzKww6vMNZafU47bMzKxA6jMIPACdmVkjVJ9BEPW4LTMzKxCfEZiZZVzeQZC8RnJb7q9jLWZmVgT5DDHx3WR4ifmSyiR9Y2vtIuLX9V6dmZmlLp8zgiuAIyJiL+D7gG8RNTNrQvIJgg0R8U+AiHgWaJtuSWZmVkj5PFncQdJPq5uOiOvrvywzMyuUfILgZjY/C6g87VtGzcwauXyGmBhd3TJJB9VvOWZmVmg1fnm9pB7AoOTnQ6C0nmsyM7MCyisIkjeRlf/x/wzoDJRGxOLUKjMzs4LI5zmCp4Ep5ELj+xHxNeBjh4CZWdOQz+2j75K7OLwH0D6Z54vEZmZNxHaDICJOBHoCs4FRkt4AdpV0cMq1mZlZAeR1jSAiVgHjgHGSOgCnAtdL6hQRndIs0MzM0pX3XUOSSoH/B3QBWiazd02hJjMzK6Ca3D56N/BzYD6wMZ1yzMys0GoSBCsiYlJqlZiZWVHUJAguk3QL8BjwafnMiJhY71WZmVnB1CQIhgH7krs+UN41FICDwMysEatJEBwUEfvUdAeS+gE3AM2BWyLiqmrafR94INnPrJrux8zMaqcm7yx+KhlnKG+SmgNjgP5AD2DQ1rYhqS3wY+DZmmzfzMzqriZB8HVgrqSFkuZJmi9p3nbWORhYFBGvR8R6YAJwwlba/SdwNbCuBvWYmVk9qEnXUL9abH9vYEml6TLgkMoNJPUBOkbEFEk/r8U+zMysDvIOgoh4s753LqkZcD0wNI+2w4HhAJ06+WFmM7P6UpOuodpYCnSsNF2SzCvXFvgqMF3SYnLdT5OSp5g3ExFjI6I0Ikrbt29fdbGZmdVS2kEwE+gmqaukHYCBwKaH0iJiVUS0i4guEdEFeAYY4LuGzMwKJ9UgiIgNwAjgUWABcF9EvCTpckkD0ty3mZnlp8avqqypiJgKTK0y79Jq2h6Vdj1mZra5tLuGzMysgXMQmJllnIPAzCzjHARmZhnnIDAzyzgHgZlZxjkIzMwyzkFgZpZxDgIzs4xzEJiZZZyDwMws4xwEZmYZ5yAwM8s4B4GZWcY5CMzMMs5BYGaWcQ4CM7OMcxCYmWWcg8DMLOMcBGZmGecgMDPLOAeBmVnGOQjMzDLOQWBmlnEOAjOzjHMQmJllnIPAzCzjHARmZhnnIDAzyzgHgZlZxjkIzMwyzkFgZpZxDgIzs4xzEJiZZVzqQSCpn6SFkhZJGrmV5T+V9LKkeZIek9Q57ZrMzKxCqkEgqTkwBugP9AAGSepRpdkcoDQiegEPANekWZOZmW0u7TOCg4FFEfF6RKwHJgAnVG4QEU9ExNpk8hmgJOWazMyskrSDYG9gSaXpsmRedc4GHt7aAknDJc2SNGvFihX1WKKZWbY1mIvFkoYApcC1W1seEWMjojQiStu3b1/Y4szMmrAWKW9/KdCx0nRJMm8zkvoC/wH8a0R8mnJNZmZWSdpnBDOBbpK6StoBGAhMqtxAUm/gf4ABEbE85XrMzKyKVIMgIjYAI4BHgQXAfRHxkqTLJQ1Iml0LtAHulzRX0qRqNmdmZilIu2uIiJgKTK0y79JKn/umXYOZmVWvwVwsNjOz4nAQmJllnIPAzCzjHARmZhnnIDAzyzgHgZlZxjkIzMwyzkFgZpZxDgIzs4xzEJiZZZyDwMws4xwEZmYZ5yAwM8s4B4GZWcY5CMzMMs5BYGaWcQ4CM7OMcxCYmWWcg8DMLOMcBGZmGecgMDPLOAeBmVnGOQjMzDLOQWBmlnEOAjOzjHMQmJllnIPAzCzjHARmZhnnIDAzyzgHgZlZxjkIzMwyzkFgZpZxDgIzs4xzEJiZZVzqQSCpn6SFkhZJGrmV5a0k3Zssf1ZSl7RrMjOzCqkGgaTmwBigP9ADGCSpR5VmZwMfRMRXgN8AV6dZk5mZbS7tM4KDgUUR8XpErAcmACdUaXMCcHvy+QHgW5KUcl1mZpZQRKS3celkoF9EnJNM/wA4JCJGVGrzYtKmLJl+LWnzXpVtDQeGJ5P7AAtTKzw/7YD3ttsqG3wsKvhYVPCxqNBQjkXniGhfdWaLYlRSGxExFhhb7DrKSZoVEaXFrqMh8LGo4GNRwceiQkM/Fml3DS0FOlaaLknmbbWNpBbAzsDKlOsyM7NE2kEwE+gmqaukHYCBwKQqbSYBZyafTwYejzT7q8zMbDOpdg1FxAZJI4BHgebAbRHxkqTLgVkRMQm4FbhT0iLgfXJh0Rg0mG6qBsDHooKPRQUfiwoN+likerHYzMwaPj9ZbGaWcQ4CM7OMcxCYmWWcg8DMLOMcBFYjklpIOk/SI5LmJT8PSzpfUsti11dIknaWdJWkf0p6X9JKSQuSebsUu75C8rGo0BiPhYMgT43xX25K7gQOBEYBxyU/o4EDgLuKVlVx3Ad8ABwVEbtFxO7A0cm8+4paWeH5WFRodMfCt4/mSdKjwOPA7RHxTjJvT3IPw30rIr5dzPoKRdIrEdG9psuaIkkLI2Kfmi5rinwsKjTGY+Ezgvx1iYiry0MAICLeiYirgc5FrKvQ3pd0iqRN/+1IaibpNHLfeLLkTUm/kLRH+QxJe0j6JbCkiHUVg49FhUZ3LBwE+Wt0/3JTMpDcUCDvSHpF0ivAO8D3aDxPhdeX04Ddgf+V9IGk94HpwG7AqcUsrAh8LCqUH4vpSTdygz8W7hrKk6RdgZHk3p/QIZn9Lrmxkq6KiMx8G5Z0CBDAa8C+wKHAyxExtaiFFZmkI8i9g2N+REwrdj2FJOlC4M8RkaUvRdWS9GVyX446Ap+TGzb/noj4qKiFVcNBUA8kDYuIccWuoxAkXUbujXMtgL+S+8M3HTgGeDQirihedYUl6bmIODj5fA5wAfAg8G1gckRcVcTyCkrSKmANuS8H9wD3V32nSFYkoXg8MIPczRRzgA+Bk4AfRsT0ohVXDQdBPZD0VkR0KnYdhSBpPrm7hlqR6xIqiYiPJO0IPBsRvYpZXyFJmhMRvZPPM4HjImKFpC8Az0REz+JWWDiS5gBfA/qS6xoZAMwG/ghMjIiPi1heQZX/PxIRn0vaCZgaEUdJ6gQ8VP7fTEPSaF5MU2yS5lW3CNijmmVN0YaI+BxYK+m18lPdiPhE0sYi11ZozZIuw2bkvlStAIiINZI2FLe0gouI2AhMA6Ylz5T0BwYB1wFbvBWriWtBrkuoFdAGICLeaqjP2jgI8rcHcCxb3hkj4KnCl1M06yXtFBFryX0DBHLPWQBZC4KdyX3rFRCS9oqItyW1SeZlyWa/b0R8Ru762aTkW3GW3ALMlPQscARwNYCk9uSG2m9w3DWUJ0m3AuMi4u9bWXZPRAwuQlkFJ6lVRHy6lfntgL0iYn4RympQkj98e0TEG8WupVAkdY+IV4pdR0MhaX9gP+DFiPhnsevZHgeBmVnG+TkCM7OMcxCYmWWcg8DMLOMcBJZZkkZJuqjY+5Z0uaS+yeefZPAuGysyB4FZkUXEpRHxt2TyJ4CDwArKQWCZIuk/ksHy/g7sk8y7UNLLyr1kZ8I21t1d0jRJL0m6RdKbktpJ6iLpxUrtLpI0Kvl8rqSZkl6Q9KetfduXNF7SycnQBF8EnpD0hKSzJP22UrtzJf2m3g6GWcJBYJkh6WvkRkg9kNwYMAcli0YCvZPhMc7fxiYuA/4eEfsDfwbyGVZkYkQcFBEHAAuAs6trGBE3AsuAoyPiaHIvMflupadRhwG35bFPsxpxEFiWHEFuhMy1ydAYk5L584C7JQ0BtjU0xJEkb2GLiCnk9/6Fr0p6Mhl/5nRg/3yLjYjV5F6GdLykfYGWfmDP0uAgMIPvAGOAPuSGBqjp0Csb2Pz/pdaVPo8HRiQD0I2usiwftwBDyZ0NZGKEWys8B4FlyQzgREk7SmoLfJfc/wMdI+IJ4Jfkxg9qs431BwNI6g/smsx/F+iQXENoRW4I4nJtgbeT7p3T86jx42QdACLiWXJj2g8mN5KnWb3zoHOWGRHxvKR7gReA5cBMci/YuSsZNE/AjRHxYTWbGA38UdJL5AYafCvZ7meSLgeeA5YClceW+RXwLLAi+Wdbtm0s8IikZcl1AshdKzgwSy8/ssLyWENmtSRpMVCa9gtYJP0F+E1EPJbmfiy73DVk1kBJ2iV5J/QnDgFLk88IzKqQNAz4cZXZ/4iIC4pRj1naHARmZhnnriEzs4xzEJiZZZyDwMws4xwEZmYZ5yAwM8u4/wOKUpfPNgc86gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ev = ExperimentVisual(df)\n",
    "\n",
    "ev.visualize(\n",
    "    plot_kind=\"bars\",\n",
    "    xvar=\"ds_quality\",\n",
    "    yvar=\"mAP_0.5\",\n",
    "    legend_var=\"ds_modelSize\",\n",
    "    title=\"AP - quality\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEXCAYAAABVr8jJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABQNElEQVR4nO29eXxc5XX//z6zaJcsWZJtvNvgHRsbzNawOASKQ8yaBcoSIGkov5Am+SYpgbYhhoQAKdmhaSmBbE1Im9CELQHCUkhJAnYweAO8Y3mVJWsdzX5+fzx3pJEsjUe2RhpJ5/16jWfm3ufeeWY0vp85y3OOqCqGYRiG0R98Qz0BwzAMY/hh4mEYhmH0GxMPwzAMo9+YeBiGYRj9xsTDMAzD6DcmHoZhGEa/MfEwjBGAiPxQRL7qPT5TRN4e6jkZIxsTD8MARORFETkoIoU9tv9QRKIi0iYijSLyrIjMHap5ZoOqvqyqc1LPRWS7iJw7lHMyRh4mHsaoR0SmA2cCClzUy5Cvq2oZMBnYD/xw0CZnGHmKiYdhwEeBP+FE4dq+BqlqCPgZcPyRvIiI+EXkXhE5ICJbReQmEVERCXj7u1kIIrJSRH6a9vy/RWSviDSLyEsisqCP11kmInXe458AU4HHPevpZhF5UkT+vscxb4rIpUfyvozRiYmHYTjx+E/vdr6IjO9tkIiUAVcBrx/h63wCWAEsAZYCH+rn8b8FZgHjgL94882Iql4DvAtcqKplqvp14EfA1akxInICMAl4sp/zMUYxJh7GqEZEzgCmAf+lqquBLcCVPYZ9QUSagM1AGXDdEb7cR4Bvq+pOVW0E7urPwar6kKq2qmoEWAmcICJjjmAejwGzRWSW9/wa4BeqGj2CcxmjFBMPY7RzLfCMqh7wnv+MQ11X96pqpapOUNWLVHVLz5OIyFTPLdQmIm19vNZEYGfa8x3ZTtJzed0tIltEpAXY7u2qyfYcKVQ1DPwCuFpEfMDfAD/p73mM0U1gqCdgGEOFiBTjrAG/iOz1NhcClSJygqq+ke25VPVdnFWSiT3AlLTnU3vsbwdK0p5PSHt8JXAxcC5OOMYABwHJZnq9bPsRTjD+AIRU9Y9ZnMcwOjHLwxjNXAIkgPnAYu82D3gZFwcZaP4L+LSITBaRKuCWHvvXAFeISFBEesZEyoEI0IATmK/143X3ATPTN3hikQS+gVkdxhFg4mGMZq4FHlbVd1V1b+oG3AdclcqCGkD+A3gaeAMX8H60x/4vAcfiLIrbcS60FD/Gubl2ARtw2WHZchfwzyLSJCJf6HHOhcBPez/MMPpGrBmUYQwN3vqSbUBQVeND8PofBW5Q1TMG+7WN4Y9ZHoYxChGREuCTwANDPRdjeGLiYRijDBE5H6jHxUJ+dpjhhtEr5rYyDMMw+o1ZHoZhGEa/GTXrPGpqanT69OlDPQ3DMIxhxerVqw+oam3P7aNGPKZPn86qVauGehqGYRjDChHptRKCua0MwzCMfmPiYRiGYfQbEw/DMAyj34yamEdvxGIx6urqCIfDQz2VvKeoqIjJkycTDAaHeiqGYeQBo1o86urqKC8vZ/r06YhkU5x0dKKqNDQ0UFdXx4wZM4Z6OoZh5AGj2m0VDoeprq424TgMIkJ1dbVZaIZhdDKqxQMw4cgS+5wMw0hn1IuHYRiG0X9GdczDMAxjpJFIKtF4kmg8SSSeoCOaYEJlEYUB/4C+jlkeGVi5ciX33nvvkLz2iy++yIoVK7Ies2/fPlasWMEJJ5zA/PnzueCCCwDYvXs3H/rQhzKdxjCMYUYyqYRjCZpDMfa3htla38baXc38aWsDL2+q59VtDby+8yAb9rSwaX8r4VhywOdglscI4bbbbuO8887jM5/5DABvvvkmABMnTuSXv/zlUE7NMIwjIJlUookkEc+CCEUStEbitEfiRGJJUq3pFQj4fAT9QqHfR2lB98t6Y3skJ/Mzy6MHd955J7Nnz+aMM87g7bffBuC73/0u8+fPZ9GiRVxxxRV9Hrty5UquvfZazjzzTKZNm8ajjz7KzTffzMKFC1m+fDmxWAyA5557jiVLlrBw4UI+9rGPEYm4P+7vfvc75s6dy4knnsijj3Z1KG1vb+djH/sYp5xyCkuWLOE3v/nNIa+9Z88eJk+e3Pl80aJFAGzfvp3jjz8egL/9279l8eLFLF68mNraWm6//XYA/uVf/oWTTz6ZRYsW8eUvf/loPj7DMPpBpwXREaO+NcL2A+2s39XMn7c28PKmA/x5ayNr3j3Iht0t1B3sIBSJU+D3Mba0gLGlhYwtLaS6tJAxxUFKCgIE/IN3STfLI43Vq1fzyCOPsGbNGuLxOCeeeCInnXQSd999N9u2baOwsJCmpqaM59iyZQsvvPACGzZs4PTTT+dXv/oVX//617n00kt58sknWb58Oddddx3PPfccs2fP5qMf/Sjf//73ufHGG/nEJz7B888/z3HHHcfll1/eec4777yTc845h4ceeoimpiZOOeUUzj333G6ve9NNN3H55Zdz3333ce6553L99dczceLEbmMefPBBAHbs2NE5j2eeeYZNmzbx6quvoqpcdNFFvPTSS5x11lkD86EaxihHVYnEk86KiCXpiMZpDTsLoiOeABVSVoTfsyCCfh/FQX9eZzmaeKTx8ssvc+mll1JSUgLARRddBLhf8VdddRWXXHIJl1xyScZzvP/97ycYDLJw4UISiQTLly8HYOHChWzfvp23336bGTNmMHv2bACuvfZa7r//fpYtW8aMGTOYNWsWAFdffTUPPOA6hD7zzDM89thjnfGXcDjMu+++2+11zz//fLZu3crvfvc7fvvb37JkyRLWrVt3yPzC4TAf/vCH+d73vse0adP43ve+xzPPPMOSJUsAaGtrY9OmTSYehtEP0gUiGk8SisRpi8Zp60gJhDcO8ItQEPAR9PuoynOByISJRxY8+eSTvPTSSzz++OPceeedrF27lkCg94+usLAQAJ/PRzAY7Pxi+Hw+4vH4Eb2+qvKrX/2KOXPmdNu+b9++bs/Hjh3LlVdeyZVXXsmKFSt46aWXOOmkk7qNufHGG7nssss6LRdV5dZbb+Xv/u7vjmhuhjFaUO2KQUTjSTqiCVojMUKRBO3ROGinRuAXZz0UBIa3QGTCYh5pnHXWWfz617+mo6OD1tZWHn/8cZLJJDt37uS9730v99xzD83NzbS1tR3xa8yZM4ft27ezefNmAH7yk59w9tlnM3fuXLZv386WLVsA+PnPf955zPnnn8/3vvc9Ui2DX3/99UPO+/zzzxMKhQBobW1ly5YtTJ06tduY+++/n9bWVm655ZZu537ooYc639OuXbvYv3//Eb8/wxjOOAsiQWs4xoG2CDsbQ7y1p4VV2xt5eVM9f9zSwF92HGTdrma2H2inJRRHEKqKCzrjD9WlhVSWFFBaGCDo941I4QCzPLpx4okncvnll3PCCScwbtw4Tj75ZESEq6++mubmZlSVT3/601RWVh7xaxQVFfHwww/z4Q9/mHg8zsknn8yNN95IYWEhDzzwAB/4wAcoKSnhzDPPpLW1FYAvfelLfPazn2XRokUkk0lmzJjBE0880e28q1ev5lOf+hSBQIBkMsnf/u3fcvLJJ7N9+/bOMffeey/BYJDFixcDzgq58cYb2bhxI6effjoAZWVl/PSnP2XcuHFH/B4NI59RVWIJJxLReJKOWIK2cJz2SIL2aIykdo31iVDgdy6myuKCESsER4Kkfs2OdJYuXao9Owlu3LiRefPmDdGMhh/2eRnDiWhnkDpBOJagPZqgtSNOKBYnkaYQPs/FlApU+0aYQDS2R1g8tYoxxUdWEVtEVqvq0p7bzfIwDGPYEusWg4jTHk3Q5q2F6CYQCMGAE4iKouCIE4ihwMTjCHj44Yf5zne+023be97zHu6///4hmpFhjGwi8QThWJJwNE5LOE5rJE4oEifeUyD8PoIBE4jBwMTjCLj++uu5/vrrh3oahjHiiCeShONJwl4corkjRms4RjyppFZDpGIQZYVB/D4TiN4IxxJsO9DO1vo23tnfyqIplQP+GiYehmEMOql1EeGYK7vREo7R0hEjFEt0jgn4fBSYSByWlo4YWz2h2FLfztYDbew62NGZNlxWGKChLUpVScGAvq6Jh2EYOSUaTxKOu6B1S0eMlnCc9nDcy2pSwC2aKwz4qC6wS1JfqCoH2qJsPdDGlv1tbD3Qzpb6dg60ddWuqi0vZGZNKWfNqmVmbSkza8rwiVJbXjjg88n5X0pElgPfAfzAg6p6d4/904CHgFqgEbhaVeu8fVOBB4EpuG/ZBaq6XUR+CJwNNHunuU5V1+T6vRiG0TfJpHoikaQ9EqelI0ZTR4xYoquia9DnFs5VFFtMIhOJpLK7uYOt9c6icELRRmvYLTQWYFJVMfOPqeDY2lJm1pYxs6aUil4yqnJVGDGn4iEifuB+4DygDnhNRB5T1Q1pw+4FfqyqPxKRc4C7gGu8fT8G7lTVZ0WkDEivK/wPqmrlYnG1r372s5/h9/vx+Xz8+7//O6eeeupQT8sYoaRcTpFYko6YC2C3dMRojyQARXHpr4UBV+HVXE6ZiSWS7GgIsfVAW6dYbGto7yyjHvAJ06pLOG1mNcfWOKGYXl1KccHA9ufoL7m2PE4BNqvqVgAReQS4GEgXj/nA57zHLwC/9sbOBwKq+iyAqh75su4RzB//+EeeeOIJ/vKXv1BYWMiBAweIRqNDPS1jhJAewG4Nx7wAdpxkWpZTgd/vynCUBG0R3WEIReNs89xNKYtiZ2OoM2usOOhnRk0p580bz8zaMo6tLWVyVQnBQayWmy25Fo9JwM6053VAz5/EbwCX4VxblwLlIlINzAaaRORRYAbwe+AWVU1F1O4UkduA57zth9hmInIDcANwSKmOkcKePXuoqanprKlVU1MzxDMyhiPJZFoAOxqnORyjpSNOuEcAuzDgo9wC2FnRFIqytb6dLWkWxZ7mcGcgu7I4yMzaUk6aOomZtaUcW1vGhDFFw8adlw/RqS8A94nIdcBLwC4ggZvbmcAS4F3gF8B1wA+AW4G9QAHwAPBF4I6eJ1bVB7z9LF26NKdL6W9/fD0bdrcM6DnnT6zgyxcuyDjmr//6r7njjjuYPXs25557Lpdffjlnn332gM7DGFmkB7CbQ04k2qPxztpp4gWwCwKHNhYyDkVV2d8acdlOaVlPje1dHoBx5YUcW1vGe+eOY2aNsyjGlg7vcie5/mbswgW7U0z2tnWiqrtxlgdeXOODqtokInXAmjSX16+B04AfqOoe7/CIiDyME6BRSVlZGatXr+bll1/mhRde4PLLL+fuu+/muuuuG+qpGUNMwms0FI4laI/EafZiE+kB7AKv8mtlsbmcsiGRVOoOhjpTY7fWt7P1QDttERfI9glMqiph0aQxndbEzJoyyopGngjn+h29BswSkRk40bgCuDJ9gIjUAI2qmsRZFA+lHVspIrWqWg+cA6zyjjlGVfeI+7ZfAhzauGKQOZyFkEv8fj/Lli1j2bJlLFy4kB/96EcmHqOI9DUTHVHXlS61ZiK1sC7VQ8IC2NkTjSfZ3tDuCYQTim0N7UTjTnyDfmF6dSnvOa7GZTzVlDGtuoSi4NAGsgeLnIqHqsZF5FPA07hU3YdUdb2I3AGsUtXHgGXAXSKiOLfVTd6xCRH5AvCcJxKrgf/wTv2fIlKLy1hbA9yYy/eRz7z99tv4fL7OJlJr1qxh2rRpQzwrI1fEEknPmkjSGo7REu4ewBavyJ+tmegf7ZF42kI7JxQ7D4Y6K+yWFrhA9vIFEzqFYnJV8aC2fc03cv7tUtWngKd6bLst7fEvgV5Tbr1Mq0W9bD9ngKc5bGlra+Pv//7vaWpqIhAIcNxxx3V2IDSGL72tmWgJx4jEu1xOqQC21XHqH43t0W7xia317extCXfuryoJMrO2jFNmjOXY2jKOrS1jfEWhufV6YD9NhjknnXQSr7zyylBPwzhCUt3p0ov+pQew09dMFAX8lBUeWVnt0Yiqsrcl7DKevLTYrfVtHAzFOsdMqCji2NpSzps/3sUoasqoKh3YMh4jFRMPw8ghiaR2lg2Peb0lQlF364gliPTob13g91EY8FsAu5/EE0nqDna40h2eWGw70E4o6lKNfQJTx5awZEqVK9vhrcguLbRL4JFin5xhHCGpjnSxhBOGaCJJ2BOGkBe8jiWSpCRAcWXDA35xLie/j5IR2t86l4Rjia5Atud+2tHQTizhVLgg4GNGdSlnz65lZk0ZM2tLmV5dSkFg9MQnEkmlw8u08/mceA40Jh6G0QfJpHMpOXFQonHPYui0HFxxv3Rx8HsB64BfLLPpCFFVWsNx9raE2dcSdvfN7n5vS5j61khXILvQz7E1ZXxg4cTOGk+TKotH5eceibsKxUkUvwg15YXUlJVRXhTISQaYiYcxaomnRCHR3aXUHo3TEU0QiSc7U13BiUTA54TBelofHbFEkvrWCHvTRGFvc5dYpNxNKcYUB5lQUcSc8RUsm1PUWeNpXPnoDWQnkuq+p4kEqlBeFGB6TQmVpQWUFQTw5VhATTyMEUlvLqVQJEEoFqcj4gr6pRoMOQQRV/U14BcLTh8lqkpzR+wQUXBiEaGhLUJ6yYegXxhfUcSEiiLmH1PB+Ioixo9xz8dXFFJiaccAnYs+E9plXdSWl1FWmBvrIhP2FzGGJekupajXw7o9GqcjlqQjEiccTziXkgh4WUsBn4+Az8UcrMHQ0ROJJ9jfEkkTBScSKaFIVYVNUVXirIfjJ1UwwROKCZ5AVJUWWLpxL/RuXZQypiQ4KNZFJkw8hjkiwuc+9zm+8Y1vAHDvvffS1tbGypUrh3ZiR0mnSynuBaJjLjsp5LmUonH3u1XpqseUEoag30elBaKPmqQqB9uj7PPcS/vSRGJvS7hb7SZwgeqUKCyaXNlpSUwYU8S48sJRs/L6aEl915OqBHxCbXkh1V7sojCQP5+hiccwp7CwkEcffZRbb7112FTUTa1tiCWUmJfC6jKUnEspFIuTSHMppdY6pFxKxcEAZYUmDANBOJbo4VLqcjPta4kQTauDJUB1WQHjK4pYMqWy02pIiUSllWQ/IhJJJRSNd37W5YVBZtSUUlkSpHSIrYtMmHgMcwKBADfccAPf+ta3uPPOO4d6OkCXSymaSBJLcyml1jdE4s4EBzoD0gGfj6CXwmorpgeORFJpbI8ekrG0tznMvtYwTWkL5sD1k5gwpojJVSWcNK3KiYMnEuPKi0ZVumsu6bQukkrA76yLmrJCyvLMusiEicdA8dtbYO/agT3nhIXw/rsPO+ymm25i0aJF3HzzzQP7+kdARzTBmp1NROPJTpdS+tqGAlvbMOCEovHuVkOam2lfS7iz0RC4fP+askImVBRx8vSx3WIP4yuKqCgK2N8mB6RbFwpUFLleHmOK89u6yISJxwigoqKCj370o3z3u9+luLh4yOYRjSdZu6sJgLFW4mHASCSVA22RPjKXwp19rVOUFjjrYXp1CafNHNvpVhpfUURteWFedqUbaaQqHYeicVQh4PcxrqKQ6tICyouCI8KCM/EYKLKwEHLJZz/7WU488USuv/76IXn9ZFJ5e18LkViSyhITjv7S5i2K65m5tLc5TH1bhESa9eD3CePKCxlfUcRfHVvTLWtpQkXRiOwdMRxIJJX2SJxY0sUuKoqDHDeujDElBZQWjDxr275lI4SxY8fykY98hB/84Ad87GMfG/TX31rfxoHWKDVlhYP+2vlOLJGkoT3KgdYI9W2Rrvu2CAfaouxvDdMe6b4orrwowISKImaNL+PMWTXOevAEoqas0NKM8wBVJRxza4ZScbtxFV7sojAwIqyLTJh4jCA+//nPc9999w366+462MG7jaFRKRyJpNIUinpiEO0mDPWt7r4pFOu2IA6grDBATVkBNWWFzJ1QzjFjuscebFFcfhL3MgNT1sWY4iCTq8qpKAmOSOsiE/YNHea0tbV1Ph4/fjyhUGhQX7+xLcLbe1sYWzryykSoKi3huGchdFkM9a3Rzm0N7dFuLiWAwoCPmrJCassLmV5d6kTCy6apLXP3xQXDI6NmtJOyLkIxF1cKpmIXZYWUFwVGdfzIxMM4YtoicdbuamZMccGwdKOEonHPOnBikHIpdVoN7dHOlqMpAj6h2rMY5h9T0SkS7r6g02Ux0oR0NBFPJGmPJoh71kVlibMuxpQEKRll1kUmTDyMIyIST7CuromioD8vfbvReLLLYmiLUN8W7XQjpQSivUfxPQGqSguoLStkRm0Zp8xwYpASiNqyQsaU2BqUkUY360IhGPAxYUwh1aVu3cVoti4yMerFQ1Xtl0QWqHa5ZhJJZeOeVhIKFUPQTCe18C09rtAZgG518Yfmjtghx1UUBagpL2TCmCKOnzSm02KoKXOCMba0YFT3pB5NpKoapKyLqpIgU8aWU1Fs1kW2jGrxKCoqoqGhgerqavuyZEBVaWhooKioCFVl0/5WmkJRqksHPkCuqjR1xLrcR23dReJAW4TG9ig9wgwUB/2dYnBsbWm3GIOrDVQwbFbuGgOPqmuO1BFznRsLg2ZdHC05Fw8RWQ58B/ADD6rq3T32TwMeAmqBRuBqVa3z9k0FHgSm4KpYXKCq20VkBvAIUA2sBq5R1e5V2rJg8uTJ1NXVUV9ff8Tvb7RQVFTE5MmT2dkYYndTmJojWASoqrRHE2nCkC4KXSIR76EMQb90CsGiSZVdFkN5QWcA2tqJGj1Jty5EoLK4gGnVJVQUBym2KgdHjaS7Iwb85CJ+4B3gPKAOeA34G1XdkDbmv4EnVPVHInIOcL2qXuPtexG4U1WfFZEyIKmqIRH5L+BRVX1ERP4NeENVv59pLkuXLtVVq1bl4m2OGupbw6zd1czYkr7XGYSicd7Z19ZryuqBtqj75ZeGT6A6FVco64ox1JR3WQ1WMsPIhm7WBS7rbVx5EdVlBZQVBswleYSIyGpVXdpze65/rp0CbFbVrd4kHgEuBjakjZkPfM57/ALwa2/sfCCgqs8CqGqbt12Ac4ArvWN+BKwEMoqHcXS0hGOs391CZYbMqpaOGDf/6k12NXV0bqssCVJTVsjkqhKWTK3qXNtQ6wlEVcnwzNQy8oOe1kVVSQHTq0soN+si5+RaPCYBO9Oe1wGn9hjzBnAZzrV1KVAuItXAbKBJRB4FZgC/B24BqoAmVY2nnXNSzt6BQTiWYG1dMyXBvn3DkXiCrz65gf2tYW4+fw6zxpVTXVZgvmSj36gqSXX3Cqi63iKqrn9LMgnRRAIFigJ+JlYWMbbUrIvBJh8cxV8A7hOR64CXgF1AAje3M4ElwLvAL4DrgN9ke2IRuQG4AWDq1KkDOedRQzyRZN2uZgT6XNiWVOWbz77DW3tbuXn5XM44bnj0FTGyo7eLePrFXDVtH137ulp1OaTbA9fhEaTbKBG32yc+/D7wi+D3+QgGBJ+kGn75GFMUoKK4wBZbDiG5Fo9duGB3isnetk5UdTfO8sCLa3xQVZtEpA5Yk+by+jVwGi64XikiAc/6OOScaed+AHgAXMxjAN/XqCCZVN7e20p7JM7YDJlVD/1hG69saeDj75lhwjFI9HUR73yedjHXtLFpTdsRr5lK6j+GdP6rINKZni24Yox+n+AXwed1bExdzFP7fD5X3yl9u4jgE9fMS4Ruz7u2dT33eWOM/CfX4vEaMMvLjtoFXEFXrAIAEakBGlU1CdyKE4fUsZUiUquq9bg4xypVVRF5AfgQLuPqWvphjRjZs72hnf1tYWpKi/oc89gbu/jNG7u5cNExXLx44iDObniSVCWRVGKJJImkEk8oSbou0tDzYt59W2q7L+1i7i7c7nEg7SIu4u79vu4X866LNYdc3H3d9tvF3OibnIqHqsZF5FPA07hU3YdUdb2I3IETgseAZcBdIqI4t9VN3rEJEfkC8JwXJF8N/Id36i8Cj4jIV4HXgR/k8n2MRvY2dbC9oT3jWo7/23yAB1/exukzq/n4GTNH/UUmkVTiySTxhBJPKolkspsQuHa6UBwMUFIYoDjoozjopyDgdwLgXajTL+49L+Yp4TCMoSanqbr5hKXqZk9zKMZf3j1IZXGwzwDkxj0t/POv1zGjppQ7Lz1+RC/AU3ViEE94VkMySVK1UxBS90G/j+ICP0VBHyXBAEVBHwUBPwG/UOD3dfrrDWM4MVSpusYwIxSN8+aupoyZK7sOdvCVJzdQXVbAl1bMH9bCkUjqIRaDot1cSD6BwoCfoqCfkgJ3Xxh0LXUDftd7PejzmUVgjCpMPIxOonGXWRXw+SgK9i4ITaEoKx9fjwArL1zAmOLg4E4yS1RTopCyGpIk0qxs8bJ8Aj6hOBigrDBAcYHfcyOliYJnMYx2l5xh9MTEwwBcZtXGPS1E4kkqi3svPRKOJfjKkxtoDEX52iULmVg5NP3S+wo6d08FhUK/j6ICP+WBQKfFEPT7CHiiEPT7bIGiYRwhJh4GqsqW+jYa2/tuI5tIKvc+8zab9rXxjxfMY86E8pzMpZsoZAg6lxQcGnQu6CYMZi0YRi4x8TDY1dRB3cFQn5lVqsoDL2/lz9sa+buzZnLazOp+v0Z/g86lRX4LOhtGHmPiMcppaIvwzt7WjG1k/+f1XTy1dg+XLpnEikWHX8vRFIqS0EODzkUBP4VBf2dswYLOhjF8MfEYxbRF4qw7TBvZl96p5+FXtnPmrBqu+6vphz1nQ3uEceWFjK8osqCzYYxgTDxGKZF4grWHaSO7dlcz3/r9OyyYWMFn3zf7sO1Xm0JRqkoKmDOhwgLRhjHCMefxKCSRVDbsbiGpLvDcGzsbQ9z51AYmjCniny6Yd9g+5S0dMYoL/Mw7xoTDMEYDJh6jDFXlnX2tNHfEqCjqfY1GY7tbyxH0+1h54QLK+xiXIhSN4/PB8ZPGHFZkDMMYGdj/9FHGuw0h9jSHGVvS+1qOjmiCO55YT0s4xpdXLGB8Rd9FEcGt/YjGkyyaXNnnwkLDMEYeJh6jiP0tYTYfaGNsSUGvwetEUrnn6bfYdqCdL54/l+PGlWU8XyyRpC0SZ9GUSushbhijDBOPUUKqjWxVH5lVqsq/vriZ1TsO8v+dfRxLp4/NeL5EUmkKRVk4qSJvS5QYhpE7TDxGAeFYgjfrXLHDvtrC/tfqOp7ZsI8PnzSZ5cdPyHi+pCqNoQhzJ1RQU57ZrWUYxsjExGOEE/PayPqQPmMSz7+1n5/+aQfL5tRyzWnTMp5PVWlojzKzppSJVUNT28owjKHHxGMEk0wq73htZPvKmHpjZxPffX4TiyaP4dPnzDrsQr6G9iiTq4qYVl2aiykbhjFMMPEYwWxvaGdfa7jP/uPbD7Tztd9uZHJlMf/4/nl9urRSNLZHqC0v5LjaclstbhijHBOPEcrh2sgeaIuw8vH1FAX9fPnCBYfNlmoJxygvCjB3QrnVnzIMw8RjJNIUirJxbytVxQW9lhQJRePc/vh6QtEEKy+cT215333KwdXACviEBZPGWEVbwzAAE48RRygaZ21dM+VFvbeRjSWS3PXbt9h5sINb3z+XGTWZ13J0RBMk1C0CHM7tZg3DGFhMPEYQ0XiStXXNBP2+Xi/0qsp9z29mzc4mPvXe41gyteqw5wvF4pwwuZLiAhMOwzC6yLl4iMhyEXlbRDaLyC297J8mIs+JyJsi8qKITE7blxCRNd7tsbTtPxSRbWn7Fuf6feQ7Ca+NbDSR7DN+8bNX3+X5t/dz5SlTOXfe+IzniyWStIRjnDC58rC1rQzDGH3ktKaEiPiB+4HzgDrgNRF5TFU3pA27F/ixqv5IRM4B7gKu8fZ1qOriPk7/D6r6yxxNfVjh2si2Zmwj+8yGvTzy2k7OmzeeK06ekvF8iaRysCPK8ceMoaq09xpYhmGMbnJteZwCbFbVraoaBR4BLu4xZj7wvPf4hV72G4fBtZHtoLqPC/3qHQe5/4XNnDi1kk8uOzZjmm1SlYOhCLPHlTN+jK0eN4xhTTwKHU2QTAz4qXMtHpOAnWnP67xt6bwBXOY9vhQoF5FUk+wiEVklIn8SkUt6HHen5+r6loj0+nNbRG7wjl9VX19/dO8kT+lsI1vSexvZLfVt3PO7t5heXcoXl889bLZUYyjClLGlTLbV44YxvEgmIdIGbfth/1uw/f9g2//Czj9BpHXAXy4fSqF+AbhPRK4DXgJ2ASmZnKaqu0RkJvC8iKxV1S3ArcBeoAB4APgicEfPE6vqA95+li5dqrl+I4NNWyTO2gxtZPe3hLn98fWUFga4bcX8Phs/pWhojzChopiZNaW2CNAw8p14BKLtThjaD0C4CTTp9vkLIFgEBTXQ3pCTl8+1eOwC0h3sk71tnajqbjzLQ0TKgA+qapO3b5d3v1VEXgSWAFtUdY93eEREHsYJ0KgiVeywuI82sm3hOCsfX080nuSeDx5PdR+xkBSpFrKzx9siQMPIO5IJiIWcZRFqhI5GiIcBAREnFMWVIIOXQJtr8XgNmCUiM3CicQVwZfoAEakBGlU1ibMoHvK2VwEhVY14Y94DfN3bd4yq7hH38/gSYF2O30deEU8k2bC7GVUo6SWzKpZIcudTG9jTHOb2ixYctg6VtZA1jDwjFnZWRbgZQgcg0gIKoBAohEARFGZeo5Vr+i0eInIGLhC+TlWfyTRWVeMi8ingacAPPKSq60XkDmCVqj4GLAPuEhHFua1u8g6fB/y7iCRxsZm707K0/lNEagEB1gA39vd9DFdUlU3722gNx3utWZVU5du/38S63S18/rzZLJpcmfF81kLWMIaYRBxi7U4sQg3OskhE3T6fH4LFUDzWWRh5xGHFQ0ReVdVTvMefwF3c/wf4soicqKp3ZzpeVZ8Cnuqx7ba0x78EDkm5VdVXgIV9nPOcw817pJJqI1vTR2bVj/+4g5c21fPR06exbM64jOdKtZA9cVqVtZA1jMFAFWIdzgXV0eTEItIKqHM5BQqgoAR8FUM908OSjeWRvkLsBuA8Va0XkXuBPwEZxcMYOFJtZGtKe8+semrtHn71lzqWL5jAh06c3MsZuki1kD1xWpW1kDWMXJGIeUHtNud+Ch0EjQMCfj8EiqEk/6yKbMjmquHz4g8+QFS1HkBV20UkntPZGZ00d3S1ke2t2OGr2xr495e2cPL0Km48O/NajlQL2UWTx1gLWcMYKFSdRRENuYB2qMEJRyqoHSiEonLnihoBZCMeY4DVuPiCpgWry7xtRo4JxxKs3dV3G9l39rXy9affZmZtGTefPzdj0NtayBrGABGPulhFuMUJRUcTqLfKwB90Qe3SmiGdYi45rHio6vQ+diVxi/qMHBJLJFlX14wfX69xib3NYb7yxAbGFAe5bcX8jLELayFrGEdIMukFtT2ror0B4h04qwIvqD1mUFNlh5ojdnaragjYNoBzMXqQaiMbiiWoKjk0QN7SEWPl4+uJJ5WvXbSg1zHpWAtZw8iSWNi5oMItLlYRbu5agNeZKpvH/480Cc07Ye962LcOJp004C9xVJFSEXlCVVcM1GSM7mxraGd/a6TXYoeReIKvPrWR/a1hvnLx8UypKsl4Lmshaxh9kEy42ES0/dAFeD6fE4riqvwOakdDUL8R9q2HfRtg//qukiQFZdC+H0oyt2DoL0ebZvOJAZmFcQh7mjrY0Ucb2aQq33r2Hd7a08LNy+eyYOKYjOeyFrKG4aHqhCEactZE+wGINNPpfvIXOBfUEC/Ay4gqtOxyIrFvnROKxm1dllHVdJhxFoxbAOMXQLAEyjK3YDgS+iUeIjLWzV0bvfs9mY8wjgTXRrbvzKqH/rCN/9vSwMffM4MzjssckLMWssaoJrUAL9Lm1X866NJnEW8BXhGUVOe3VREPQ/3bTij2bXDWRbjJ7QuWwLj5sOQMGD/fPS4s7378UNW2EpGpuLIg7wOa3CapwJVRv0VVt+dkZqOUVBvZiqJgrxf7x97YxW/e2M2Fi47h4sUTM54r1UJ28ZSx1kLWGPmkFuBF26HjoJcq20ZnUmiwEApKwZfH65pUoW2f535a76yKA5u7srjGTIGpp8L4451VUTltyFJ/s/kUfwF8G7hK1b0Dr8nTh3H9OU7L2exGGYdrI/vKlgM8+PI2Tp9ZzcfPmJkxdpFqIXvStCprIWuMTFKpspE2r6xH+gK8wPBIlY1H4MAmJxJ7PbEIeZZCoAhq58IJf+OsivHzoahySKebTjbiUaOqv0jf4InIIyLyldxMa/SRaiMbSyQZU3xo1tTGPS1845l3mD2+nM//9eyMazniXgvZxVOshawxgoiFnSURanAuqFjIbRcvqD0cFuC113dZFfvWO+FIxty+8mNg4hJnUYw/HsbOyGsrKZuZrRaRfwV+RFdjpynAtcDruZrYaCLVRvZgKNprgHzXwQ6+8uQGqssK+NKK+RldUImk0mgtZI2RQEos2g+4W2pdhT/ggtoFeW5VJOPO5bRvneeC2uBcUuAC87Vz4PgPwoQFLlZRUp35fHlGNuLxUeDjwO10dQGsAx4HfpCjeY0q6g66NrI1vQhHUyjKysfXI8DKCxdkLCei1kLWGM6kWxZt9U4sFAgEXWA4n9dVgEvz3b++K6hd/1ZXddzScc6iWPhhd199nFuFPozJZoV5FPi+d+sTEblVVe8aqImNFg60htm0v63XNrLhWIKvPLmBxlCUOy85nomVmVeFN1gLWWM4MZzFIhl36bH71ndlQbXudvt8AaiZDfMvdkIxbj6UZa5wPRwZSIfahwETj37QGo6xfk8rlcXBQ2IYiaRy7zNvs2lfG7deMI+5EzKXaLYWskbeM5zFItzs3E6poPb+jd5CQlyvjfHHw4KL3dqKmlluFfoIZyDFw65Y/cAVO2ymKOA7pNihqvIfL2/lz9sa+buzZnL6zMy+0KYOayFr5CG9iUVnzCKPxUKTcHB798B2sxfuFZ9zOc25wAtsz4eyCfm9TiRHDKR46ACea0QTTyRZv7sZ+mgj+z+v7+LJtXu4dMkkVizKvJajNRyjOGAtZI08YLiKRaTVWRKdays2uhRggKIxzqqYs9zd185xmV2GWR6Djaryzr5W2iJxxpYcatq+vKmeh1/ZzhnH1XDdX03PeK5QNA4Cx0+2FrLGENApFo3Qtn94iIUqNL/bVdpj33o4uIPOTn5jZ8Bx7/MW4c2Hikmj0qrIhqzFQ0RqVPVAhiH/PQDzGfHsaGhnb0uY2rJDf72s29XMN599hwUTK/h/587utTRJCmshaww6GcWiOD/FIhaC/W91BbX3b4BIi9tXWO6C2cee41xQtfNcC1gjK7IpT3Ih8BAQF5EE8BGvv3g3VPVrOZjfiGJ/S5itB3ovdrizMcRXn9rAhDFF/NMF8zJaEtZC1hgUYmGv0mxDD7Hw56dloeoynlJB7X3roXFrj4KBZ7qg9oQFrtTHKOq/MdBkc+W5EzhTVd8SkVNxda7OzvYFRGQ58B3ADzyoqnf32D8NJ061QCNwtarWefsSwFpv6LuqepG3fQauNEo1rsvhNV5Kcd6SaiNb2Uuxw8Z2t5Yj6Pex8sIFGVeFWwtZI2fEI12lPtrrvRXceSwWnQUD0wLbnQUDS2HcPFhyjZcuO+/QgoGjgXjEKwQ58GQjHnFVfQtAVf8sIln/BbwaWPcD5+EWFr4mIo+p6oa0YfcCP1bVH4nIObh032u8fR2quriXU98DfEtVHxGRf8MtYsy4DmUo6YgmWFvXexvZjmiCO55YT3NHjLsvW8T4ir6DcdZC1hhQOsWi0fV7iIa8suRezCLfVnB3HIQ9b8CeN/soGHialwE1tAUDh5R4xIl+0vtcCitcfayCgRf+bMRjnIh8rq/nqvrNDMeeAmxW1a0AIvIIcDGQLh7zgdT5XgB+nWky4hYxnANc6W36EbCSPBWPWCLJ+l3N+OXQNrKJpHLP02+x7UA7//yB+Rw3ru8eAtZC1jhqDicWZXkmFqFGTyzegD1rXPosuGyncfPytmDgoNKbWFTNhKIKZ2nlcBV7NuLxH0B5H88Pl547ia56WOCsj1N7jHkDuAzn2roUKBeRalVtAIpEZBUQB+5W1V/jXFVNqhpPO+ckekFEbgBuAJg6dephpjrwJJPKW3tbem0jq6p8/8XNrN5xkJuWHcfJ08dmPJe1kDX6zbATiwYnFLvXuPumHW57sBgmLIJZ58PExW4RXh4XDMwpvYlF5QzXPz3HYtGTbMqT3N7XPhE5eQDm8AXgPhG5DngJ2AV4nwzTVHWXiMwEnheRtUBztidW1QeABwCWLl066OtQth5o50BrtNc2sv+1uo6nN+zjwydNZvnxEzKep7E9Qk15gbWQNTKTLhaheoi0579Y7F7jrIo9b0DTu257sMSJxZz3wzGLoeY4E4s8EIue9PsvIiLzgb/xbk3A0gzDd+Eq8KaY7G3rRFV34ywPRKQM+KCqNnn7dnn3W0XkRWAJ8CugUkQCnvVxyDnzgT1NHexs7D2z6vm39vPTP+1g2ZxarjltWsbzpFrIzptQYavHje4MN7FoP5BmWazpWrUdLIVjFrpV2xMXuxXcJhbueR6JRU+y+guJyHS6BCMGTAOWZtFF8DVglpcdtQu4gq5YRercNUCjqiaBW3GZV4hIFRBS1Yg35j3A11VVReQF4EO4jKtrgd9k8z4Gi6ZQlLf2tlDVS7HDN3Y28d3nN7Fo0hg+fc6sjJaEtZA1upESi46Dzg0VaXcL2FLZUHknFvXd3VDdxGIRzFvhLIvq40ZncBuGlVj0JJt1Hn8EKnAX6g+q6iYR2ZZN+1lVjYvIp4Cncam6D6nqehG5A1ilqo8By4C7RERxbqubvMPnAf8uIknAh4t5pALtX8Q1o/oqrqdI3pSGb4/EebOuifKiQ4sdbj/Qztd+u5FJlcXcesG8QzKv0rEWssYhYhFtpzN1NlCcf2LRtr8ruL17DbR4DoGCUphwAsy70FkWY481sRiGYtGTbCyPfbiA9HjcWoxN9KOOlao+BTzVY9ttaY9/Cfyyl+NeARb2cc6tuEyuvCISd8UOCwP+Qy74B9oi3P7EeoqCflZeuICyDIv7rIXsKCUe8RblNXYXC5/fBY3zraVq2/4uodizBlq8kuQFZXDMCTD/Ek8sZppYjACx6Ek2AfNLRGQMLi6xUkRm4WIOp6jqqzmf4TDBtZFtJd5LG9lQNM7tj6+nPZLgng8upLa873LN8USS5nCUJVOqrIXsSCSZcA2CElG3eKtTMA4MD7FICcXuNV39KwrLXYB7waXODWVikSYW5SNGLHqSVcxDVZuBh4GHRWQc8BHgmyIyVVUHPwc2z1BVNu9vpamXNrKxRJK7fvsWOw92cNuK+cyo6Xsth7WQHeYkk64fdUoYElGXHhtrd/fxcFdnuVQdUcEFhwNFeSgW+5xIpASjdY/bXljuLIvjL+uyLEZrmY9RJBY96U9hxKXAPwLTgdQnUpWDOQ076g52sLspTHXpoWs57nthM2t2NvGZ983ixKl9f1zqrR63FrJ5iqoThHRxiHW4C0e03T2Oh+leXFqdMPgD4As6378vc1OvIaV1T/dsqNa9bnthhROLhR/yLIsZJhajUCx60p98uP8E/gFXayqZm+kMPw60hnlnXyvVpYdmVv3s1Xd5/q39XHnKVM6dNz7jeRpCEaZaC9mhIxHvcicl413upFjIu3V4BfbEuyXdBdQfdAIRLILCvq3KvKR1T3c3VNs+t72wwonEwo84y6JquomFicUh9Ec86r3sKMMj1Ua2qqTgkMyqZzbs5ZHXdnLevPFccfKUPs7gsBayOSaZ7B5nSEQ9q8FzJ8VCTjAAJwzqUmB9nsXgD0Bx1fDu66DqLIn0AHdKLIrGOMti0eUmFiYWWdMf8fiyiDwIPAdEUhtV9dEBn9UwIBxL8GZdM8UB/yEpt6t3HOT+FzazZEoln1x2bEZBsBayR0nKnZQuDrGOLqshHoZExI1Ldyn5/U4YfAF3URhpAV7VQy2L9v1uX9EYZ1mccIW7r5pmYmFi0W/6Ix7XA3Nx8Y6U20qBUSce6W1ke6bSbqlv457fvcW06lJuef/cjIv7rIVsFqS7kxIxJwYpYUgFoUkXBnVC4Au4//jD0Z10JKi6dRXpMYv2erevqNJZFMdc6e4rpw1vK+poyCQWBWUQsESVbOmPeJysqnNyNpNhQqY2svtbw9zx+AZKCwN8ecV8Sgr6/nithSxpaasx7xZxgtAZa+joKrndzZ3kuZICBa7z22i8EHaKxZo0sfAafRZXOYti4mJ3Xzl1dH5G0IdYTIfiShOLo6Q/4vGKiMzv0Ytj1LG9jzaybeE4Kx/fQCSe4J4PLqK6l2KIKUZFC1nVHnGGdHeSl53Ua9pqmjupeMzodaf0RBWa67qv4A71IhYTF8MYEwsTi9zTH/E4DVgjIttwMQ8BVFUX5WRmeci+5jBb69sPqZIbSyS586kN7Gnq4PaLFmQsmz7iW8h2HIT6dyDSyqHupFTaasBZDPmctjrUqLpaUOluqFCD21c8tsuqmLjYa6dqYgGYWAwi/bl6Lc/ZLIYBzR0xNu5tYWxJ9zaySVW+/ftNrNvdwufPm82iyZV9nmNEt5CNR6FxCzTtdO1KS8aO3gvakaAKze92FRHcvQY6Gt2+kurubqgxk0fnZ5tMeG1VIyYWeUDW4qGqO3I5kXwm1Ua2tCBwSAD8J3/cwUub6vnoadNYNmdcn+cYsS1kVV1gdt8GF58orRmdF7aeaLJrEWEsBNG0x6l4TipNuLnOWRYdB92xJTUw6cQuwaiYNPo+00TUCUU8SmcpPV/AiUTRJNcpz8RiSBmBfpOBRVVZv7sZv+/QNrK/XbeHX/6ljuULJvChkyZnPMeIbCEb64D6t6F1n4tPBPqO8+Q9ql0B+9TFvfPC3+N5X2LQOTaVBZYF4nOWxaSTuiyL0SQWmvREIpK21ga3Gr+kxolFsNiVnB/O368RiInHYUiq66vRs2bVq9sa+Lf/3cLJ06u48ezMazlGXAvZZNJVUD3wlgtwl/dtceWURKz3X/K9XcyzufhrloUTAkVdF7RgiYvflIyF4OSubcFitz39eWps+vNA0egRilRGXTziGRMKEoCicigb71a2B4tcuXm/XZryHfsLHQHv7Gvl60+/zcyaMm4+f27GNRojroVspBX2b4Rwk8vy6U/Ht2QC4h3Z/5LvTQzSLYFkLLvX9QUPvZgXlkHZuLQLepoYBEugoMfz1PGB4pG3oHCgSWXaxcNOMFJp1oFiz+1U6SyLYPHoEs8RholHP9nbHOYrT2xgTHGQ21bMz5hqO6JayCbi0LQDGra4C2tp7aFjVGH9/0Ddq71f/PvjyuntYl5cefhf8r2Jga0Szh3JuItLxMPeKn51f7+CMiif6FazB4vdzf4OIwoTj37Q0hFj5ePriSeVr120IGPZ9BHVQjbU6ALi8Q7nnuntl3ekFV68G3b8n1vBXFzpgueH+yXfmxj4C+3XaL6Rvm4n7mXqq7qAddEYlwFWUNbldvIN8++8cVhMPLIkEk/w1ac2sr81zFcuPp4pVSV9jh0xLWS7pd+W9d1v4sA78OyXXdbVX/09LLjMLv7DmWTCi014FYZTbqeCUmdxFo3pEnwLYo9aTDyyIKnKt559h417Wvji8rksmDimz7EjooVsevotyb7Tb1XhrSfgle9CURVc+B0Yv2DQp2scBal6YYlYmtsp4FJhy45x6yhSbieL9RhpmHhkwS9e28n/bWngY++ZzhnH9d3tbUS0kE2l37btdxeQvn5ZxjrgD9+CTc/A5JPhnH9ygVAjP0mlIscjvQSxx3pB7JKulFizHI3DkHPxEJHlwHcAP/Cgqt7dY/804CGgFmgErlbVurT9FcAG4Neq+ilv24vAMUCHN+yvVXV/Lub/w1e28/T6faxYdAyXLJ7U57hh30K2Z/ptWS8B8RRN7zo31cHtcNL1sORq+1WaT6SaWcUj3YPYheUuiF1c2ZVubEFs4wjJqXiIiB+4HzgPqANeE5HHehRXvBf4sar+SETOAe4Crknb/xXgpV5Of5WqrsrR1AFnSTy1dg8nTa3kb8+Y2Weq7bBvIduf9NstL8BLXwd/AVzwLzB56aBN0+hBZ0psxKUtq7cS21/gLIkxU5xgBIrczYLYxgCSa8vjFGCzqm4FEJFHgItxlkSK+cDnvMcvAL9O7RCRk4DxwO+AQb9KBfw+fvyxU/jD5vqMazmGbQvZbNJvO8fG4M//But+5eIa7/uyWyfRE1WINHc1XxIAn3ODSKqFq3R/3tu9VdPtTmcQO+KsxFR5+mCpW2DXmRJbYiU7jEEh1+IxCdiZ9rwOOLXHmDeAy3CurUuBchGpBg4C3wCuBs7t5dwPi0gC+BXwVdXUz66BpSjoz5gxNWxbyGaTfpuibT/8fiXs3wDHfwhO/bve3R3JhKv8WjHZZWdp0m3TpKt7lfTuNentizuRSXrj0K5xJLt+SXfi+ek7H/fcLWnHpI1LDRUfw0LQUtZEehDbF3ACUTbRrchOuZ3MXWgMEfkQMP8CcJ+IXIdzT+0CEsAngadUta6Xi/JVqrpLRMpx4nEN8OOeg0TkBuAGgKlTpw74xIdlC9l4FBo2u2J8mdJvU+x8FZ7/qrvQn7sSZi7r47wRCLdA7dyBaz6k6gmNd49meJwmNoccky5i+ShoeK/lnStYAsXVXqVYb1W7BbGNPCPX4rELmJL2fLK3rRNV3Y2zPBCRMuCDqtokIqcDZ4rIJ4EyoEBE2lT1FlXd5R3bKiI/w7nHDhEPVX0AeABg6dKlA2qZDLsWsqrOgti/Mbvqt8kE/OXH7jZ2Bpx7O1RO6X1sNOQsmEknHl6M+oMISJ79sh4wQVNPqLztqVXxVtfJGCbk+lv6GjBLRGbgROMK4Mr0ASJSAzSqahK4FZd5hapelTbmOmCpqt4iIgGgUlUPiEgQWAH8PsfvoxvDroVsNOQW8rXtd9Vv/YfxiXc0wQt3Qt1rMOt8OPP/OTdJr2ObnUtlyqkuODvSyUdBM4whIKfioapxEfkU8DQuVfchVV0vIncAq1T1MWAZcJeIKM5tddNhTlsIPO0Jhx8nHP+Rq/fQk2HVQjaZhJY6t27DH8ycfpti7zp4biWEm+GsL8CcD/S9QDDU4LKzJiy0lcaGMcqQHMWZ846lS5fqqlX9z+xNJJWXN9VTXVpILJGkuSPGidOq8r8TYLjFuagiLc53frjqt6qw7pfwp39z2Tvn3Q41s3ofmwqMV06FmtkWtDWMEYyIrFbVQ7JdzbmaJcOmhewh6bdZxCCi7fC/X4dt/wvT3gPLbunbBZWIOlfVQAbGDcMYdph4ZMmwaCGbnn5bWp1damnjVnj2Nre6/NQbYdHlfQtCqsT6QAfGDcMYdph4ZEHAJ0yqKsnfFrLxiLM0mnc6iyHbC/s7v4OXv+VSdld8G45Z1PfYVGB86mmjIzBuGEZGTDwOg98nzDumgrH5WK/qkPTb2uzcSPGIq4T71pMwcQmc88+uj3ZfrxFqcOUujllkgXHDMAATj6yoLsvDC2Y05LKo2ve7gPjh0m9TtOyCZ1dCwyZYfDUsva7vYHoqMD5mCtTOscC4YRidmHgMNw5Jv+2lvlRfbP8DvHiXi4Usvwumnt73WAuMG4aRAROP4UQq/TbcDCWHqX6bTjIOrz4Ibz7iLIhzV0L5MX2Pj3o9xycuyW5tiGEYow4Tj+FAIu56ZzRucWUs+nNBDzXA72+HvW/C/Ivh9Jsyu7g6mp17auooWTFuGMYRYeKR74QaYd96F+QurelfZdfdr8Nzdzgr4r3/BLPO63usBcYNw+gHJh75Smf6bZ0rwV1Ylv2xmoQ1P4dVP4Axk2HFt6Bqet/jkwlob3CFDy0wbhhGFph45Buq0LbPS7/Vw1e/7Um4BV78Grz7Jzj2HFefKljS9/hEFEJNMM4C44ZhZI+JRz5xpOm3Kerfcr3FQw3wns+6GEcmMUgFxiedaIFxwzD6hYlHPnA06bfgLJSNj8Er97mugBd9D8bNy3xMuBnwWWDcMIwjwsRjqAm3wP71EG7tX/ptilgIXv4mbP6966nx3n907Ur7wgLjhmEMACYeQ0Uq/fbgVheTOBK30cEd8PsvQ9O7sPTjsOSqzNlYFhg3DGOAMPEYCtLTb0uyrH7bk83PwUv/4jr8XfAvMOmkzONTgfHauVBlgXHDMI4OE4/BJB6BA5tc+fP+pt+mSEThj/8KG34N44+Hc7/sCiJmwgLjhmEMMCYeg8HRpt+maN3rVovXb4RFH4FTbjh8jCQVGJ9yChRVHNH0DcMwemLikWui7V76bf2Rpd+mePfP8MKdLm5x3h0w46zM4zsD42NgwiII5nETK8Mwhh0mHrkimXTNmQ684wSjv+m3nedJwOofwus/gepj4dzb3arxwx0TaoCKyW7xnwXGDcMYYEw8ckG4GfZvgHAblFT2P/02RcdBeO4rsPsvMOcCeM9nDp9a2xkYnwNV0ywwbhhGTjiCNJ/+ISLLReRtEdksIrf0sn+aiDwnIm+KyIsiMrnH/goRqROR+9K2nSQia71zflckT66QiRjUb4Kdf3a//stqjlw49r4Jv/oE7FsHZ38Rzr758MIRDbl1I5NOhLHTTTgMw8gZORUPEfED9wPvB+YDfyMi83sMuxf4saouAu4A7uqx/yvASz22fR/4BDDLuy0f4Kn3n/YG2PFHaNrh0m8LSo/sPKrw5i/g8c86sbjkX2HO+w9/XLjZCdaUUy2jyjCMnJNry+MUYLOqblXVKPAIcHGPMfOB573HL6TvF5GTgPHAM2nbjgEqVPVPqqrAj4FLcvYODkc8AnvXQd0qCASh9AjXbQBE2+DZ2+BP34dpZ8Bl/w7Vx2U+RtUJV7DEMqoMwxg0ch3zmATsTHteB5zaY8wbwGXAd4BLgXIRqQYOAt8ArgbO7XHOuh7nnNTbi4vIDcANAFOnTj3iN9ErPdNvy2qPzk10YJNbLd66D067CRZ+6PDnSw+M184Bv4WwDMMYHHIe88iCLwBni8jrwNnALiABfBJ4SlXrMh2cCVV9QFWXqurS2toBdOVE210Qe88bzj1VUnV0wvHWU/CbmyAehQu/DYs+fPjzJaJupXrNHBg/34TDMIxBJddXnF3AlLTnk71tnajqbpzlgYiUAR9U1SYROR04U0Q+CZQBBSLShrNQJmc6Z85IJlxzpqNNv00RD8MfvgPv/NYFuc/5EhRXHf64WMgFx63HuGEYQ0SuxeM1YJaIzMBd4K8ArkwfICI1QKOqJoFbgYcAVPWqtDHXAUtV9RbveYuInAb8Gfgo8L0cv4+BS79N0Vznem80boETPwonXpvdeozOFeOnWnzDMIwhI6fioapxEfkU8DTgBx5S1fUicgewSlUfA5YBd4mI4rKqbsri1J8EfggUA7/1brmjfpOrfltQ5tJvj5ZtL8GL9zixWH6P66lxOFSdm6qowlaMG4Yx5IhLWBr5LF26VFetWtX/A5MJ2PK8a7J0pFlUneeKw58fgLX/BbXz4LyVUDY+uzlYYNwwjCFARFar6tKe2+0qlC1HKxzt9a6o4b51sOAyOO3/c10DD0ciCh3NLjBuK8YNw8gTTDwGg7pV8PxXIRGB990Gx56T3XGdgfHFRx+cNwzDGEBMPHKJJuH1n8Kqh53VcN7tUDktu2PDLe7eAuOGYeQhJh65ItzsSqjvfBWOOxfO/DwEiw9/nAXGDcMYBph45IL9G+DZla4q7hmfg3kXZhersMC4YRjDBLs6DSSqsP5/4E//6roFXnyfE4FsSMSc2FTPtoq4hmHkPSYeA0U0BC/f69J6p54Oy27NPlbRbcW4BcYNw8h/TDwGgsZtrqhhcx2c8gk44W+yT+0NNwNigXHDMIYVJh5Hy6Zn4eVvuGD4B77hrIdssMC4YRjDGBOPIyUegT/eDxsfg2NOcOs3SqqzO7YzMD4JaudaYNwwjGGHXbWOhJY9zk114B3nojr549kXSrTAuGEYIwATj/6y4xV44WuAwl/fCdPfk/2xFhg3DGOEYOKRLck4rHoI1vwMambDuSuhYmL2x3cGxk+BojG5mqVhGMagYOKRDR0HXW2qPWtg7oXwV5+CQGF2x6q64wvLLTBuGMaIwcTjcOx4BZ76B4h1uLUbs8/P/thkwmVUVUy0wLhhGCMKu5plIpmAJz8HwRJY8U0YOzP7Yy0wbhjGCMbEIxM+P1zxM1erqnJq9sfFQhCxwLhhGCOXo+xwNAoYO9NZHtkSboZEHKaeYsJhGMaIxSyPgSTU6AXGF2ZXft0wDGOYYuIxEGgS2g94gfF5Fhg3DGPEk3O3lYgsF5G3RWSziNzSy/5pIvKciLwpIi+KyOS07X8RkTUisl5Ebkw75kXvnGu829D5hxIxJxzVs2H88SYchmGMCnJ6pRMRP3A/cB5QB7wmIo+p6oa0YfcCP1bVH4nIOcBdwDXAHuB0VY2ISBmwzjt2t3fcVaq6KpfzPyyxDoi0wzGLoXz8kE7FMAxjMMm15XEKsFlVt6pqFHgEuLjHmPnA897jF1L7VTWqqhFve+EgzLV/hFsgHnWBcRMOwzBGGbm+IE8CdqY9r/O2pfMGcJn3+FKgXESqAURkioi86Z3jnjSrA+Bhz2X1JZHeF1GIyA0iskpEVtXX1w/E+3GEGlxAfOqpVmrEMIxRST78mv8CcLaIvA6cDewCEgCqulNVFwHHAdeKSOon/lWquhA407td09uJVfUBVV2qqktra2uPfqaahLb9UDYeJp5oGVWGYYxaci0eu4Apac8ne9s6UdXdqnqZqi4B/snb1tRzDLAOJxSo6i7vvhX4Gc49lls6A+OzLDBuGMaoJ9fi8RowS0RmiEgBcAXwWPoAEakR6ezZeivwkLd9sogUe4+rgDOAt0UkICI13vYgsAInLLlDgY5mFxivnmmlRgzDGPXk9OezqsZF5FPA04AfeEhV14vIHcAqVX0MWAbcJSIKvATc5B0+D/iGt12Ae1V1rYiUAk97wuEHfg/8R+7ehUBpjRMNi28YhmEAIKo61HMYFJYuXaqrVg1tZq9hGMZwQ0RWq+rSntvzIWBuGIZhDDNMPAzDMIx+Y+JhGIZh9BsTD8MwDKPfmHgYhmEY/cbEwzAMw+g3Jh6GYRhGvzHxMAzDMPrNqFkkKCL1wI4jPLwGODCA0xkobF79w+bVP2xe/WOkzmuaqh5SWXbUiMfRICKrelthOdTYvPqHzat/2Lz6x2ibl7mtDMMwjH5j4mEYhmH0GxOP7HhgqCfQBzav/mHz6h82r/4xquZlMQ/DMAyj35jlYRiGYfQbEw/DMAyj35h49EBEHhKR/SKyLm3bWBF5VkQ2efdVgzynKSLygohsEJH1IvKZfJiXN4ciEXlVRN7w5na7t32GiPxZRDaLyC+8NsSDPTe/iLwuIk/ky5y8eWwXkbUiskZEVnnb8uFvWSkivxSRt0Rko4icPtTzEpE53ueUurWIyGeHel7e3P6f951fJyI/9/4vDPl3TEQ+481pvYh81ts24J+Xiceh/BBY3mPbLcBzqjoLeM57PpjEgc+r6nzgNOAmEZmfB/MCiADnqOoJwGJguYicBtwDfEtVjwMOAh8fgrl9BtiY9jwf5pTivaq6OC3/Ph/+lt8Bfqeqc4ETcJ/dkM5LVd/2PqfFwElACPifoZ6XiEwCPg0sVdXjcS2xr2CIv2MicjzwCeAU3N9whYgcRy4+L1W1W48bMB1Yl/b8beAY7/ExwNtDPL/fAOfl4bxKgL8Ap+JWtAa87acDTw/yXCZ7/0nOAZ4AZKjnlDa37UBNj21D+rcExgDb8JJo8mVePeby18D/5cO8gEnATmAsEPC+Y+cP9XcM+DDwg7TnXwJuzsXnZZZHdoxX1T3e473A+KGaiIhMB5YAf86XeXnuoTXAfuBZYAvQpKpxb0gd7j/bYPJt3H+apPe8Og/mlEKBZ0RktYjc4G0b6r/lDKAeeNhz9T0oIqV5MK90rgB+7j0e0nmp6i7gXuBdYA/QDKxm6L9j64AzRaRaREqAC4Ap5ODzMvHoJ+qke0jym0WkDPgV8FlVbcmXealqQp1bYTLOXJ47FPNIISIrgP2qunoo55GBM1T1ROD9OBfkWek7h+hvGQBOBL6vqkuAdnq4Nob4u18AXAT8d899QzEvL2ZwMU50JwKlHOruHnRUdSPOdfYM8DtgDZDoMWZAPi8Tj+zYJyLHAHj3+wd7AiISxAnHf6rqo/kyr3RUtQl4AWeuV4pIwNs1Gdg1iFN5D3CRiGwHHsG5rr4zxHPqxPvViqrux/nvT2Ho/5Z1QJ2q/tl7/kucmAz1vFK8H/iLqu7zng/1vM4FtqlqvarGgEdx37sh/46p6g9U9SRVPQsXd3mHHHxeJh7Z8Rhwrff4WlzMYdAQEQF+AGxU1W/my7y8udWKSKX3uBgXi9mIE5EPDcXcVPVWVZ2sqtNxro7nVfWqoZxTChEpFZHy1GOcH38dQ/y3VNW9wE4RmeNteh+wYajnlcbf0OWygqGf17vAaSJS4v3/TH1e+fAdG+fdTwUuA35GLj6vwQzmDIcb7gu6B4jhfo19HOcvfw7YBPweGDvIczoDZ2a+iTND1+B8mUM6L29ui4DXvbmtA27zts8EXgU241wNhUP091wGPJEvc/Lm8IZ3Ww/8k7c9H/6Wi4FV3t/y10BVnsyrFGgAxqRty4d53Q685X3vfwIU5sl37GWckL0BvC9Xn5eVJzEMwzD6jbmtDMMwjH5j4mEYhmH0GxMPwzAMo9+YeBiGYRj9xsTDMAzD6DcmHoZhGEa/MfEwjH4gIitF5AtD/doicoeInOs9/qxXx8gwBg0TD8MYhqjqbar6e+/pZ3EVjQ1j0DDxMIzDICL/JCLviMgfgDnetk+La871pog8kuHYahF5xmvM86CI7BCRGhGZLt0bjn1BRFZ6jz8hIq+Ja7D1q96sChH5oYh8SEQ+jSvM94K4hmEfE5Fvp437hIh8a8A+DMPwMPEwjAyIyEm4+liLcSVhTvZ23QIsUdVFwI0ZTvFl4A+qugBXBHFqFi/7qKqerK7B1kYyNBRS1e8Cu3HNpd4L/BdwoVdIE+B64KEsXtMw+oWJh2Fk5kzgf1Q1pK4M/mPe9jeB/xSRq3GdHvviLOCnAKr6JK7K6eE4XkReFpG1wFXAgmwnq6ptwPO4DnJzgaCqrs32eMPIFhMPwzgyPgDcjytb/lpaGe5sidP9/19R2uMfAp9S1YW44nvp+7LhQeA6nNXxcD+PNYysMPEwjMy8BFwiIsVeKfULcf9vpqjqC8AXcS1cyzIcfyWAiLwfV6kWYB8wzouJFAIr0o4pB/Z4rqersphjq3cMAOp6ckzxXvfnfR1kGEdDf38tGcaoQlX/IiK/wJW33g+8hiuP/1MRGYPrjf5ddY2weuN24Ocish54BdcHAlWNicgduPLdu3ClvVN8CddmuN67LyczDwC/E5HdXtwDXOxjsapm4yYzjH5jJdkNYxDxuhsuVdUDOX6dJ4BvqepzuXwdY/RibivDGEGISKWIvAN0mHAYucQsD8MYAETkeuAzPTb/n6reNBTzMYxcY+JhGIZh9BtzWxmGYRj9xsTDMAzD6DcmHoZhGEa/MfEwDMMw+s3/D6c+x6U1LM/LAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ev = ExperimentVisual(df)\n",
    "\n",
    "ev.visualize(\n",
    "    xvar=\"ds_quality\",\n",
    "    yvar=\"mAP_0.5\",\n",
    "    legend_var=\"ds_modelSize\",\n",
    "    title=\"AP - quality\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}