aml_hyperdrive.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set up workspace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import shutil\n",
    "\n",
    "from azureml.core.workspace import Workspace\n",
    "from azureml.core import Experiment\n",
    "\n",
    "from azureml.core.compute import ComputeTarget, AmlCompute\n",
    "from azureml.core.compute_target import ComputeTargetException\n",
    "from azureml.train.dnn import PyTorch\n",
    "from azureml.widgets import RunDetails"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Only required for the first time.\n",
    "# subscription_id = '<subscription id>'\n",
    "# resource_grp = '<resource grp>'\n",
    "# ws_name = '<ws name>'\n",
    "# ws  =  Workspace(subscription_id,resource_grp, ws_name)\n",
    "# ws.write_config()\n",
    "\n",
    "ws = Workspace.from_config()\n",
    "\n",
    "ds = ws.get_default_datastore()\n",
    "blob_raw = ds.path('raw')\n",
    "blob_processed = ds.path('precessed')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "project_folder = './'\n",
    "os.makedirs(project_folder, exist_ok=True)\n",
    "\n",
    "experiment_name = 'Testing-AML'\n",
    "experiment = Experiment(ws, name=experiment_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a compute target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found existing compute target\n"
     ]
    }
   ],
   "source": [
    "cluster_name = \"gpucluster\"\n",
    "\n",
    "try:\n",
    "    compute_target = ComputeTarget(workspace=ws, name=cluster_name)\n",
    "    print('Found existing compute target')\n",
    "except ComputeTargetException:\n",
    "    print('Creating a new compute target...')\n",
    "    compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_NC24s_v3', \n",
    "                                                           max_nodes=4,vm_priority='lowpriority')\n",
    "\n",
    "    compute_target = ComputeTarget.create(ws, cluster_name, compute_config)\n",
    "\n",
    "    compute_target.wait_for_completion(show_output=True, min_node_count=None, timeout_in_minutes=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess\n",
    "Follow run_aml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train\n",
    "\n",
    "## Estimator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from azureml.core.runconfig import MpiConfiguration\n",
    "script_params = {\n",
    "    '--input_dir':blob_processed.as_mount(),\n",
    "    '--backend':\"nccl\",\n",
    "    '--is_distributed':\"\",\n",
    "    '--isaml':\"\",\n",
    "    '--val_batch_size':32,\n",
    "    '--epochs':4,\n",
    "    '--output_dir': './outputs', #saving output now\n",
    "    '--fp16':\"\",\n",
    "    '--bert_type':'bert-large-uncased'\n",
    "}\n",
    "\n",
    "image_name = 'krishansubudhi/transformers_pytorch:1.3'\n",
    "mpi=MpiConfiguration()\n",
    "mpi.process_count_per_node = 4\n",
    "\n",
    "estimator = PyTorch(source_directory=project_folder, \n",
    "                    script_params=script_params,\n",
    "                    compute_target=compute_target,\n",
    "                    entry_script='TrainGAP.py',\n",
    "                    node_count=1,\n",
    "                    use_gpu=True,\n",
    "                    distributed_training = mpi,                \n",
    "    \n",
    "                    #Docker image\n",
    "                    use_docker=True,\n",
    "                    custom_docker_image=image_name,\n",
    "                    user_managed=True)\n",
    "estimator._estimator_config.environment.python.interpreter_path = '/opt/miniconda/envs/amlbert/bin/python'\n",
    "\n",
    "\n",
    "#run2 = experiment.submit(estimator)\n",
    "#RunDetails(run2).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.2098, 0.5341, 0.2561]])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.nn.functional.softmax( torch.tensor([[2.711256980895996094e+00,3.645849227905273438e+00,2.910761356353759766e+00]]), dim = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['azureml-logs/55_azureml-execution-tvmps_3abd023539818271cd1df36be6fc6a887099bc0df61fb2531b0374e5660e0c63_p.txt',\n",
       " 'azureml-logs/65_job_prep-tvmps_3abd023539818271cd1df36be6fc6a887099bc0df61fb2531b0374e5660e0c63_p.txt',\n",
       " 'azureml-logs/70_driver_log_0.txt',\n",
       " 'azureml-logs/70_driver_log_1.txt',\n",
       " 'azureml-logs/70_driver_log_2.txt',\n",
       " 'azureml-logs/70_driver_log_3.txt',\n",
       " 'azureml-logs/70_mpi_log.txt',\n",
       " 'azureml-logs/75_job_post-tvmps_3abd023539818271cd1df36be6fc6a887099bc0df61fb2531b0374e5660e0c63_p.txt',\n",
       " 'azureml-logs/process_info.json',\n",
       " 'azureml-logs/process_status.json',\n",
       " 'logs/azureml/0_164_azureml.log',\n",
       " 'logs/azureml/1_181_azureml.log',\n",
       " 'logs/azureml/2_186_azureml.log',\n",
       " 'logs/azureml/3_185_azureml.log',\n",
       " 'logs/azureml/azureml.log',\n",
       " 'outputs/amp_checkpoint.pt',\n",
       " 'outputs/evaluate_labels.csv',\n",
       " 'outputs/evaluate_logits.csv',\n",
       " 'outputs/model_checkpoint.pt']"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run2.get_file_names()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperdrive\n",
    "https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-tune-hyperparameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Sampling space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "from azureml.train.hyperdrive import GridParameterSampling,RandomParameterSampling\n",
    "from azureml.train.hyperdrive.parameter_expressions import choice,loguniform\n",
    "parameter_sampling = RandomParameterSampling(\n",
    "    {\n",
    "        'lr':loguniform( math.log(1E-5), math.log(1E-4) ),\n",
    "        'per_gpu_batch_size':choice(4,8,16),\n",
    "        'model_type':choice('mul','concat'),\n",
    "        'bert_type':choice('bert-base-uncased','bert-large-uncased'),\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Primary metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "from  azureml.train.hyperdrive import PrimaryMetricGoal\n",
    "primary_metric_name=\"val_acc\"\n",
    "primary_metric_goal=PrimaryMetricGoal.MAXIMIZE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Early termination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.train.hyperdrive import MedianStoppingPolicy\n",
    "early_termination_policy = MedianStoppingPolicy(evaluation_interval=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Resources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_total_runs=20\n",
    "max_concurrent_runs=10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Runconfig and run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.train.hyperdrive import HyperDriveConfig\n",
    "hyperdrive_run_config = HyperDriveConfig(estimator = estimator,\n",
    "                                        hyperparameter_sampling = parameter_sampling,\n",
    "                                        primary_metric_name = primary_metric_name,\n",
    "                                        primary_metric_goal= primary_metric_goal,\n",
    "                                        max_total_runs = max_total_runs,\n",
    "                                        max_concurrent_runs = max_concurrent_runs,\n",
    "                                        policy=early_termination_policy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['--input_dir', '--backend', '--is_distributed', '--val_batch_size', '--epochs', '--output_dir', '--fp16'])\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6859165e45954508b3ba705a71134a35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "_HyperDriveWidget(widget_settings={'childWidgetDisplay': 'popup', 'send_telemetry': False, 'log_level': 'INFO'…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/aml.mini.widget.v1": "{\"status\": \"Completed\", \"workbench_run_details_uri\": \"https://ml.azure.com/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207?wsid=/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourcegroups/bert-base/workspaces/SubstrateIntelligenceNLR-WS2\", \"run_id\": \"Testing-AML-hyperdrive_1572978287720207\", \"run_properties\": {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207\", \"created_utc\": \"2019-11-05T18:24:47.802732Z\", \"properties\": {\"primary_metric_config\": \"{\\\"name\\\": \\\"val_acc\\\", \\\"goal\\\": \\\"maximize\\\"}\", \"resume_from\": \"null\", \"runTemplate\": \"HyperDrive\", \"azureml.runsource\": \"hyperdrive\", \"platform\": \"AML\", \"baggage\": \"eyJvaWQiOiAiMGI2OTk5MDUtMTUxYS00MWQ4LTljNjAtMGVjMGVhM2VmNjk1IiwgInRpZCI6ICI3MmY5ODhiZi04NmYxLTQxYWYtOTFhYi0yZDdjZDAxMWRiNDciLCAidW5hbWUiOiAiMDRiMDc3OTUtOGRkYi00NjFhLWJiZWUtMDJmOWUxYmY3YjQ2In0\", \"ContentSnapshotId\": \"5e5c046f-b479-4db3-a04e-ddcefae8ec4c\", \"score\": \"0.9074889867841409\", \"best_child_run_id\": \"Testing-AML-hyperdrive_1572978287720207_7\", \"best_metric_status\": \"Succeeded\"}, \"tags\": {\"max_concurrent_jobs\": \"10\", \"max_total_jobs\": \"20\", \"max_duration_minutes\": \"10080\", \"policy_config\": \"{\\\"name\\\": \\\"MEDIANSTOPPING\\\", \\\"properties\\\": {\\\"evaluation_interval\\\": 1, \\\"delay_evaluation\\\": 0}}\", \"generator_config\": \"{\\\"name\\\": \\\"RANDOM\\\", \\\"parameter_space\\\": {\\\"lr\\\": [\\\"loguniform\\\", [-11.512925464970229, -9.210340371976182]], \\\"per_gpu_batch_size\\\": [\\\"choice\\\", [[4, 8, 16]]], \\\"model_type\\\": [\\\"choice\\\", [[\\\"mul\\\", \\\"concat\\\"]]], \\\"bert_type\\\": [\\\"choice\\\", [[\\\"bert-base-uncased\\\", \\\"bert-large-uncased\\\"]]]}}\", \"primary_metric_config\": \"{\\\"name\\\": \\\"val_acc\\\", \\\"goal\\\": \\\"maximize\\\"}\", \"platform_config\": \"{\\\"ServiceAddress\\\": \\\"https://eastus.experiments.azureml.net\\\", \\\"ServiceArmScope\\\": \\\"subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive\\\", \\\"SubscriptionId\\\": \\\"42ae47bd-b19b-42c1-b0b9-19fd5be9d51b\\\", \\\"ResourceGroupName\\\": \\\"bert-base\\\", \\\"WorkspaceName\\\": \\\"SubstrateIntelligenceNLR-WS2\\\", \\\"ExperimentName\\\": \\\"Testing-AML-hyperdrive\\\", \\\"Definition\\\": {\\\"Overrides\\\": {\\\"script\\\": \\\"TrainGAP.py\\\", \\\"arguments\\\": [\\\"--input_dir\\\", \\\"$AZUREML_DATAREFERENCE_b446e67714204ee1b1cae840b2be866a\\\", \\\"--backend\\\", \\\"nccl\\\", \\\"--is_distributed\\\", \\\"--isaml\\\", \\\"--val_batch_size\\\", 32, \\\"--epochs\\\", 4, \\\"--output_dir\\\", \\\"./outputs\\\", \\\"--fp16\\\", \\\"--bert_type\\\", \\\"bert-large-uncased\\\"], \\\"target\\\": \\\"gpucluster\\\", \\\"framework\\\": \\\"Python\\\", \\\"communicator\\\": \\\"IntelMpi\\\", \\\"maxRunDurationSeconds\\\": null, \\\"nodeCount\\\": 1, \\\"environment\\\": {\\\"name\\\": null, \\\"version\\\": null, \\\"environmentVariables\\\": {\\\"EXAMPLE_ENV_VAR\\\": \\\"EXAMPLE_VALUE\\\", \\\"NCCL_SOCKET_IFNAME\\\": \\\"eth0\\\", \\\"NCCL_IB_DISABLE\\\": \\\"1\\\", \\\"NCCL_TREE_THRESHOLD\\\": \\\"0\\\"}, \\\"python\\\": {\\\"userManagedDependencies\\\": true, \\\"interpreterPath\\\": \\\"/opt/miniconda/envs/amlbert/bin/python\\\", \\\"condaDependenciesFile\\\": null, \\\"baseCondaEnvironment\\\": null, \\\"condaDependencies\\\": {\\\"name\\\": \\\"project_environment\\\", \\\"dependencies\\\": [\\\"python=3.6.2\\\", {\\\"pip\\\": [\\\"azureml-defaults\\\", \\\"torch==1.3\\\", \\\"torchvision==0.4.1\\\", \\\"horovod==0.18.1\\\", \\\"tensorboard==1.14.0\\\", \\\"future==0.17.1\\\"]}], \\\"channels\\\": [\\\"conda-forge\\\"]}}, \\\"docker\\\": {\\\"enabled\\\": true, \\\"baseImage\\\": \\\"krishansubudhi/transformers_pytorch:1.3\\\", \\\"baseDockerfile\\\": null, \\\"sharedVolumes\\\": true, \\\"shmSize\\\": \\\"2g\\\", \\\"arguments\\\": [], \\\"baseImageRegistry\\\": {\\\"address\\\": null, \\\"username\\\": null, \\\"password\\\": null}}, \\\"spark\\\": {\\\"repositories\\\": [], \\\"packages\\\": [], \\\"precachePackages\\\": false}, \\\"databricks\\\": {\\\"mavenLibraries\\\": [], \\\"pypiLibraries\\\": [], \\\"rcranLibraries\\\": [], \\\"jarLibraries\\\": [], \\\"eggLibraries\\\": []}, \\\"inferencingStackVersion\\\": null}, \\\"history\\\": {\\\"outputCollection\\\": true, \\\"snapshotProject\\\": true, \\\"directoriesToWatch\\\": [\\\"logs\\\"]}, \\\"spark\\\": {\\\"configuration\\\": {\\\"spark.app.name\\\": \\\"Azure ML Experiment\\\", \\\"spark.yarn.maxAppAttempts\\\": 1}}, \\\"hdi\\\": {\\\"yarnDeployMode\\\": \\\"cluster\\\"}, \\\"tensorflow\\\": {\\\"workerCount\\\": 1, \\\"parameterServerCount\\\": 1}, \\\"mpi\\\": {\\\"processCountPerNode\\\": 4}, \\\"dataReferences\\\": {\\\"b446e67714204ee1b1cae840b2be866a\\\": {\\\"dataStoreName\\\": \\\"workspaceblobstore\\\", \\\"pathOnDataStore\\\": \\\"precessed\\\", \\\"mode\\\": \\\"mount\\\", \\\"overwrite\\\": false, \\\"pathOnCompute\\\": null}}, \\\"data\\\": {}, \\\"sourceDirectoryDataStore\\\": null, \\\"amlcompute\\\": {\\\"vmSize\\\": null, \\\"vmPriority\\\": null, \\\"retainCluster\\\": false, \\\"name\\\": null, \\\"clusterMaxNodeCount\\\": 1}}, \\\"TargetDetails\\\": null, \\\"SnapshotId\\\": \\\"5e5c046f-b479-4db3-a04e-ddcefae8ec4c\\\", \\\"TelemetryValues\\\": {\\\"amlClientType\\\": \\\"azureml-sdk-train\\\", \\\"amlClientModule\\\": \\\"azureml.train.hyperdrive._search\\\", \\\"amlClientFunction\\\": \\\"search\\\", \\\"tenantId\\\": \\\"72f988bf-86f1-41af-91ab-2d7cd011db47\\\", \\\"amlClientRequestId\\\": \\\"acaf0620-2824-421b-a94e-57ab42fc0bdb\\\", \\\"amlClientSessionId\\\": \\\"9bd7923b-74b2-4928-8a46-c16b7d3696dd\\\", \\\"subscriptionId\\\": \\\"42ae47bd-b19b-42c1-b0b9-19fd5be9d51b\\\", \\\"estimator\\\": \\\"PyTorch\\\", \\\"samplingMethod\\\": \\\"RANDOM\\\", \\\"terminationPolicy\\\": \\\"MedianStopping\\\", \\\"primaryMetricGoal\\\": \\\"maximize\\\", \\\"maxTotalRuns\\\": 20, \\\"maxConcurrentRuns\\\": 10, \\\"maxDurationMinutes\\\": 10080, \\\"computeTarget\\\": \\\"AmlCompute\\\", \\\"vmSize\\\": null}}}\", \"resume_child_runs\": \"null\", \"all_jobs_generated\": \"true\", \"cancellation_requested\": \"false\", \"progress_metadata_evaluation_timestamp\": \"\\\"2019-11-05T18:24:48.316975\\\"\", \"progress_metadata_digest\": \"\\\"7b5a99ea7aab99d3424b308fd479f06d940e19507e616ae1dc951db8ec850ece\\\"\", \"progress_metadata_active_timestamp\": \"\\\"2019-11-05T18:24:48.316975\\\"\", \"Testing-AML-hyperdrive_1572978287720207_0\": \"{\\\"bert_type\\\": \\\"bert-base-uncased\\\", \\\"lr\\\": 1.2405514603795932e-05, \\\"model_type\\\": \\\"mul\\\", \\\"per_gpu_batch_size\\\": 8}\", \"Testing-AML-hyperdrive_1572978287720207_1\": \"{\\\"bert_type\\\": \\\"bert-large-uncased\\\", \\\"lr\\\": 5.4827302693841856e-05, \\\"model_type\\\": \\\"concat\\\", \\\"per_gpu_batch_size\\\": 4}\", \"Testing-AML-hyperdrive_1572978287720207_2\": \"{\\\"bert_type\\\": \\\"bert-base-uncased\\\", \\\"lr\\\": 6.770852139572916e-05, \\\"model_type\\\": \\\"concat\\\", \\\"per_gpu_batch_size\\\": 16}\", \"Testing-AML-hyperdrive_1572978287720207_3\": \"{\\\"bert_type\\\": \\\"bert-large-uncased\\\", \\\"lr\\\": 5.818544334346397e-05, \\\"model_type\\\": \\\"mul\\\", \\\"per_gpu_batch_size\\\": 16}\", \"Testing-AML-hyperdrive_1572978287720207_4\": \"{\\\"bert_type\\\": \\\"bert-large-uncased\\\", \\\"lr\\\": 2.0815384481182826e-05, \\\"model_type\\\": \\\"mul\\\", \\\"per_gpu_batch_size\\\": 4}\", \"Testing-AML-hyperdrive_1572978287720207_5\": \"{\\\"bert_type\\\": \\\"bert-large-uncased\\\", \\\"lr\\\": 8.480288927108734e-05, \\\"model_type\\\": \\\"concat\\\", \\\"per_gpu_batch_size\\\": 16}\", \"Testing-AML-hyperdrive_1572978287720207_6\": \"{\\\"bert_type\\\": \\\"bert-large-uncased\\\", \\\"lr\\\": 1.902429613120226e-05, \\\"model_type\\\": \\\"concat\\\", \\\"per_gpu_batch_size\\\": 8}\", \"Testing-AML-hyperdrive_1572978287720207_7\": \"{\\\"bert_type\\\": \\\"bert-large-uncased\\\", \\\"lr\\\": 2.207997595066282e-05, \\\"model_type\\\": \\\"mul\\\", \\\"per_gpu_batch_size\\\": 16}\", \"Testing-AML-hyperdrive_1572978287720207_8\": \"{\\\"bert_type\\\": \\\"bert-large-uncased\\\", \\\"lr\\\": 8.56043793493588e-05, \\\"model_type\\\": \\\"mul\\\", \\\"per_gpu_batch_size\\\": 8}\", \"Testing-AML-hyperdrive_1572978287720207_9\": \"{\\\"bert_type\\\": \\\"bert-large-uncased\\\", \\\"lr\\\": 8.448649148887327e-05, \\\"model_type\\\": \\\"concat\\\", \\\"per_gpu_batch_size\\\": 8}\", \"environment_preparation_status\": \"PREPARED\", \"prepare_run_id\": \"Testing-AML-hyperdrive_1572978287720207_preparation\", \"Testing-AML-hyperdrive_1572978287720207_2_cancelled\": \"true\", \"Testing-AML-hyperdrive_1572978287720207_10\": \"{\\\"bert_type\\\": \\\"bert-large-uncased\\\", \\\"lr\\\": 6.709830566187861e-05, \\\"model_type\\\": \\\"mul\\\", \\\"per_gpu_batch_size\\\": 4}\", \"Testing-AML-hyperdrive_1572978287720207_11\": \"{\\\"bert_type\\\": \\\"bert-large-uncased\\\", \\\"lr\\\": 7.009645060380695e-05, \\\"model_type\\\": \\\"mul\\\", \\\"per_gpu_batch_size\\\": 8}\", \"Testing-AML-hyperdrive_1572978287720207_5_cancelled\": \"true\", \"Testing-AML-hyperdrive_1572978287720207_8_cancelled\": \"true\", \"Testing-AML-hyperdrive_1572978287720207_12\": \"{\\\"bert_type\\\": \\\"bert-base-uncased\\\", \\\"lr\\\": 3.3323897517536887e-05, \\\"model_type\\\": \\\"mul\\\", \\\"per_gpu_batch_size\\\": 16}\", \"Testing-AML-hyperdrive_1572978287720207_6_cancelled\": \"true\", \"Testing-AML-hyperdrive_1572978287720207_9_cancelled\": \"true\", \"Testing-AML-hyperdrive_1572978287720207_13\": \"{\\\"bert_type\\\": \\\"bert-base-uncased\\\", \\\"lr\\\": 9.96584194144014e-05, \\\"model_type\\\": \\\"mul\\\", \\\"per_gpu_batch_size\\\": 4}\", \"Testing-AML-hyperdrive_1572978287720207_14\": \"{\\\"bert_type\\\": \\\"bert-base-uncased\\\", \\\"lr\\\": 7.607335219870598e-05, \\\"model_type\\\": \\\"concat\\\", \\\"per_gpu_batch_size\\\": 8}\", \"Testing-AML-hyperdrive_1572978287720207_15\": \"{\\\"bert_type\\\": \\\"bert-large-uncased\\\", \\\"lr\\\": 1.1808218461391666e-05, \\\"model_type\\\": \\\"mul\\\", \\\"per_gpu_batch_size\\\": 16}\", \"Testing-AML-hyperdrive_1572978287720207_1_cancelled\": \"true\", \"Testing-AML-hyperdrive_1572978287720207_16\": \"{\\\"bert_type\\\": \\\"bert-base-uncased\\\", \\\"lr\\\": 3.593147086897413e-05, \\\"model_type\\\": \\\"concat\\\", \\\"per_gpu_batch_size\\\": 8}\", \"Testing-AML-hyperdrive_1572978287720207_17\": \"{\\\"bert_type\\\": \\\"bert-large-uncased\\\", \\\"lr\\\": 8.860957698790345e-05, \\\"model_type\\\": \\\"mul\\\", \\\"per_gpu_batch_size\\\": 4}\", \"Testing-AML-hyperdrive_1572978287720207_18\": \"{\\\"bert_type\\\": \\\"bert-large-uncased\\\", \\\"lr\\\": 2.45364601977547e-05, \\\"model_type\\\": \\\"mul\\\", \\\"per_gpu_batch_size\\\": 16}\", \"Testing-AML-hyperdrive_1572978287720207_19\": \"{\\\"bert_type\\\": \\\"bert-large-uncased\\\", \\\"lr\\\": 8.707551225259661e-05, \\\"model_type\\\": \\\"mul\\\", \\\"per_gpu_batch_size\\\": 8}\", \"Testing-AML-hyperdrive_1572978287720207_14_cancelled\": \"true\", \"Testing-AML-hyperdrive_1572978287720207_10_cancelled\": \"true\", \"Testing-AML-hyperdrive_1572978287720207_17_cancelled\": \"true\", \"Testing-AML-hyperdrive_1572978287720207_19_cancelled\": \"true\"}, \"end_time_utc\": \"2019-11-05T18:44:31.342918Z\", \"status\": \"Completed\", \"log_files\": {\"azureml-logs/hyperdrive.txt\": \"https://substrateintel3704284680.blob.core.windows.net/azureml/ExperimentRun/dcid.Testing-AML-hyperdrive_1572978287720207/azureml-logs/hyperdrive.txt?sv=2019-02-02&sr=b&sig=Zkm3wXnUfo7WSKB3j4GHQryn2GZ96zsGGlQ%2B82heBco%3D&st=2019-11-05T18%3A34%3A45Z&se=2019-11-06T02%3A44%3A45Z&sp=r\"}, \"log_groups\": [[\"azureml-logs/hyperdrive.txt\"]], \"run_duration\": \"0:19:43\", \"hyper_parameters\": {\"lr\": [\"loguniform\", [-11.512925464970229, -9.210340371976182]], \"per_gpu_batch_size\": [\"choice\", [[4, 8, 16]]], \"model_type\": [\"choice\", [[\"mul\", \"concat\"]]], \"bert_type\": [\"choice\", [[\"bert-base-uncased\", \"bert-large-uncased\"]]]}}, \"child_runs\": [{\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_4\", \"run_number\": 205, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:25:33.149627Z\", \"end_time\": \"2019-11-05T18:34:51.624352Z\", \"created_time\": \"2019-11-05T18:25:22.4106Z\", \"created_time_dt\": \"2019-11-05T18:25:22.4106Z\", \"duration\": \"0:09:29\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-large-uncased\", \"param_lr\": 2.0815384481182826e-05, \"param_model_type\": \"mul\", \"param_per_gpu_batch_size\": 4, \"best_metric\": 0.8942731277533039}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_5\", \"run_number\": 206, \"metric\": null, \"status\": \"Canceled\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:25:35.940796Z\", \"end_time\": \"2019-11-05T18:31:24.768467Z\", \"created_time\": \"2019-11-05T18:25:22.462343Z\", \"created_time_dt\": \"2019-11-05T18:25:22.462343Z\", \"duration\": \"0:06:02\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-large-uncased\", \"param_lr\": 8.480288927108734e-05, \"param_model_type\": \"concat\", \"param_per_gpu_batch_size\": 16, \"best_metric\": 0.6894273127753304}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_8\", \"run_number\": 207, \"metric\": null, \"status\": \"Canceled\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:25:34.672875Z\", \"end_time\": \"2019-11-05T18:31:22.096667Z\", \"created_time\": \"2019-11-05T18:25:22.701217Z\", \"created_time_dt\": \"2019-11-05T18:25:22.701217Z\", \"duration\": \"0:05:59\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-large-uncased\", \"param_lr\": 8.56043793493588e-05, \"param_model_type\": \"mul\", \"param_per_gpu_batch_size\": 8, \"best_metric\": 0.4052863436123348}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_0\", \"run_number\": 208, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:25:36.288769Z\", \"end_time\": \"2019-11-05T18:28:57.498139Z\", \"created_time\": \"2019-11-05T18:25:22.730552Z\", \"created_time_dt\": \"2019-11-05T18:25:22.730552Z\", \"duration\": \"0:03:34\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-base-uncased\", \"param_lr\": 1.2405514603795932e-05, \"param_model_type\": \"mul\", \"param_per_gpu_batch_size\": 8, \"best_metric\": 0.8590308370044053}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_6\", \"run_number\": 209, \"metric\": null, \"status\": \"Canceled\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:25:35.895218Z\", \"end_time\": \"2019-11-05T18:31:52.890478Z\", \"created_time\": \"2019-11-05T18:25:22.908123Z\", \"created_time_dt\": \"2019-11-05T18:25:22.908123Z\", \"duration\": \"0:06:29\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-large-uncased\", \"param_lr\": 1.902429613120226e-05, \"param_model_type\": \"concat\", \"param_per_gpu_batch_size\": 8, \"best_metric\": 0.8414096916299559}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_1\", \"run_number\": 210, \"metric\": null, \"status\": \"Canceled\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:25:36.805021Z\", \"end_time\": \"2019-11-05T18:33:01.648953Z\", \"created_time\": \"2019-11-05T18:25:23.116672Z\", \"created_time_dt\": \"2019-11-05T18:25:23.116672Z\", \"duration\": \"0:07:38\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-large-uncased\", \"param_lr\": 5.4827302693841856e-05, \"param_model_type\": \"concat\", \"param_per_gpu_batch_size\": 4, \"best_metric\": 0.8502202643171806}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_9\", \"run_number\": 211, \"metric\": null, \"status\": \"Canceled\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:25:36.31104Z\", \"end_time\": \"2019-11-05T18:31:52.171474Z\", \"created_time\": \"2019-11-05T18:25:23.135535Z\", \"created_time_dt\": \"2019-11-05T18:25:23.135535Z\", \"duration\": \"0:06:29\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-large-uncased\", \"param_lr\": 8.448649148887327e-05, \"param_model_type\": \"concat\", \"param_per_gpu_batch_size\": 8, \"best_metric\": 0.8259911894273128}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_3\", \"run_number\": 212, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:25:36.701656Z\", \"end_time\": \"2019-11-05T18:34:47.077772Z\", \"created_time\": \"2019-11-05T18:25:23.123899Z\", \"created_time_dt\": \"2019-11-05T18:25:23.123899Z\", \"duration\": \"0:09:23\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-large-uncased\", \"param_lr\": 5.818544334346397e-05, \"param_model_type\": \"mul\", \"param_per_gpu_batch_size\": 16, \"best_metric\": 0.8810572687224669}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_7\", \"run_number\": 213, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:25:33.695584Z\", \"end_time\": \"2019-11-05T18:34:05.871558Z\", \"created_time\": \"2019-11-05T18:25:23.287578Z\", \"created_time_dt\": \"2019-11-05T18:25:23.287578Z\", \"duration\": \"0:08:42\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-large-uncased\", \"param_lr\": 2.207997595066282e-05, \"param_model_type\": \"mul\", \"param_per_gpu_batch_size\": 16, \"best_metric\": 0.9074889867841409}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_2\", \"run_number\": 214, \"metric\": null, \"status\": \"Canceled\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:25:43.385765Z\", \"end_time\": \"2019-11-05T18:29:13.836395Z\", \"created_time\": \"2019-11-05T18:25:23.623377Z\", \"created_time_dt\": \"2019-11-05T18:25:23.623377Z\", \"duration\": \"0:03:50\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-base-uncased\", \"param_lr\": 6.770852139572916e-05, \"param_model_type\": \"concat\", \"param_per_gpu_batch_size\": 16, \"best_metric\": 0.8171806167400881}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_10\", \"run_number\": 215, \"metric\": null, \"status\": \"Canceled\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:29:49.174711Z\", \"end_time\": \"2019-11-05T18:36:20.596678Z\", \"created_time\": \"2019-11-05T18:29:37.223545Z\", \"created_time_dt\": \"2019-11-05T18:29:37.223545Z\", \"duration\": \"0:06:43\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-large-uncased\", \"param_lr\": 6.709830566187861e-05, \"param_model_type\": \"mul\", \"param_per_gpu_batch_size\": 4, \"best_metric\": 0.44493392070484583}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_11\", \"run_number\": 216, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:29:52.914868Z\", \"end_time\": \"2019-11-05T18:38:37.884869Z\", \"created_time\": \"2019-11-05T18:29:38.01557Z\", \"created_time_dt\": \"2019-11-05T18:29:38.01557Z\", \"duration\": \"0:08:59\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-large-uncased\", \"param_lr\": 7.009645060380695e-05, \"param_model_type\": \"mul\", \"param_per_gpu_batch_size\": 8, \"best_metric\": 0.8656387665198237}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_12\", \"run_number\": 217, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:31:57.996233Z\", \"end_time\": \"2019-11-05T18:35:42.141999Z\", \"created_time\": \"2019-11-05T18:31:47.264321Z\", \"created_time_dt\": \"2019-11-05T18:31:47.264321Z\", \"duration\": \"0:03:54\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-base-uncased\", \"param_lr\": 3.3323897517536887e-05, \"param_model_type\": \"mul\", \"param_per_gpu_batch_size\": 16, \"best_metric\": 0.8612334801762115}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_14\", \"run_number\": 218, \"metric\": null, \"status\": \"Canceled\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:32:29.248301Z\", \"end_time\": \"2019-11-05T18:35:46.052878Z\", \"created_time\": \"2019-11-05T18:32:18.637943Z\", \"created_time_dt\": \"2019-11-05T18:32:18.637943Z\", \"duration\": \"0:03:27\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-base-uncased\", \"param_lr\": 7.607335219870598e-05, \"param_model_type\": \"concat\", \"param_per_gpu_batch_size\": 8, \"best_metric\": 0.8105726872246696}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_13\", \"run_number\": 219, \"metric\": null, \"status\": \"Failed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:32:32.975298Z\", \"end_time\": \"2019-11-05T18:35:27.384675Z\", \"created_time\": \"2019-11-05T18:32:18.706399Z\", \"created_time_dt\": \"2019-11-05T18:32:18.706399Z\", \"duration\": \"0:03:08\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-base-uncased\", \"param_lr\": 9.96584194144014e-05, \"param_model_type\": \"mul\", \"param_per_gpu_batch_size\": 4}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_15\", \"run_number\": 220, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:32:33.908404Z\", \"end_time\": \"2019-11-05T18:41:01.04172Z\", \"created_time\": \"2019-11-05T18:32:19.336006Z\", \"created_time_dt\": \"2019-11-05T18:32:19.336006Z\", \"duration\": \"0:08:41\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-large-uncased\", \"param_lr\": 1.1808218461391666e-05, \"param_model_type\": \"mul\", \"param_per_gpu_batch_size\": 16, \"best_metric\": 0.8810572687224669}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_16\", \"run_number\": 221, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:34:09.744525Z\", \"end_time\": \"2019-11-05T18:37:37.819691Z\", \"created_time\": \"2019-11-05T18:33:53.083568Z\", \"created_time_dt\": \"2019-11-05T18:33:53.083568Z\", \"duration\": \"0:03:44\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-base-uncased\", \"param_lr\": 3.593147086897413e-05, \"param_model_type\": \"concat\", \"param_per_gpu_batch_size\": 8, \"best_metric\": 0.8237885462555066}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_17\", \"run_number\": 222, \"metric\": null, \"status\": \"Canceled\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:35:12.791108Z\", \"end_time\": \"2019-11-05T18:42:08.024255Z\", \"created_time\": \"2019-11-05T18:34:56.118031Z\", \"created_time_dt\": \"2019-11-05T18:34:56.118031Z\", \"duration\": \"0:07:11\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-large-uncased\", \"param_lr\": 8.860957698790345e-05, \"param_model_type\": \"mul\", \"param_per_gpu_batch_size\": 4, \"best_metric\": 0.7907488986784141}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_18\", \"run_number\": 223, \"metric\": null, \"status\": \"Completed\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:35:07.86319Z\", \"end_time\": \"2019-11-05T18:43:59.815857Z\", \"created_time\": \"2019-11-05T18:34:57.037792Z\", \"created_time_dt\": \"2019-11-05T18:34:57.037792Z\", \"duration\": \"0:09:02\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-large-uncased\", \"param_lr\": 2.45364601977547e-05, \"param_model_type\": \"mul\", \"param_per_gpu_batch_size\": 16, \"best_metric\": 0.9052863436123348}, {\"run_id\": \"Testing-AML-hyperdrive_1572978287720207_19\", \"run_number\": 224, \"metric\": null, \"status\": \"Canceled\", \"run_type\": \"azureml.scriptrun\", \"training_percent\": null, \"start_time\": \"2019-11-05T18:35:12.170853Z\", \"end_time\": \"2019-11-05T18:42:10.454341Z\", \"created_time\": \"2019-11-05T18:34:57.182157Z\", \"created_time_dt\": \"2019-11-05T18:34:57.182157Z\", \"duration\": \"0:07:13\", \"hyperdrive_id\": \"1572978287720207\", \"arguments\": null, \"param_bert_type\": \"bert-large-uncased\", \"param_lr\": 8.707551225259661e-05, \"param_model_type\": \"mul\", \"param_per_gpu_batch_size\": 8, \"best_metric\": 0.8458149779735683}], \"children_metrics\": {\"categories\": [0, 1, 2, 3], \"series\": {\"train_loss\": [{\"run_id\": 208, \"name\": 208, \"data\": [0.5572365522384644, 1.0827986001968384, 1.151267409324646, 0.972877025604248, 0.42907658219337463, 0.9860101938247681, 0.9829902648925781, 1.2095719575881958, 0.8401955962181091, 0.8654980659484863, 0.47125357389450073, 1.3616966009140015, 0.7911338806152344, 0.7089431881904602, 0.4804113507270813, 0.48830080032348633, 0.383902907371521, 0.35828980803489685, 0.37553277611732483, 0.5530180931091309, 0.6785906553268433, 0.7974437475204468, 0.28405359387397766, 0.3915614187717438, 0.47468405961990356, 1.0359852313995361, 0.3962622284889221, 0.20958653092384338, 0.27959686517715454, 0.5197943449020386, 0.3368751108646393, 0.24256744980812073, 0.7077205777168274, 0.26878705620765686, 0.8072637319564819, 0.3661864101886749, 0.290615051984787, 0.10684949159622192, 0.5017489194869995, 0.42983657121658325, 0.35271596908569336, 0.40971696376800537, 0.4945049285888672, 0.7814481258392334, 0.6245015859603882, 0.5015272498130798, 0.49881261587142944, 0.76854407787323, 0.2777969241142273, 0.4833085834980011, 0.5848617553710938, 0.4189920425415039, 0.3582122027873993, 0.8494163155555725, 0.6735191345214844, 0.45549073815345764, 0.28470274806022644, 0.9697436094284058, 0.20083856582641602, 0.6405556201934814, 0.8678503036499023, 0.16306936740875244, 0.13722890615463257, 0.39503800868988037, 0.1625322699546814, 0.24977320432662964, 0.6577317118644714, 0.1653049886226654, 0.6045944094657898, 0.408671498298645, 0.3859981298446655, 0.14820536971092224, 0.9835172891616821, 0.24494504928588867, 0.9906607866287231, 0.49956077337265015, 0.24227359890937805, 0.1535831093788147, 0.31763744354248047, 0.09761953353881836, 0.06075263023376465, 0.249119371175766, 0.3039996027946472, 0.25873786211013794, 0.12402337789535522, 0.11530375480651855, 0.3095744252204895, 0.11784911155700684, 0.46373167634010315, 0.23871919512748718, 0.37672650814056396, 0.2082844376564026, 0.25154492259025574, 0.10119515657424927, 0.1047438383102417, 0.09488540887832642, 0.532730758190155, 0.5222707986831665, 0.12688523530960083, 0.06832224130630493, 0.037655532360076904, 0.36160343885421753, 0.11091029644012451, 0.26688069105148315, 0.18731695413589478, 0.12049457430839539, 0.7019185423851013, 0.3718458116054535, 0.5442301630973816, 0.29057297110557556, 0.7856484651565552, 0.5802745819091797, 0.15097472071647644, 0.22074493765830994, 0.2665080726146698, 0.14914178848266602, 0.42361685633659363, 0.410106897354126, 0.18467041850090027, 0.09781724214553833, 0.718558132648468, 0.07623934745788574, 0.16024532914161682, 0.3877299427986145, 0.13454598188400269, 0.02712118625640869, 0.05617254972457886, 0.11051681637763977, 0.2229670286178589, 0.1407712697982788, 0.1321912407875061, 0.2459733784198761, 0.08027157187461853, 0.06441113352775574, 0.08055812120437622, 0.14058518409729004, 0.10966378450393677, 0.4666237235069275, 0.24982154369354248, 0.07068613171577454, 0.05249309539794922, 0.24015212059020996, 0.04660439491271973, 0.02330559492111206, 0.1498447060585022, 0.24399790167808533, 0.2830226421356201, 0.09164205193519592, 0.05980497598648071, 0.09608125686645508, 0.04415774345397949, 0.3966768682003021, 0.10948291420936584, 0.04351752996444702, 0.0853809118270874, 0.3406795263290405, 0.08798974752426147, 0.02037566900253296, 0.21058261394500732, 0.1401991844177246, 0.2238054871559143, 0.10669422149658203, 0.058346569538116455, 0.0355677604675293, 0.16034305095672607, 0.1008966863155365, 0.1836000680923462, 0.12823927402496338, 0.05507728457450867, 0.14457812905311584, 0.20883098244667053, 0.33248603343963623, 0.19054201245307922, 0.5690329670906067, 0.11111794412136078, 0.07472443580627441, 0.05502775311470032, 0.3911053240299225, 0.11260247230529785, 0.17037057876586914, 0.5037358999252319, 0.09226381778717041, 0.051115572452545166, 0.07617267966270447, 0.05055314302444458, 0.1757071614265442, 0.05329596996307373, 0.0490877628326416, 0.021043896675109863, 0.03884744644165039, 0.02397194504737854, 0.08496907353401184, 0.08368465304374695, 0.03757345676422119, 0.07461237907409668, 0.06300285458564758, 0.04597115516662598, 0.06071200966835022, 0.12054222822189331, 0.025899648666381836, 0.11555597186088562, 0.11466068029403687, 0.05409109592437744, 0.04334527254104614, 0.16382242739200592, 0.034194350242614746, 0.055465519428253174, 0.03608071804046631, 0.113104909658432, 0.13859325647354126, 0.03504228591918945, 0.041279494762420654, 0.18531757593154907, 0.03612715005874634, 0.2397984266281128, 0.08527722954750061, 0.01258939504623413, 0.0344386100769043, 0.15814194083213806, 0.013701200485229492, 0.021145164966583252, 0.1101524829864502, 0.12480074167251587, 0.2159789800643921, 0.025801360607147217, 0.01046299934387207, 0.01040881872177124, 0.07193279266357422, 0.03334653377532959, 0.18890845775604248, 0.03784209489822388, 0.03163444995880127, 0.13348367810249331, 0.092611163854599, 0.3670099079608917, 0.04033017158508301, 0.24901849031448364, 0.04774293303489685, 0.023570269346237183, 0.04344502091407776, 0.08359423279762268, 0.044390738010406494, 0.05304887890815735, 0.27697643637657166, 0.08335253596305847, 0.07254046201705933, 0.06936436891555786, 0.014296621084213257, 0.01182025671005249, 0.059921860694885254, 0.020110905170440674, 0.004199028015136719], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 214, \"name\": 214, \"data\": [0.9157823026180267, 1.2913713455200195, 0.9502306580543518, 1.1178103685379028, 0.792391300201416, 0.9482003450393677, 0.9455956220626831, 0.5293947756290436, 0.614003375172615, 0.7214201092720032, 1.0497246384620667, 0.4977562874555588, 0.7093117237091064, 0.5444692373275757, 0.5413652509450912, 0.35211730003356934, 0.7290256917476654, 0.8943822830915451, 0.38328877091407776, 0.6838280558586121, 0.6238921880722046, 0.4481084495782852, 0.6952411830425262, 0.7608675956726074, 0.3170476630330086, 0.6974776983261108, 0.8096807599067688, 0.45295777916908264, 0.8575769662857056, 0.3678337335586548, 0.8308647722005844, 0.2883324548602104, 0.3873675763607025, 0.32171645760536194, 0.3410599157214165, 0.25193219631910324, 0.4769314378499985, 0.33623167872428894, 0.2534484937787056, 0.12476745992898941, 0.2852562963962555, 0.47368769347667694, 0.14902161806821823, 0.24438630789518356, 0.1838240772485733, 0.15717336535453796, 0.1623065173625946, 0.16723470389842987, 0.25340643525123596, 0.03259894251823425, 0.13326753675937653, 0.1555246263742447, 0.11837992817163467, 0.27136972546577454, 0.2187657207250595, 0.06106115132570267, 0.18773538619279861, 0.20023809373378754, 0.12227708101272583, 0.3059844821691513, 0.09164883196353912, 0.1260177120566368, 0.02887853980064392, 0.06175455451011658, 0.037995293736457825, 0.03210847079753876, 0.11948400735855103, 0.09217367321252823, 0.03567969799041748, 0.097797691822052, 0.16160624474287033, 0.03176718205213547, 0.05296634882688522, 0.251651406288147, 0.01599934697151184, 0.06719844788312912, 0.04332028329372406, 0.02257426083087921, 0.05964024364948273, 0.013612434267997742, 0.013152822852134705, 0.02144336700439453, 0.0905081033706665, 0.0247938334941864, 0.02163536846637726, 0.02811954915523529, 0.00973471999168396, 0.09820578992366791, 0.019964583218097687, 0.044435203075408936, 0.03181140124797821, 0.011600255966186523, 0.010245487093925476, 0.008631631731987, 0.00830128788948059, 0.0894627720117569, 0.004759103059768677, 0.00890505313873291, 0.012192323803901672, 0.07161952555179596, 0.01744908094406128, 0.010556116700172424, 0.06062042713165283, 0.04669763147830963, 0.0057811439037323, 0.02314707636833191, 0.03757473826408386, 0.09754648804664612, 0.0048093050718307495, 0.009359791874885559, 0.026355117559432983, 0.003026321530342102, 0.012915551662445068, 0.046656638383865356, 0.008746415376663208, 0.027945339679718018, 0.008794665336608887, 0.005984753370285034, 0.00771757960319519, 0.00631105899810791, 0.025073006749153137, 0.00516374409198761, 0.004174679517745972, 0.016156062483787537], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 206, \"name\": 206, \"data\": [1.1167228370904922, 1.7079859673976898, 1.1696460619568825, 1.2068363726139069, 0.912063717842102, 1.1626622378826141, 0.9032573997974396, 0.6560910418629646, 1.0080018788576126, 0.7637191936373711, 1.1423381865024567, 0.5743369311094284, 0.810310423374176, 0.8639910221099854, 0.7868769392371178, 0.45703140646219254, 0.783160850405693, 1.102762058377266, 0.4905644878745079, 0.6368994563817978, 0.7956662029027939, 0.8294700533151627, 0.9404245018959045, 0.726971223950386, 0.5411487519741058, 0.6444834694266319, 0.8175046816468239, 0.7330257594585419, 0.8649610728025436, 0.47734756767749786, 0.5857319831848145, 0.6340147629380226, 0.7626024931669235, 0.6934176385402679, 0.546731673181057, 0.7895732372999191, 1.0659139752388, 0.5537808164954185, 0.4702890068292618, 0.44497033953666687, 0.805268794298172, 0.9808951914310455, 0.6352619305253029, 0.5716895759105682, 0.6759754568338394, 0.6896798983216286, 0.6201160699129105, 0.6579843312501907, 0.912145122885704, 0.4398382008075714, 0.5159491375088692, 0.6259790584445, 0.9376888871192932, 0.9374227821826935, 0.7066880837082863, 0.8406246155500412, 0.9409397467970848, 0.8129956126213074, 0.8914149254560471, 0.5970199555158615, 0.4445860683917999, 0.7247421443462372, 1.0863975584506989, 1.21555794775486, 1.3114847838878632, 0.9615100473165512, 1.163577452301979, 0.8647516667842865, 0.8742070198059082, 0.6703519821166992, 0.7740816175937653, 0.8866942077875137, 1.1744450777769089, 0.6545166820287704, 0.692001149058342, 0.6427387818694115, 1.560541793704033, 0.8678315430879593, 1.9758685231208801, 1.2513286769390106, 1.039769858121872, 0.817339226603508, 0.9246092140674591, 1.0804436057806015, 1.1876036822795868, 0.9676795601844788, 0.9204954952001572, 1.2343966960906982], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 207, \"name\": 207, \"data\": [0.6940784156322479, 0.9248494803905487, 0.9465457499027252, 0.8160428404808044, 0.5361743867397308, 0.8973810970783234, 0.6232515573501587, 0.620444655418396, 0.3404288589954376, 1.0031020045280457, 0.3725815713405609, 0.9888916611671448, 0.5217797160148621, 0.5520433485507965, 0.279981791973114, 0.609885573387146, 0.661024272441864, 0.1463683843612671, 0.48467010259628296, 0.35958531498908997, 0.4248439073562622, 0.35710805654525757, 0.28652578592300415, 0.519045352935791, 0.1942080855369568, 0.986745297908783, 0.3422303795814514, 0.04637366533279419, 0.155553936958313, 0.631227970123291, 0.31121641397476196, 0.3540821075439453, 0.4723583459854126, 0.5997762978076935, 0.6017484068870544, 0.5999954342842102, 0.23688864707946777, 0.4179701805114746, 0.4811670184135437, 0.09875261783599854, 0.6987464427947998, 1.2699757814407349, 1.0644246935844421, 0.6314229965209961, 0.6287730932235718, 0.32631516456604004, 0.6093409657478333, 0.8828568458557129, 0.44972091913223267, 0.3955453634262085, 0.8669792413711548, 0.7450249195098877, 0.2830711007118225, 0.9783589839935303, 0.8155813813209534, 0.260983943939209, 0.41837871074676514, 0.7084834575653076, 0.28573524951934814, 0.8821414709091187, 0.554011344909668, 1.9411909580230713, 1.0730245113372803, 2.856188416481018, 4.202103137969971, 1.8899962902069092, 0.6925021409988403, 1.740865707397461, 1.5407302379608154, 1.2580567598342896, 1.1157078742980957, 1.0740519762039185, 0.9192711114883423, 1.4502403736114502, 1.1606403589248657, 0.8597404956817627, 0.6920143961906433, 0.7772798538208008, 0.8343418836593628, 0.8116440773010254, 0.7417378425598145, 0.9040546417236328, 1.3057276010513306, 0.9729132652282715, 0.7457275390625, 0.7901891469955444, 0.7413183450698853, 0.8070857524871826, 0.8512816429138184, 0.8565999269485474, 0.8841670751571655, 1.0112806558609009, 0.806289553642273, 0.7798011302947998, 1.2281053066253662, 0.8331930637359619, 1.1879777908325195, 0.8453640937805176, 0.8967543840408325, 0.8662382364273071, 0.9483801126480103, 0.8255813121795654, 0.802125096321106, 0.9644622802734375, 1.0200071334838867, 1.1121937036514282, 1.1029523611068726, 1.0667273998260498, 1.1378945112228394, 0.7284290790557861, 0.7017794847488403, 1.0657061338424683, 0.9941630363464355, 1.3753256797790527, 0.993224024772644, 1.0559906959533691, 0.9333869218826294, 1.2063159942626953, 0.9321274757385254, 0.8976013660430908, 0.9260948896408081, 0.7753223180770874, 0.8140028715133667, 0.7610734701156616, 1.124461054801941, 1.0469576120376587, 1.1819502115249634, 1.0060118436813354, 0.8149794340133667, 0.9964779615402222, 0.9104453325271606, 1.177672266960144, 0.9694879055023193, 0.8485640287399292, 0.81443190574646, 1.372085690498352, 1.0133737325668335, 0.9670091867446899, 0.8589460849761963, 0.870037317276001, 0.911543607711792, 0.7711809873580933, 0.8538691997528076, 1.0204505920410156, 1.2400387525558472, 1.0167646408081055, 0.7879685163497925, 0.7421221733093262, 0.7307342290878296, 0.7477760314941406, 0.8332526683807373, 0.794165849685669, 0.7444356679916382, 1.0595290660858154, 0.809302806854248, 0.7226647138595581, 1.20652437210083, 0.7632452249526978, 1.057974934577942, 0.868363618850708, 0.8277792930603027, 0.8545171022415161, 0.8509615659713745, 0.8769503831863403, 0.8627175092697144, 1.0560001134872437, 0.9803197383880615, 1.0141476392745972, 0.9625951051712036, 1.05538809299469], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 209, \"name\": 209, \"data\": [1.207099199295044, 1.0263464748859406, 1.0639475882053375, 1.084607183933258, 0.9993481934070587, 1.1051235496997833, 0.8197365403175354, 1.106050431728363, 1.1191830039024353, 0.9184456467628479, 0.6713790744543076, 1.7614116668701172, 0.9972478747367859, 0.8950376212596893, 0.6766493916511536, 0.5613842755556107, 0.950432151556015, 0.5010026842355728, 0.6793113499879837, 0.7089298963546753, 1.3645952939987183, 0.9304391741752625, 0.48353710770606995, 0.6894046366214752, 0.5919545590877533, 0.8480260074138641, 0.5296459496021271, 0.5436287522315979, 0.5078623741865158, 0.8617103099822998, 0.5800345242023468, 0.514025554060936, 0.8174559026956558, 0.6425396651029587, 1.499889850616455, 0.43573740124702454, 0.366436630487442, 0.5005393773317337, 0.6032164841890335, 0.6154110133647919, 0.5733206123113632, 0.7545107305049896, 0.7684583067893982, 0.8667657971382141, 0.8754475861787796, 0.6846412867307663, 0.982103019952774, 0.45527786016464233, 0.5692859441041946, 0.5395198315382004, 0.551103264093399, 0.8316503763198853, 0.5492657870054245, 1.0827260315418243, 0.5184576213359833, 0.7732982486486435, 0.5655647218227386, 1.2091744542121887, 0.34331099689006805, 0.5249907225370407, 0.7995442748069763, 0.4063040614128113, 0.39256010949611664, 0.42434531450271606, 0.49936075508594513, 0.5967267155647278, 0.3927946984767914, 0.7797522097826004, 0.2044735848903656, 0.24613147974014282, 0.2861068397760391, 0.6919880509376526, 0.43952512741088867, 1.068562239408493, 0.5544535964727402, 0.20762762427330017, 0.15551939606666565, 0.3171691447496414, 0.5900556743144989, 0.21163460612297058, 0.3386823534965515, 0.3149438202381134, 0.7996604442596436, 0.34391674399375916, 0.23687279224395752, 0.13634490966796875, 0.20113128423690796, 0.749862551689148, 0.20788158476352692, 0.08986850082874298, 0.14458969235420227, 0.7929700314998627, 0.3674738109111786, 0.14876604080200195, 0.33702945709228516, 0.307669922709465, 0.7630765438079834, 0.5855245888233185, 0.25085602700710297, 0.11341731250286102, 0.3669561445713043, 0.10504548251628876, 0.3413199186325073, 0.23672470450401306, 0.17841559648513794, 0.5691604763269424, 0.2920585423707962, 0.3772938847541809, 0.38136157393455505, 0.14701053500175476, 0.392669677734375, 0.2377261519432068, 0.1746639907360077, 0.6166347563266754, 0.11354809999465942, 0.14319941401481628, 0.44793835282325745, 0.07952587306499481, 0.19467371702194214, 0.5792125165462494, 0.05408109724521637, 0.13006065785884857, 0.2395651936531067, 0.11779490113258362, 0.09789757430553436, 0.051783740520477295, 0.28552916646003723, 0.0796096920967102, 0.0833202600479126, 0.3851900100708008, 0.0961579978466034, 0.03206087648868561, 0.028052419424057007, 0.5646249651908875, 0.17997324466705322, 0.3660862147808075, 0.19734960794448853, 0.030909642577171326, 0.016133040189743042, 0.1448318362236023, 0.11261701583862305, 0.042196303606033325, 0.1324133723974228, 0.19000963866710663, 0.25983208417892456, 0.04385879635810852, 0.1031535267829895, 0.03695482015609741, 0.05342972278594971, 0.4080246686935425, 0.3021927773952484, 0.04437309503555298, 0.05066323280334473, 0.17366281151771545, 0.15449681878089905, 0.05681513249874115, 0.06325726211071014, 0.038454681634902954, 0.15499666333198547, 0.04881101846694946, 0.024817734956741333, 0.051630258560180664, 0.14481453597545624, 0.11985740065574646, 0.2966584265232086, 0.06039084494113922, 0.04573279619216919, 0.20378968119621277, 0.12797492742538452, 0.09175904095172882, 0.05598601698875427, 0.08079549670219421, 0.03553345799446106, 0.033286646008491516, 0.05458472669124603, 0.16628743708133698, 0.03199511766433716, 0.18607304990291595, 0.27172961831092834, 0.09508651494979858, 0.11123634874820709, 0.04352794587612152, 0.04170042276382446, 0.10227659344673157, 0.07696923613548279, 0.10184568166732788], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 211, \"name\": 211, \"data\": [1.207099199295044, 1.0263464748859406, 1.8090386986732483, 1.5807651281356812, 0.6342995762825012, 1.383101224899292, 1.0956274271011353, 1.1529648303985596, 1.0716173648834229, 1.0661195814609528, 0.7678272128105164, 1.3231483101844788, 0.9495944976806641, 0.8386580348014832, 0.6813420057296753, 0.5024419277906418, 0.5886348783969879, 0.38036707043647766, 0.6103556305170059, 0.60182985663414, 1.5504024624824524, 0.6854262351989746, 0.19679737091064453, 0.3746602535247803, 0.5381686836481094, 0.8567779958248138, 0.48697346448898315, 0.26442642509937286, 0.26024776697158813, 0.6619899421930313, 0.5416226536035538, 0.36130620539188385, 0.5106745958328247, 0.32002297043800354, 1.054868459701538, 0.791305661201477, 0.37175658345222473, 0.17955055832862854, 0.803871750831604, 0.25398364663124084, 0.8377174437046051, 0.6252793967723846, 0.4772126376628876, 0.4845774173736572, 0.907523512840271, 0.7239075005054474, 0.4977852702140808, 0.5491698384284973, 0.2932370901107788, 0.2841149568557739, 0.49456003308296204, 0.311021625995636, 0.4965025782585144, 0.6764017939567566, 0.9929115772247314, 0.4113955646753311, 0.2763214409351349, 0.8677517175674438, 0.1619885116815567, 0.4666275084018707, 0.669927567243576, 0.3325057029724121, 0.21032312512397766, 0.1860731840133667, 0.23549184203147888, 0.20058177411556244, 0.7852660417556763, 0.7121894061565399, 0.2598673552274704, 0.30145692825317383, 0.10999999940395355, 0.6782094836235046, 0.4197297692298889, 1.1080163717269897, 0.5307157039642334, 0.48738595843315125, 0.14782217144966125, 0.17886903882026672, 0.3445017337799072, 0.03347316384315491, 0.29550403356552124, 0.24045827984809875, 0.6812405586242676, 0.350517213344574, 0.1927112191915512, 0.46188995242118835, 0.16080503165721893, 0.4920458048582077, 0.37975627183914185, 0.012272357940673828, 0.12488003075122833, 0.06522920727729797, 0.004362523555755615, 0.030701518058776855, 0.27778109908103943, 0.013987243175506592, 0.24223485589027405, 0.08479046821594238, 0.04419949650764465, 0.006799280643463135, 0.2848580926656723, 0.009849607944488525, 0.13159997761249542, 0.12111543118953705, 0.04271934926509857, 0.1820693016052246, 0.13971039652824402, 0.4793063998222351, 0.16385236382484436, 0.01092827320098877, 0.012255728244781494, 0.7436906695365906, 0.06973463296890259, 0.3772439956665039, 0.1417551338672638, 0.04433333873748779, 0.04218912124633789, 0.03150118887424469, 0.12724974751472473, 0.8193812072277069, 0.038393884897232056, 0.02972474694252014, 0.13688147068023682, 0.01654067635536194, 0.038229554891586304, 0.09134191274642944, 0.6485102027654648, 0.05607369542121887, 0.029032588005065918, 0.5996507406234741, 0.14428076148033142, 0.03185632824897766, 0.06212225556373596, 0.1467515230178833, 0.05938123166561127, 0.485964760184288, 0.313005194067955, 0.03662411868572235, 0.0065871477127075195, 0.12775229662656784, 0.04079920053482056, 0.04807789623737335, 0.1280147284269333, 0.021676987409591675, 0.03908407688140869, 0.027439266443252563, 0.022215038537979126, 0.03491239249706268, 0.02011042833328247, 0.08241379261016846, 0.04734523594379425, 0.017768263816833496, 0.013108938932418823, 0.014655500650405884, 0.006133466958999634, 0.0031837522983551025, 0.017908602952957153, 0.017927706241607666, 0.012552350759506226, 0.001651674509048462, 0.0023603439331054688, 0.714072048664093, 0.01915159821510315, 0.002839028835296631, 0.00402107834815979, 0.0022275447845458984, 0.009754002094268799, 0.022870123386383057, 0.0699811577796936, 0.16381654143333435, 0.013494282960891724, 0.18040645122528076, 0.0016971826553344727, 0.11531519889831543, 0.07936406135559082, 0.4518592059612274, 0.004492402076721191, 0.26913002133369446, 0.402447909116745, 0.1431826949119568, 0.10359187424182892, 0.3705325126647949, 0.016865074634552002, 0.06184551119804382, 0.08996140956878662, 0.025776803493499756], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 210, \"name\": 210, \"data\": [1.0405151844024658, 1.373683214187622, 0.979673445224762, 1.1258878707885742, 1.0897021293640137, 1.2153218984603882, 1.0005279779434204, 1.1316267251968384, 0.7270680069923401, 1.0329248905181885, 1.236818790435791, 1.127394199371338, 0.5823133587837219, 0.8185867071151733, 1.0418375730514526, 0.9706516265869141, 0.9150351285934448, 1.0252113342285156, 0.7799272537231445, 1.1499274969100952, 1.0255893468856812, 0.24099880456924438, 2.297767162322998, 0.7449781894683838, 1.235079050064087, 0.550780713558197, 0.8519342541694641, 0.7743831872940063, 0.6152312755584717, 0.6015198230743408, 0.7665821313858032, 0.4342458248138428, 0.7970069050788879, 0.4425681233406067, 0.37296587228775024, 0.906125009059906, 0.5025632381439209, 0.8397800922393799, 0.05951845645904541, 1.0279793739318848, 1.6230418682098389, 1.4920649528503418, 0.6773638725280762, 0.6889174580574036, 0.1073114275932312, 0.2148209810256958, 0.15235772728919983, 0.6979225277900696, 0.42864924669265747, 1.0169310569763184, 1.1028186082839966, 0.2570655345916748, 0.6643994450569153, 0.44941475987434387, 0.2997783422470093, 0.30366250872612, 0.3251948058605194, 0.0735936164855957, 1.2183945178985596, 0.20213618874549866, 0.3140001595020294, 0.11742591857910156, 0.3269270658493042, 0.3834548890590668, 0.2411985993385315, 1.0433669090270996, 0.2365594506263733, 0.3602221608161926, 0.7810381650924683, 0.9877433180809021, 0.7873390913009644, 0.44867607951164246, 0.09179472923278809, 0.32704809308052063, 0.4154265522956848, 0.4997040629386902, 0.5105615258216858, 0.40853846073150635, 0.08605581521987915, 0.17669618129730225, 1.0622190237045288, 0.3234940767288208, 0.28261831402778625, 0.37038400769233704, 0.1795438528060913, 0.21167460083961487, 0.08682447671890259, 0.8315162062644958, 0.32766950130462646, 1.4107542037963867, 0.8294602036476135, 0.680728018283844, 0.37338870763778687, 0.41185063123703003, 1.1142702102661133, 0.24205893278121948, 0.5405521392822266, 0.26773977279663086, 0.8677463531494141, 0.08294481039047241, 0.07951313257217407, 0.2100839614868164, 0.11369562149047852, 1.3513257503509521, 0.5630148649215698, 0.06088283658027649, 1.4580135345458984, 0.6440107226371765, 0.20750603079795837, 1.0771859884262085, 0.40656977891921997, 1.0312232971191406, 0.409576952457428, 0.20280176401138306, 0.19329583644866943, 1.378666639328003, 0.1724490225315094, 0.06315436959266663, 0.05443507432937622, 0.12435445189476013, 1.0740680694580078, 0.43131256103515625, 0.5507505536079407, 0.4273108243942261, 0.0757138729095459, 0.32067734003067017, 0.4265100955963135, 0.2253551483154297, 0.26704537868499756, 0.18866726756095886, 0.2381274402141571, 0.19601893424987793, 0.3418092727661133, 0.5013071298599243, 0.5734323263168335, 0.8401823043823242, 0.03027021884918213, 0.07454657554626465, 0.8258798122406006, 0.07767128944396973, 0.09624594449996948, 0.7887842059135437, 0.6361621022224426, 0.118146151304245, 0.8129380345344543, 0.4266320765018463, 0.2752469778060913, 1.074006199836731, 0.333587110042572, 0.673205554485321, 0.5185655951499939, 0.11651971936225891, 0.10124415159225464, 0.023607850074768066, 0.06396597623825073, 0.09039899706840515, 0.6993541121482849, 0.3173747956752777, 0.034532010555267334, 0.049254000186920166, 0.02477318048477173, 0.018599987030029297, 0.30840861797332764, 0.012688040733337402, 0.09780478477478027, 2.1762642860412598, 0.17916208505630493, 0.10770311951637268, 0.19780832529067993, 0.009867250919342041, 0.10753381252288818, 0.020413756370544434, 0.17241746187210083, 0.04893249273300171, 0.0028210878372192383, 0.7265909314155579, 0.019424796104431152, 0.030996322631835938, 0.6552942395210266, 0.05541718006134033, 0.04057818651199341, 0.40573519468307495, 0.026632726192474365, 0.13637381792068481, 0.08607029914855957, 0.014218389987945557, 0.0049517154693603516, 0.2151913046836853, 0.1316128373146057, 0.5450919270515442, 0.06783965229988098, 0.02539956569671631, 0.08413040637969971, 0.35282355546951294, 0.27444151043891907, 0.01982855796813965, 0.10573965311050415, 0.03147745132446289, 0.008688509464263916, 0.009099185466766357, 0.0026701688766479492, 0.0157507061958313, 0.12156367301940918, 0.1417291760444641, 0.04601830244064331, 0.9720318913459778, 0.03177785873413086, 0.02515465021133423, 0.00816875696182251, 0.020640790462493896, 0.04182994365692139, 0.0025475621223449707, 1.6277488470077515, 0.15935403108596802, 0.3595021367073059, 0.7684608101844788, 0.2886982560157776, 0.09933167695999146, 0.8089697957038879, 0.3736796975135803, 0.13313904404640198, 0.07375755906105042, 0.024943828582763672, 0.03176969289779663, 0.05965590476989746, 0.1941748857498169, 0.06236708164215088, 0.13041278719902039, 0.23754006624221802, 0.009986162185668945, 0.005077481269836426, 0.4977152347564697, 0.14730864763259888, 0.019857287406921387, 1.6129816770553589, 0.013476550579071045, 0.030226171016693115, 0.22349625825881958, 0.015357017517089844, 0.01206505298614502, 1.3059675693511963, 0.004984021186828613, 0.010656952857971191, 0.0017175674438476562, 0.06513985991477966, 0.6912176609039307, 0.2076231837272644, 0.005078256130218506, 0.011677563190460205, 0.003122568130493164, 0.01758265495300293, 0.026441097259521484, 0.09217256307601929, 0.020506441593170166, 0.05430185794830322, 0.004628777503967285, 0.07079175114631653, 0.14230787754058838, 0.019243717193603516, 0.0037419795989990234, 0.31399521231651306, 0.048774659633636475, 0.02113032341003418, 1.715087890625, 0.005375921726226807, 1.0560616254806519, 0.045245468616485596, 0.49956732988357544, 0.23772460222244263, 0.3046417236328125, 0.10858309268951416, 0.005433201789855957, 0.8839158415794373, 0.28269505500793457, 0.37254732847213745, 0.29778850078582764, 0.05277222394943237, 0.005518615245819092, 0.004730165004730225, 0.007464468479156494, 0.07028889656066895, 0.07719975709915161, 0.004517972469329834, 0.006406664848327637, 0.14090609550476074, 0.1934361457824707, 0.020431578159332275, 0.04138064384460449, 0.0017077922821044922, 0.022913455963134766, 0.6040070056915283, 0.05681157112121582, 0.44611477851867676, 0.12991279363632202, 0.00041615962982177734, 0.0022389888763427734, 0.006600022315979004, 0.01418226957321167, 0.0074724555015563965, 0.0004869699478149414, 0.03490126132965088, 0.39702117443084717, 0.008037865161895752, 0.037111639976501465, 0.0037859678268432617, 0.001408219337463379, 0.005641758441925049, 0.04068636894226074, 0.010076701641082764, 0.011813700199127197, 0.00257265567779541, 0.032351911067962646, 0.002041339874267578, 0.00764387845993042, 0.580643892288208, 0.015631139278411865, 0.06949776411056519, 0.017044544219970703, 0.008467912673950195, 0.017139971256256104, 0.044117271900177, 0.006580710411071777, 0.0018864870071411133, 0.015041530132293701, 0.1589137315750122, 0.0009393692016601562, 0.2313910722732544, 0.01076364517211914, 0.0943688154220581, 0.029828667640686035, 0.012482881546020508, 0.001178741455078125, 0.003969788551330566, 0.0009118318557739258, 0.00235903263092041, 0.0016374588012695312, 0.003833174705505371, 0.03415095806121826, 0.002250194549560547, 0.06135594844818115, 0.007357895374298096, 0.002758026123046875, 0.005179405212402344, 0.0008540153503417969, 0.018747270107269287, 0.0010848045349121094, 0.0005537271499633789, 0.0050203800201416016, 0.455505907535553, 0.0002454519271850586, 0.001233816146850586, 0.0071375370025634766, 0.009184002876281738, 0.0017571449279785156, 0.00280916690826416, 0.0005712509155273438, 0.001619577407836914, 0.015352964401245117, 0.006295442581176758, 0.1594923734664917, 0.0005807876586914062, 0.009740829467773438, 0.01715034246444702, 0.014011383056640625, 0.0006086826324462891, 0.2813991606235504, 0.06986761093139648, 0.0028275251388549805, 0.00035881996154785156, 0.0009722709655761719, 0.011863112449645996, 0.003966808319091797, 0.00340878963470459, 0.00686800479888916, 0.00040793418884277344, 0.105693519115448, 0.0699424147605896, 0.31142371892929077, 0.004536271095275879, 0.0039406418800354, 0.07905972003936768, 0.01830834150314331, 0.1687696874141693, 0.07436174154281616, 0.0025120973587036133, 1.2537245750427246, 0.006939411163330078, 0.0036923885345458984, 0.16096699237823486, 0.00612252950668335, 0.027309656143188477, 6.61611557006836e-05, 0.15742844343185425, 0.004034519195556641, 0.012604713439941406, 0.0007330179214477539, 0.0009870529174804688, 0.6986620426177979, 0.03558146953582764, 0.005818545818328857, 0.06460464000701904, 0.004112362861633301, 0.05772554874420166, 0.0003203153610229492, 0.0009604692459106445, 0.022460639476776123, 0.01984238624572754, 0.027287840843200684, 0.0010302066802978516, 0.02965247631072998, 0.057693660259246826, 0.006075501441955566, 0.0032671689987182617, 0.002011418342590332, 0.039842963218688965, 0.0018055438995361328, 0.005339503288269043, 0.005434751510620117, 0.0045815110206604, 0.033139050006866455, 0.02767390012741089, 0.01597827672958374, 0.007332026958465576, 0.0009344816207885742, 0.0002518892288208008, 0.001129150390625, 0.0004423856735229492, 0.000335693359375, 0.005174398422241211, 0.0005513429641723633, 0.00015664100646972656, 0.0021216869354248047, 0.0004731416702270508, 0.014123737812042236, 0.00044405460357666016, 0.00011873245239257812, 0.0006952285766601562, 0.0027834177017211914, 0.00928640365600586, 0.001431107521057129, 0.14350569248199463, 0.006192564964294434, 0.0026935338973999023, 0.01061093807220459, 0.09034746885299683, 0.4106842279434204, 0.021434009075164795], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 213, \"name\": 213, \"data\": [0.8094639480113983, 0.9810653030872345, 0.7552203983068466, 0.8719150573015213, 0.744130551815033, 0.882108747959137, 0.7557084262371063, 0.4109628349542618, 0.2489866465330124, 0.5646486431360245, 0.6597674190998077, 0.22661781311035156, 0.8161292970180511, 0.24024555087089539, 0.266513392329216, 0.274454802274704, 0.3339318186044693, 0.6868532299995422, 0.12222646176815033, 0.2738691568374634, 0.35077235102653503, 0.2910373657941818, 0.4188259094953537, 0.6988600790500641, 0.6884551793336868, 0.3764638304710388, 0.34391656517982483, 0.9646655321121216, 0.8527508676052094, 0.23922431468963623, 0.25957056879997253, 0.10737895965576172, 0.2788826674222946, 0.8179301917552948, 0.10498586297035217, 0.18742966651916504, 0.34495197236537933, 0.264954149723053, 0.20353229343891144, 0.05485627055168152, 0.11593246459960938, 0.11888280510902405, 0.09170198440551758, 0.4234006404876709, 0.1291562169790268, 0.18289950489997864, 0.21420085430145264, 0.10524246096611023, 0.1573319435119629, 0.030203700065612793, 0.1456737518310547, 0.2284594178199768, 0.20218975841999054, 0.29220861196517944, 0.7099863886833191, 0.06715002655982971, 0.07944303750991821, 0.030599623918533325, 0.23296672105789185, 0.09730702638626099, 0.0273798406124115, 0.13152649998664856, 0.01771596074104309, 0.07339203357696533, 0.31309792399406433, 0.026945292949676514, 0.10013926029205322, 0.0395846962928772, 0.01789635419845581, 0.12985005974769592, 0.004857897758483887, 0.0348685085773468, 0.031106382608413696, 0.021520495414733887, 0.055921852588653564, 0.0747767984867096, 0.024633288383483887, 0.011064648628234863, 0.08198952674865723, 0.04329767823219299, 0.01327022910118103, 0.014628887176513672, 0.08279615640640259, 0.027669638395309448, 0.07749548554420471, 0.263571172952652, 0.013979077339172363, 0.07024312019348145, 0.04163214564323425, 0.1197824776172638, 0.019527822732925415, 0.004195839166641235, 0.01755458116531372, 0.008606493473052979, 0.02281421422958374, 0.04750847816467285, 0.0051438212394714355, 0.01690766215324402, 0.027588337659835815, 0.01321682333946228, 0.009057015180587769, 0.003588378429412842, 0.013788342475891113, 0.02524733543395996, 0.0034517645835876465, 0.11525195837020874, 0.005742132663726807, 0.026340007781982422, 0.0120888352394104, 0.02279040217399597, 0.0062104761600494385, 0.005595475435256958, 0.009804368019104004, 0.016222268342971802, 0.008378654718399048, 0.006204426288604736, 0.017896831035614014, 0.0045393407344818115, 0.01666468381881714, 0.009828239679336548, 0.009173572063446045, 0.030927568674087524, 0.00260084867477417, 0.018574267625808716], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 205, \"name\": 205, \"data\": [0.8012077212333679, 0.5869491100311279, 0.581861674785614, 1.1267940998077393, 0.90498286485672, 1.099434494972229, 0.8531586527824402, 0.7776079773902893, 0.45184552669525146, 0.5309653878211975, 0.8497861623764038, 0.8393737077713013, 0.2510811686515808, 0.7094557285308838, 0.6794812679290771, 0.47180473804473877, 0.33275747299194336, 0.4091517925262451, 0.2752147912979126, 1.8520641326904297, 0.10732841491699219, 0.22792816162109375, 1.9682948589324951, 0.6577295064926147, 0.6999465823173523, 0.24948978424072266, 0.2202383279800415, 0.7437618374824524, 0.05235934257507324, 0.20752239227294922, 0.19328033924102783, 0.7550782561302185, 0.5071195363998413, 0.26524782180786133, 0.06539952754974365, 0.19502949714660645, 0.1029818058013916, 1.2173080444335938, 0.038608670234680176, 0.4347677230834961, 0.10955727100372314, 0.5868314504623413, 0.03309595584869385, 0.601861834526062, 0.006026029586791992, 0.39847755432128906, 0.4270699620246887, 0.19432270526885986, 0.12148308753967285, 0.057001709938049316, 2.178152322769165, 0.2283085584640503, 0.9001597166061401, 0.22769713401794434, 0.05405235290527344, 0.4011399745941162, 0.26129376888275146, 0.04479110240936279, 0.6776445508003235, 0.2868466377258301, 0.8366398811340332, 0.04249215126037598, 0.20262014865875244, 0.36678290367126465, 0.1939302682876587, 0.5412617325782776, 0.4874332547187805, 0.3434324264526367, 0.9304642677307129, 1.086838960647583, 0.49352550506591797, 0.17759108543395996, 0.09424483776092529, 0.23255133628845215, 0.09441184997558594, 0.1432664394378662, 0.3279920816421509, 0.4133594036102295, 0.031036972999572754, 0.6648389101028442, 0.7778161764144897, 0.09203064441680908, 0.08590030670166016, 0.7128570079803467, 0.08090317249298096, 0.14420008659362793, 0.2739851474761963, 1.0124595165252686, 0.2257295846939087, 0.49811941385269165, 0.7824342250823975, 0.4282885789871216, 0.13661819696426392, 1.0446794033050537, 0.9223181009292603, 0.23236584663391113, 1.3438833951950073, 0.05409514904022217, 0.401350736618042, 0.0759507417678833, 0.09260380268096924, 0.3375633955001831, 0.11311709880828857, 0.47292590141296387, 0.1784592866897583, 0.29251420497894287, 0.5919005870819092, 0.4180903434753418, 0.10405170917510986, 2.247572660446167, 0.4726071357727051, 0.2423456311225891, 0.14604723453521729, 0.19700050354003906, 0.07415342330932617, 2.518043041229248, 0.06191551685333252, 0.06007516384124756, 0.004823923110961914, 1.2643072605133057, 0.48786282539367676, 0.052797794342041016, 0.42744457721710205, 0.2744481563568115, 0.02694249153137207, 0.32860875129699707, 0.15962576866149902, 0.035695552825927734, 0.07363831996917725, 0.14493215084075928, 0.17336750030517578, 0.151513934135437, 0.136114239692688, 0.17121636867523193, 0.039584994316101074, 1.4606835842132568, 0.5636752843856812, 0.01215982437133789, 0.07009899616241455, 0.03968691825866699, 0.07534050941467285, 0.0381474494934082, 0.07909917831420898, 0.09310662746429443, 0.24168914556503296, 0.12053465843200684, 0.06465530395507812, 0.30847394466400146, 0.03683018684387207, 1.1384408473968506, 0.11411881446838379, 0.035062313079833984, 0.020912885665893555, 0.0030090808868408203, 0.08930540084838867, 0.062320709228515625, 0.32181400060653687, 0.04892587661743164, 0.003093719482421875, 0.0037450790405273438, 0.12578487396240234, 0.006364107131958008, 0.06933343410491943, 0.00466609001159668, 0.05164778232574463, 0.05141270160675049, 0.05313706398010254, 0.022034168243408203, 0.09959352016448975, 0.008486032485961914, 0.02254509925842285, 0.06039309501647949, 0.024498939514160156, 0.04646444320678711, 0.05539298057556152, 0.630863606929779, 0.13449764251708984, 0.020218610763549805, 0.06449353694915771, 0.003559589385986328, 0.0023908615112304688, 0.046856045722961426, 0.003503561019897461, 1.0174572467803955, 0.01147603988647461, 0.7766439914703369, 0.0024785995483398438, 0.19001424312591553, 0.17438411712646484, 0.06937992572784424, 0.05906403064727783, 0.18148064613342285, 0.02725505828857422, 0.17660319805145264, 0.11116790771484375, 0.015158653259277344, 0.04908585548400879, 0.010254144668579102, 0.1272197961807251, 0.0060427188873291016, 0.0066874027252197266, 0.14547467231750488, 0.11963653564453125, 0.03711199760437012, 0.01274871826171875, 0.2703526020050049, 0.008352518081665039, 0.01754903793334961, 0.23026734590530396, 0.013529777526855469, 0.004228949546813965, 0.032073259353637695, 0.4914517402648926, 0.028324604034423828, 0.0331493616104126, 0.5946387052536011, 0.22642922401428223, 0.10260820388793945, 0.14714491367340088, 0.04459238052368164, 0.006387233734130859, 0.06729364395141602, 0.0015223026275634766, 0.02886784076690674, 0.004896879196166992, 2.06254243850708, 0.04443943500518799, 0.03711831569671631, 0.10229814052581787, 0.0072498321533203125, 0.0013854503631591797, 0.05491042137145996, 0.01819133758544922, 0.019642353057861328, 0.7193818092346191, 0.0017964839935302734, 0.03413045406341553, 0.02778470516204834, 0.022459030151367188, 0.012372016906738281, 0.2802000045776367, 0.03543496131896973, 0.007131218910217285, 0.0005338191986083984, 0.061232566833496094, 0.04725837707519531, 0.028888463973999023, 0.018807411193847656, 0.0002510547637939453, 0.007572650909423828, 0.0007498264312744141, 0.0018165111541748047, 0.008250236511230469, 0.00714719295501709, 0.012459874153137207, 0.02725803852081299, 0.03545498847961426, 0.011337041854858398, 0.05662035942077637, 0.005193471908569336, 0.8072113394737244, 0.008547544479370117, 0.0016598701477050781, 0.006647706031799316, 0.0040130615234375, 0.005676746368408203, 0.0035550594329833984, 0.009571433067321777, 0.001741170883178711, 0.05952620506286621, 0.0073621273040771484, 0.001092672348022461, 0.07637476921081543, 0.013110160827636719, 0.133774995803833, 0.03372526168823242, 0.007843613624572754, 0.003893136978149414, 0.001432180404663086, 0.012694835662841797, 0.022143840789794922, 0.37509047985076904, 0.007977962493896484, 0.0004687309265136719, 0.005563020706176758, 0.0026912689208984375, 0.0005865097045898438, 0.0303802490234375, 0.00093841552734375, 0.009716033935546875, 0.008141756057739258, 0.09738755226135254, 0.0020058155059814453, 0.0220717191696167, 0.00048804283142089844, 0.03657972812652588, 0.009219765663146973, 0.005544185638427734, 0.005316019058227539, 0.003953456878662109, 0.3633265495300293, 0.15176498889923096, 0.012009501457214355, 0.050130486488342285, 0.0009226799011230469, 0.0005214214324951172, 0.008069276809692383, 0.0011522769927978516, 0.02350473403930664, 0.0055065155029296875, 0.003999233245849609, 0.0009133815765380859, 0.010126352310180664, 0.21076107025146484, 0.004616260528564453, 0.06749987602233887, 0.004244804382324219, 0.0009706020355224609, 0.011074304580688477, 0.012363910675048828, 0.01423025131225586, 0.005544185638427734, 0.019519567489624023, 0.036192893981933594, 0.01985466480255127, 0.001829385757446289, 0.011263608932495117, 0.09773588180541992, 0.002388477325439453, 0.00597071647644043, 0.3222081661224365, 0.0021300315856933594, 0.0028237104415893555, 0.002490997314453125, 0.005454063415527344, 0.0018811225891113281, 0.08198368549346924, 0.047390103340148926, 0.008980989456176758, 0.013819575309753418, 0.007472872734069824, 0.043885111808776855, 0.0068302154541015625, 0.011502265930175781, 0.0010759830474853516, 0.00432896614074707, 0.003068208694458008, 0.003885507583618164, 0.06355488300323486, 0.005827426910400391, 0.013962507247924805, 0.03450596332550049, 0.07494246959686279, 0.11316883563995361, 0.010587573051452637, 0.0005331039428710938, 0.03921306133270264, 0.02074742317199707, 0.0033507347106933594, 0.02588522434234619, 0.0022008419036865234, 0.0021828413009643555, 0.002520322799682617, 0.015223503112792969, 0.0017359256744384766, 0.15570533275604248, 0.005630970001220703, 0.0015882253646850586, 0.0008065700531005859, 0.006344795227050781, 0.033373117446899414, 0.005066871643066406, 0.002699136734008789, 0.0011949539184570312, 0.0006804466247558594, 0.011896848678588867, 0.004492282867431641, 0.003314971923828125, 0.0022068023681640625, 0.001988053321838379, 0.016197562217712402, 0.0032300949096679688, 0.038407325744628906, 0.00044608116149902344, 0.0008971691131591797, 0.28698277473449707, 0.014969944953918457, 0.0003197193145751953, 0.00549626350402832, 0.0010448694229125977, 0.005747199058532715, 0.0008771419525146484, 0.005522251129150391, 0.0009691715240478516, 0.0006744861602783203, 0.0009453296661376953, 0.0004127025604248047, 0.03444993495941162, 0.002109050750732422, 0.0025783777236938477, 0.004853963851928711, 0.002385258674621582, 0.001913309097290039, 0.0009212493896484375, 0.007547855377197266, 0.004168033599853516, 0.008817076683044434, 0.0012993812561035156, 0.00046539306640625, 0.009020566940307617, 0.0017552375793457031, 0.0037665367126464844, 0.0007767677307128906, 0.0007462501525878906, 0.0022257566452026367, 0.00055694580078125, 0.010314226150512695, 0.0010769367218017578, 0.007122397422790527, 0.001069784164428711, 0.04855632781982422, 0.002668619155883789, 0.002657175064086914, 0.14799284934997559, 0.0006122589111328125, 0.052940964698791504, 0.03858041763305664, 0.0021965503692626953, 0.0012934207916259766, 0.0007584095001220703, 0.0005207061767578125, 0.11571478843688965, 0.001972198486328125, 0.0027463436126708984, 0.0026247501373291016, 0.003764629364013672, 0.00028133392333984375, 0.0019159317016601562, 0.0001480579376220703, 0.010815143585205078, 0.035874247550964355, 0.0032470226287841797, 0.004235982894897461, 0.009146332740783691, 0.01220703125, 0.0008270740509033203, 0.002002716064453125, 0.0010936260223388672, 0.0006191730499267578, 0.0007815361022949219, 0.0009186267852783203, 0.021897315979003906, 0.001262664794921875, 0.001542806625366211, 0.002572298049926758, 0.0020503997802734375, 0.0007662773132324219, 0.001246333122253418, 0.0011467933654785156, 0.01092672348022461, 0.0009375810623168945, 0.002432107925415039, 0.006174325942993164, 0.015126943588256836, 0.005005240440368652, 0.0087052583694458, 0.0016400814056396484, 0.0018661022186279297, 0.0013363361358642578, 0.002185821533203125, 0.0010945796966552734, 0.002627849578857422, 0.0005514621734619141, 0.01321876049041748, 0.0008788108825683594, 0.0055272579193115234, 0.0018877983093261719, 0.0015192031860351562, 0.013208627700805664, 0.0043152570724487305, 0.00018072128295898438, 0.08518540859222412, 0.006315708160400391, 0.00033974647521972656, 0.07079625129699707, 0.005160808563232422, 0.013088703155517578, 0.0008081197738647461, 0.02267169952392578, 0.0014400482177734375, 0.0003592967987060547, 0.00033473968505859375, 0.0006684064865112305, 0.0010302066802978516, 0.010445833206176758, 0.007703304290771484, 0.02276158332824707, 0.0007185935974121094, 0.0010426044464111328, 0.002658843994140625], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 212, \"name\": 212, \"data\": [0.8094639480113983, 0.9126565456390381, 0.6429198086261749, 0.6516834050416946, 0.6233788728713989, 0.881320595741272, 0.6068270802497864, 0.21998381614685059, 0.14564082026481628, 0.4088059663772583, 0.4603027105331421, 0.20536351203918457, 0.7136300504207611, 0.22777599096298218, 0.3198719620704651, 0.3845759928226471, 0.39010512828826904, 0.7528856694698334, 0.16340455412864685, 0.3061390817165375, 0.4481566250324249, 0.3147653341293335, 0.43929536640644073, 0.7799201160669327, 0.5506484508514404, 0.5467658340930939, 0.4136907607316971, 0.8120139241218567, 0.7468706965446472, 0.36520448327064514, 0.35949641466140747, 0.1804918646812439, 0.2824704945087433, 0.7446352243423462, 0.09811621904373169, 0.16409993171691895, 0.42495693266391754, 0.18306264281272888, 0.9731876701116562, 0.2094326615333557, 0.06404808163642883, 0.1659260392189026, 0.04403001070022583, 0.167930468916893, 0.0751207023859024, 0.09550490975379944, 0.5259103775024414, 0.1069628894329071, 0.09051331877708435, 0.04699075222015381, 0.14184415340423584, 0.3429439067840576, 0.15158048272132874, 0.19441251456737518, 0.3739551305770874, 0.05924329161643982, 0.07812783122062683, 0.16915464401245117, 0.2878698706626892, 0.18085217475891113, 0.017655164003372192, 0.21834266185760498, 0.06110185384750366, 0.03671663999557495, 0.29722678661346436, 0.03740525245666504, 0.018640726804733276, 0.6472221612930298, 0.03205841779708862, 0.2324756234884262, 0.007174253463745117, 0.044327765703201294, 0.01222074031829834, 0.09485352039337158, 0.1662479043006897, 0.08347368240356445, 0.06448549032211304, 0.144476979970932, 0.05413702130317688, 0.07379859685897827, 0.023396819829940796, 0.1868038773536682, 0.1378779411315918, 0.050921231508255005, 0.07753834128379822, 0.20422610640525818, 0.02010244131088257, 0.058251917362213135, 0.041722118854522705, 0.051125138998031616, 0.021448403596878052, 0.0033555924892425537, 0.019005566835403442, 0.00940239429473877, 0.012301534414291382, 0.08853012323379517, 0.0030960142612457275, 0.004408687353134155, 0.014285117387771606, 0.015366554260253906, 0.15813472867012024, 0.00594407320022583, 0.018045485019683838, 0.024795711040496826, 0.006901443004608154, 0.02044263482093811, 0.000925898551940918, 0.0104103684425354, 0.00405353307723999, 0.0602547824382782, 0.007723122835159302, 0.0024369359016418457, 0.01914316415786743, 0.12210360169410706, 0.020603567361831665, 0.0649455189704895, 0.008559167385101318, 0.010557681322097778, 0.04273638129234314, 0.01225540041923523, 0.051272034645080566, 0.003773331642150879, 0.0016079843044281006, 0.005182147026062012], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 217, \"name\": 217, \"data\": [0.8200175762176514, 1.0017643570899963, 0.6381671875715256, 0.9171938300132751, 0.7628255188465118, 0.8231825232505798, 0.6716506779193878, 0.3539079874753952, 0.19518768787384033, 0.3723178803920746, 0.6791934967041016, 0.26234692335128784, 0.6912363767623901, 0.2915211468935013, 0.35746346414089203, 0.2946307361125946, 0.44793348014354706, 0.6622220873832703, 0.14406418800354004, 0.4183736890554428, 0.40756915509700775, 0.5836390405893326, 0.600903183221817, 0.649418979883194, 0.41939887404441833, 0.7178955674171448, 0.41629597544670105, 0.49021781980991364, 0.6246958673000336, 0.34302137792110443, 0.5937387943267822, 0.15542437136173248, 0.35577070713043213, 0.3968183398246765, 0.35094551742076874, 0.33321893215179443, 0.36965471506118774, 0.26878567039966583, 0.20594121515750885, 0.06600546836853027, 0.173121377825737, 0.08969645202159882, 0.23060031235218048, 0.2371395081281662, 0.13958841562271118, 0.3694293349981308, 0.17272284626960754, 0.15139319002628326, 0.15788552165031433, 0.02683502435684204, 0.19598513841629028, 0.13569819927215576, 0.3016887307167053, 0.3315921574831009, 0.47994743287563324, 0.3038395196199417, 0.16120345890522003, 0.12179262936115265, 0.2653280049562454, 0.12001192569732666, 0.04987324774265289, 0.1611357033252716, 0.06263785064220428, 0.22853894531726837, 0.0886533260345459, 0.042950063943862915, 0.3517131209373474, 0.0807623565196991, 0.07631111145019531, 0.0871364027261734, 0.01773563027381897, 0.05192609131336212, 0.14408253133296967, 0.042911261320114136, 0.0959513932466507, 0.03924286365509033, 0.12511996924877167, 0.009718924760818481, 0.11219318211078644, 0.07506366074085236, 0.04048249125480652, 0.06367585062980652, 0.08038921654224396, 0.061689674854278564, 0.15030130743980408, 0.23321130871772766, 0.04371090233325958, 0.11466975510120392, 0.07774312794208527, 0.10067048668861389, 0.030132919549942017, 0.02782060205936432, 0.026948481798171997, 0.018096059560775757, 0.04320070147514343, 0.025453045964241028, 0.0377793163061142, 0.0657287985086441, 0.026103541254997253, 0.02757956087589264, 0.017420083284378052, 0.021704822778701782, 0.035749003291130066, 0.021690726280212402, 0.06305328011512756, 0.025527000427246094, 0.03052958846092224, 0.063677579164505, 0.008158117532730103, 0.007099956274032593, 0.02642485499382019, 0.06065736711025238, 0.04582644999027252, 0.07072386145591736, 0.025067567825317383, 0.04182027280330658, 0.016133785247802734, 0.008869528770446777, 0.027857527136802673, 0.02762541174888611, 0.07180671393871307, 0.015664413571357727, 0.004685074090957642, 0.017842859029769897], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 218, \"name\": 218, \"data\": [0.8399580717086792, 0.9916065335273743, 1.4175128936767578, 1.0636235475540161, 0.9613556265830994, 1.1918710470199585, 1.0604413747787476, 1.1403954029083252, 0.9549961090087891, 0.725087583065033, 0.736125648021698, 1.2235194444656372, 0.876407265663147, 0.9673491716384888, 0.48132798075675964, 0.4110943675041199, 0.5538128614425659, 0.4454523026943207, 0.744128406047821, 0.7444812059402466, 1.2245246171951294, 0.849579393863678, 0.26676854491233826, 0.5849162936210632, 0.35835525393486023, 1.0414844751358032, 0.5344597101211548, 0.29936572909355164, 0.42912980914115906, 0.7724164724349976, 0.30714231729507446, 0.3448357582092285, 0.7026803493499756, 0.47271624207496643, 1.0149697065353394, 0.5272883176803589, 0.23054084181785583, 0.2968211770057678, 0.5079448819160461, 0.31980985403060913, 0.4932119846343994, 0.6243999004364014, 0.43910613656044006, 0.39752060174942017, 0.9856051206588745, 0.6782869100570679, 0.5611319541931152, 0.867904782295227, 0.2674461603164673, 0.2743079364299774, 0.6206765174865723, 0.3703395128250122, 0.6062641143798828, 0.9104806184768677, 0.7290499806404114, 0.34279751777648926, 0.6264803409576416, 0.674223780632019, 0.2617591619491577, 0.2532656788825989, 1.26747465133667, 0.17808228731155396, 0.028220534324645996, 0.26460033655166626, 0.2167951464653015, 0.5210871696472168, 0.41760697960853577, 0.7204867005348206, 0.34261354804039, 0.2498725801706314, 0.2762455344200134, 0.11690680682659149, 0.635435938835144, 0.14203813672065735, 0.9077138304710388, 0.41841548681259155, 0.1806553602218628, 0.049917370080947876, 0.294511079788208, 0.12716862559318542, 0.07993750274181366, 0.38338935375213623, 0.16672798991203308, 0.40215516090393066, 0.14480724930763245, 0.18224018812179565, 0.08514206111431122, 0.09733015298843384, 0.33482110500335693, 0.40009644627571106, 0.04052077233791351, 0.05689100921154022, 0.23665092885494232, 0.03957068920135498, 0.0284634530544281, 0.28549718856811523, 0.060648560523986816, 0.22053276002407074, 0.11523737013339996, 0.10721474885940552, 0.01538550853729248, 0.324134886264801, 0.16115698218345642, 0.2924528121948242, 0.08964180946350098, 0.034935593605041504, 0.07779082655906677, 0.08609127998352051, 0.20037490129470825, 0.11417992413043976, 1.2082220315933228, 0.04988813400268555, 0.022791504859924316, 0.059750109910964966, 0.6198655962944031, 0.13846701383590698, 0.15290285646915436, 0.17308790981769562, 0.06101861596107483, 0.12962117791175842, 0.6141234636306763, 0.07114693522453308, 0.06706503033638, 0.04513886570930481, 0.1482016146183014, 0.0012985467910766602, 0.06849169731140137, 0.09088301658630371, 0.40578699111938477, 0.09333285689353943, 0.16706201434135437, 0.09336614608764648, 0.020726561546325684, 0.04241505265235901, 0.014865279197692871, 0.050591856241226196, 0.06736162304878235, 0.08489924669265747, 0.22185887396335602, 0.029313355684280396, 0.06181183457374573, 0.040380269289016724, 0.05388283729553223, 0.006890624761581421, 0.005527287721633911, 0.07327829301357269, 0.047707781195640564, 0.0843624472618103, 0.004747241735458374, 0.03345364332199097, 0.26961904764175415, 0.005640625953674316, 0.02756279706954956, 0.0070534348487854, 0.0030216872692108154, 0.08960999548435211, 0.003844141960144043, 0.08554334938526154, 0.021598339080810547, 0.011896878480911255, 0.015465766191482544, 0.0077912211418151855, 0.013937383890151978, 0.011177808046340942, 0.01803910732269287, 0.009573876857757568, 0.11777263879776001, 0.008852899074554443, 0.019320815801620483, 0.16693280637264252, 0.016130119562149048, 0.010809898376464844, 0.020059093832969666, 0.09515109658241272, 0.005995273590087891, 0.005368590354919434, 0.06696018576622009, 0.08924999833106995, 0.04017254710197449, 0.027060076594352722, 0.12856696546077728, 0.07223570346832275, 0.0034203827381134033, 0.0062462687492370605, 0.007874399423599243, 0.006882131099700928, 0.15340842306613922, 0.003193795680999756, 0.0021758675575256348, 0.020500198006629944, 0.009779423475265503, 0.0027822256088256836, 0.00733870267868042, 0.006439626216888428, 0.006876587867736816, 0.0036183297634124756, 0.03110596537590027, 0.00958395004272461, 0.05420789122581482, 0.018837183713912964, 0.08025220036506653, 0.14483517408370972, 0.013545215129852295, 0.004232317209243774, 0.024224579334259033, 0.024408936500549316, 0.00192338228225708, 0.004828423261642456, 0.014110773801803589, 0.005128741264343262, 0.0059278905391693115, 0.002387911081314087, 0.028842493891716003, 0.0022236108779907227, 0.0028216540813446045, 0.02809073030948639, 0.03868137300014496, 0.004596740007400513, 0.006588876247406006, 0.0029838979244232178, 0.014349311590194702, 0.017847955226898193, 0.0025753378868103027, 0.024779796600341797, 0.00148811936378479, 0.0005202293395996094, 0.003307133913040161, 0.04505652189254761, 0.002137303352355957, 0.05172143876552582, 0.031482994556427, 0.005791604518890381, 0.0026205778121948242, 0.012535035610198975, 0.00344046950340271, 0.016552388668060303, 0.0054070353507995605, 0.006060242652893066, 0.0026643574237823486, 0.0063751935958862305, 0.004251241683959961, 0.0018023848533630371, 0.003977596759796143, 0.18567946553230286, 0.011768221855163574, 0.07599848508834839, 0.004569053649902344, 0.001565694808959961, 0.0019463896751403809, 0.00969851016998291, 0.0015229582786560059, 0.000335693359375], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 215, \"name\": 215, \"data\": [0.8012077212333679, 0.5869491100311279, 0.5824516415596008, 0.9857093095779419, 0.678602933883667, 0.6892863512039185, 0.38496315479278564, 0.3868105411529541, 0.26460587978363037, 0.09997010231018066, 2.2060112953186035, 1.3329787254333496, 0.040165066719055176, 0.5519577264785767, 0.18785905838012695, 0.11606860160827637, 0.5425724983215332, 0.19836997985839844, 0.2703937292098999, 1.5000979900360107, 0.1428813934326172, 0.6039927005767822, 2.639204740524292, 0.2598748207092285, 1.1701579093933105, 0.1165010929107666, 0.22536420822143555, 1.0558420419692993, 0.16402602195739746, 0.36224842071533203, 0.2888965606689453, 0.9454237222671509, 0.14990901947021484, 0.05168557167053223, 0.03834354877471924, 0.17737269401550293, 0.14500212669372559, 0.8975567817687988, 0.010392189025878906, 2.076021909713745, 0.7954922914505005, 1.3824446201324463, 0.2977752685546875, 0.9928840398788452, 0.03745841979980469, 0.5870462656021118, 0.3603968620300293, 0.3491619825363159, 0.37886369228363037, 0.3589906692504883, 1.8070870637893677, 0.36322879791259766, 0.31677186489105225, 0.2979590892791748, 0.3158450126647949, 0.04283714294433594, 0.1276094913482666, 0.06971883773803711, 0.7686117887496948, 0.36991405487060547, 0.282320499420166, 0.10536849498748779, 0.3544846773147583, 0.5724035501480103, 0.41109228134155273, 0.2492302656173706, 0.20508861541748047, 0.16459345817565918, 1.0185942649841309, 1.8093476295471191, 0.3884035348892212, 0.14821743965148926, 0.036438584327697754, 0.07422447204589844, 0.4464881420135498, 0.0647268295288086, 0.28481388092041016, 0.5181005001068115, 0.0459979772567749, 0.1456359624862671, 0.6265596151351929, 0.08474087715148926, 0.20885765552520752, 0.18197786808013916, 0.32020068168640137, 0.31052684783935547, 0.06590449810028076, 0.9497421383857727, 0.6637787818908691, 6.763317108154297, 6.770305633544922, 0.5203171968460083, 6.247835159301758, 0.6396828889846802, 0.6834272146224976, 0.6065614223480225, 0.9620840549468994, 0.7064800262451172, 5.318598747253418, 0.6737473011016846, 0.6732864379882812, 3.7019429206848145, 2.759422779083252, 3.072895050048828, 1.3350696563720703, 2.009413003921509, 1.1054060459136963, 1.5050101280212402, 0.863961935043335, 1.3669865131378174, 0.6233057975769043, 3.2597055435180664, 1.474541425704956, 0.7665810585021973, 0.7410645484924316, 0.7914876937866211, 0.8635890483856201, 1.1687533855438232, 0.8363595008850098, 0.8987085819244385, 0.8566558361053467, 0.9310579299926758, 0.8498330116271973, 0.8807458877563477, 0.7536351680755615, 0.8434062004089355, 0.8415272235870361, 0.7744240760803223, 1.3134260177612305, 1.2613461017608643, 1.1637063026428223, 1.1075234413146973, 0.8814728260040283, 0.8406260013580322, 0.8207802772521973, 0.8937473297119141, 1.1362903118133545, 0.9259662628173828, 1.0726399421691895, 1.1746325492858887, 1.1112070083618164, 0.7720842361450195, 1.2066402435302734, 0.8454470634460449, 0.8170127868652344, 0.7235445976257324, 0.7694666385650635, 2.1446473598480225, 0.7816295623779297, 1.2832059860229492, 0.7997646331787109, 1.1670198440551758, 0.8052794933319092, 0.7857930660247803, 0.834460973739624, 0.9452338218688965, 0.8540608882904053, 0.9172954559326172, 0.8334968090057373, 0.9081017971038818, 0.8902053833007812, 0.7663707733154297, 0.7311046123504639, 0.7345240116119385, 1.1225714683532715, 0.8623878955841064, 1.971611499786377, 0.6264212131500244, 1.3377726078033447, 0.7886950969696045, 0.7038006782531738, 0.8238465785980225, 0.678372859954834, 0.7078032493591309, 0.6933193206787109, 0.7587316036224365, 0.742159366607666, 0.8596692085266113, 0.7818238735198975, 0.9244697093963623, 0.855492115020752, 0.8054563999176025, 0.9565517902374268, 1.2502033710479736, 0.8662986755371094, 0.7259690761566162, 0.7184998989105225, 0.8002557754516602, 0.8117763996124268, 0.7496881484985352, 1.5946810245513916, 0.9178307056427002, 0.7758667469024658, 1.1066405773162842, 1.0988621711730957, 0.8590736389160156, 0.8606276512145996, 0.8561172485351562, 0.9158196449279785, 0.8976600170135498, 0.8877546787261963, 0.9203364849090576, 0.9039239883422852, 0.9003591537475586, 0.7500219345092773, 0.7380883693695068, 0.7329978942871094, 1.413865327835083, 0.6907145977020264, 0.6934769153594971, 1.405451774597168, 0.6936309337615967, 1.6378202438354492, 0.6544358730316162, 1.4855566024780273, 1.5073940753936768, 0.7646565437316895, 1.5214858055114746, 0.7828719615936279, 0.7815313339233398, 0.7657878398895264, 0.7698829174041748, 0.7214887142181396, 1.1441280841827393, 0.8793909549713135, 0.8016080856323242, 1.1033668518066406, 1.1944246292114258, 1.2145044803619385, 1.0520179271697998, 0.9499619007110596, 1.1086130142211914, 0.9318149089813232, 0.8404302597045898, 1.0944852828979492, 0.8427395820617676, 1.9282400608062744, 1.1076610088348389, 0.7596163749694824, 0.7847023010253906, 0.7276532649993896, 0.748910665512085, 1.1791422367095947, 0.7741036415100098, 0.702695369720459, 0.7761592864990234, 0.7806744575500488, 0.8309531211853027, 0.74723219871521, 0.8665473461151123, 0.744877815246582, 0.8466310501098633, 0.7599773406982422, 1.2040979862213135, 1.0211725234985352, 1.067950963973999, 1.1198854446411133, 0.7963440418243408, 0.8773996829986572, 0.9218707084655762, 0.7891404628753662, 1.251924753189087, 0.843909740447998, 1.0080654621124268, 1.0733318328857422, 1.193105936050415, 0.7297687530517578, 1.2092905044555664, 0.8335731029510498, 0.848876953125, 0.7806916236877441, 0.7290124893188477, 2.0911009311676025, 0.7911596298217773, 1.1224584579467773, 0.8068358898162842, 1.1642158031463623, 0.7564506530761719, 0.8054461479187012, 0.8146283626556396, 0.9391012191772461, 0.8518836498260498, 0.8670377731323242, 0.9736642837524414, 0.8084611892700195, 0.8374729156494141, 0.8217291831970215, 0.8421123027801514, 0.8352351188659668, 1.2091927528381348, 0.7232306003570557, 1.881533145904541, 0.8597636222839355, 1.288231611251831, 0.7863459587097168, 0.7522990703582764, 0.7139091491699219, 0.7794702053070068, 0.8363826274871826, 0.7241621017456055, 0.7806084156036377, 0.8083949089050293, 0.8597331047058105, 0.841799259185791, 0.848273515701294, 0.8286421298980713, 0.8268883228302002, 0.7694053649902344, 1.3088443279266357, 0.8133058547973633, 0.8747739791870117, 0.7953395843505859, 0.7870914936065674, 0.6335771083831787, 0.817018985748291, 1.6010291576385498, 0.7712867259979248, 0.7791550159454346, 1.046419620513916, 1.1483325958251953, 0.8623270988464355, 0.859121561050415, 0.7745790481567383, 0.9010229110717773, 0.8569352626800537, 0.8223133087158203, 0.7786071300506592, 0.8620877265930176, 0.8575217723846436, 0.7664535045623779, 0.7772040367126465, 1.2668120861053467, 0.8221635818481445, 0.7923991680145264, 0.7721383571624756, 1.3267841339111328, 0.746199369430542, 1.5194995403289795, 0.7047593593597412, 1.4519855976104736, 1.476515769958496, 0.8076560497283936, 1.506232738494873, 0.7592759132385254, 0.6459846496582031, 0.7296223640441895, 0.7347476482391357, 0.6857059001922607, 1.2597181797027588, 0.8190250396728516, 0.7350006103515625, 1.1421046257019043, 1.0977299213409424, 1.282447338104248, 1.0053796768188477, 1.0861623287200928, 1.044358491897583, 0.9337196350097656, 0.9232025146484375, 1.1057744026184082, 0.8446271419525146, 1.855642318725586, 1.2221660614013672, 0.7660849094390869, 0.7885487079620361, 0.7088532447814941, 0.8633148670196533, 1.2038536071777344, 0.8358578681945801], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 221, \"name\": 221, \"data\": [0.8399580717086792, 0.9916065335273743, 1.1599897146224976, 1.0813463926315308, 0.9602495431900024, 0.9735352993011475, 0.9350212812423706, 1.2453758716583252, 1.1204485893249512, 0.6673659086227417, 0.6231060028076172, 1.3341259956359863, 0.8222852349281311, 1.0323097705841064, 0.4724958539009094, 0.4764856696128845, 0.6918522119522095, 0.47055840492248535, 0.7072690725326538, 0.7677833437919617, 1.1569660902023315, 0.9090896844863892, 0.3350810110569, 0.5820011496543884, 0.4316646456718445, 1.0033740997314453, 0.5730640888214111, 0.37724947929382324, 0.48650842905044556, 0.6788156628608704, 0.2894464135169983, 0.3724270164966583, 0.5886635780334473, 0.3468519151210785, 0.4918237626552582, 0.7269100546836853, 0.8736051321029663, 0.5099080204963684, 1.1501556634902954, 0.7050440311431885, 0.553488552570343, 0.7339110374450684, 0.6064735651016235, 0.40434032678604126, 0.7542861700057983, 0.7008779048919678, 0.7669856548309326, 0.7279561758041382, 0.25269991159439087, 0.4563411474227905, 0.6676813960075378, 0.7984716296195984, 0.60671067237854, 0.9825689792633057, 0.5008851289749146, 0.3693605661392212, 0.5368911623954773, 1.1110548973083496, 0.28798243403434753, 0.4365603029727936, 1.293013095855713, 0.32968512177467346, 0.051145583391189575, 0.3771490156650543, 0.2637901306152344, 0.26942941546440125, 0.5188719034194946, 0.34383437037467957, 0.3866937458515167, 0.1899767369031906, 0.4495307505130768, 0.099693164229393, 0.5377290844917297, 0.23163363337516785, 0.7063885927200317, 0.6077350378036499, 0.2007254660129547, 0.09107053279876709, 0.29460641741752625, 0.21204528212547302, 0.08333966135978699, 0.3451709449291229, 0.3344281017780304, 0.7056482434272766, 0.24007192254066467, 0.11521677672863007, 0.2513827085494995, 0.21365807950496674, 0.31216225028038025, 0.31314337253570557, 0.10135892033576965, 0.09457400441169739, 0.3289867639541626, 0.13349688053131104, 0.3353445529937744, 0.24524027109146118, 0.09550723433494568, 0.4229097366333008, 0.14186537265777588, 0.03914979100227356, 0.042978376150131226, 0.2802577316761017, 0.1673506200313568, 0.22151441872119904, 0.18661248683929443, 0.07534739375114441, 0.22319766879081726, 0.2757635712623596, 0.2554405927658081, 0.22191175818443298, 0.22820347547531128, 0.10020236670970917, 0.05854681134223938, 0.09766183793544769, 0.2820202708244324, 0.07031922042369843, 0.23217174410820007, 0.17546412348747253, 0.05518120527267456, 0.044718459248542786, 0.47005367279052734, 0.03378187119960785, 0.13639375567436218, 0.15986420214176178, 0.07895958423614502, 0.005891621112823486, 0.03046785295009613, 0.016994088888168335, 0.05310973525047302, 0.09168994426727295, 0.11935541033744812, 0.06506718695163727, 0.04189552366733551, 0.0401759147644043, 0.023617476224899292, 0.27803292870521545, 0.09516163170337677, 0.14594843983650208, 0.06808847188949585, 0.02002069354057312, 0.014811009168624878, 0.09503169357776642, 0.09235955774784088, 0.5323507785797119, 0.027055591344833374, 0.07042625546455383, 0.10962502658367157, 0.07365049421787262, 0.28499045968055725, 0.04959782958030701, 0.012958228588104248, 0.44573816657066345, 0.14941063523292542, 0.01272401213645935, 0.015480279922485352, 0.10803952813148499, 0.008253723382949829, 0.040792316198349, 0.0585501492023468, 0.09674191474914551, 0.0379595011472702, 0.019021272659301758, 0.015362530946731567, 0.01860177516937256, 0.0363692045211792, 0.011527448892593384, 0.07616224884986877, 0.021462738513946533, 0.02064913511276245, 0.030787885189056396, 0.05900371074676514, 0.02900826930999756, 0.11365699768066406, 0.030016034841537476, 0.010333448648452759, 0.01111719012260437, 0.04104119539260864, 0.052525103092193604, 0.0395711213350296, 0.02403673529624939, 0.061678096652030945, 0.023459702730178833, 0.040575504302978516, 0.013776212930679321, 0.01261061429977417, 0.011141687631607056, 0.053086549043655396, 0.015898048877716064, 0.002438068389892578, 0.027151063084602356, 0.018114209175109863, 0.006954371929168701, 0.02344384789466858, 0.07112400233745575, 0.014698565006256104, 0.00380551815032959, 0.023639261722564697, 0.010257214307785034, 0.019593507051467896, 0.02407941222190857, 0.05502788722515106, 0.06914415955543518, 0.024976909160614014, 0.0035590529441833496, 0.01219785213470459, 0.005430012941360474, 0.003373444080352783, 0.028009355068206787, 0.04325540363788605, 0.019679248332977295, 0.013523757457733154, 0.0022482872009277344, 0.03793364763259888, 0.004500210285186768, 0.01115986704826355, 0.03572091460227966, 0.12434214353561401, 0.007499098777770996, 0.017803072929382324, 0.010553032159805298, 0.008888483047485352, 0.02399882674217224, 0.0036736726760864258, 0.06722897291183472, 0.003093898296356201, 0.0018777251243591309, 0.007754325866699219, 0.03371673822402954, 0.00333482027053833, 0.05006423592567444, 0.02703593671321869, 0.00941208004951477, 0.012107819318771362, 0.031196296215057373, 0.011895835399627686, 0.007837653160095215, 0.0025635957717895508, 0.0086326003074646, 0.004457443952560425, 0.01899796724319458, 0.016802668571472168, 0.006919920444488525, 0.008335649967193604, 0.4156095087528229, 0.012630760669708252, 0.011942058801651001, 0.002263307571411133, 0.005988776683807373, 0.002055943012237549, 0.019833654165267944, 0.01060342788696289, 0.001442551612854004], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 216, \"name\": 216, \"data\": [0.6940784156322479, 0.9248494803905487, 0.9786793887615204, 0.8334912359714508, 0.3600984811782837, 0.9390982985496521, 0.4710276424884796, 0.6377175450325012, 0.2625613510608673, 1.2507629990577698, 0.40971043705940247, 0.8788215816020966, 0.5442004501819611, 0.3435690999031067, 0.24390703439712524, 0.21385687589645386, 0.5880240797996521, 0.18770408630371094, 0.3896980881690979, 0.2536386251449585, 0.3609400689601898, 0.4554940462112427, 0.06950962543487549, 0.23873421549797058, 0.23725849390029907, 1.161621630191803, 0.4655660390853882, 0.11866569519042969, 0.19380474090576172, 0.797152042388916, 0.43164145946502686, 0.3657011389732361, 0.40770086646080017, 0.5467590689659119, 1.1228747963905334, 0.24993199110031128, 0.1493813395500183, 0.10310786962509155, 0.607988566160202, 0.08289182186126709, 0.5381930768489838, 0.3605373203754425, 0.1372305154800415, 0.5848913490772247, 0.7944945693016052, 0.6609717011451721, 0.919623851776123, 0.7288670539855957, 0.7525075078010559, 0.583931565284729, 0.5500531494617462, 0.6802407503128052, 0.17768388986587524, 0.40139299631118774, 1.0804239511489868, 0.3721201717853546, 0.19893014430999756, 1.1245445013046265, 0.3379044532775879, 0.25248873233795166, 0.3319427967071533, 0.7443518042564392, 0.3477667570114136, 0.10770654678344727, 0.09534990787506104, 0.18140947818756104, 0.27682700753211975, 0.28860044479370117, 0.1392589807510376, 0.04407018423080444, 0.07549968361854553, 0.06873363256454468, 0.03422129154205322, 0.4870026707649231, 0.23016875982284546, 0.19727280735969543, 0.31874367594718933, 0.4548599123954773, 0.08482164144515991, 0.48102104663848877, 0.18445676565170288, 0.061764657497406006, 0.5315259695053101, 0.02051854133605957, 0.007476866245269775, 0.032967567443847656, 0.023475050926208496, 0.5615147948265076, 0.2006608545780182, 0.1706235408782959, 0.1725122034549713, 0.30779188871383667, 0.26203638315200806, 0.10940110683441162, 0.16410034894943237, 0.33073562383651733, 0.1054452657699585, 0.31574004888534546, 0.12070876359939575, 0.02400350570678711, 0.30323171615600586, 0.04333096742630005, 0.31139257550239563, 0.07480627298355103, 0.13307136297225952, 0.4648153781890869, 0.14027366042137146, 0.7720370590686798, 0.5985533595085144, 0.9639021754264832, 0.3671264052391052, 0.1054612398147583, 0.16409152746200562, 0.14610826969146729, 0.06347465515136719, 0.14930212497711182, 0.40140968561172485, 0.08537614345550537, 0.4709738492965698, 0.6679494380950928, 0.010588228702545166, 0.037394165992736816, 0.10531497001647949, 0.00993037223815918, 0.0803297758102417, 0.14001482725143433, 0.04476368427276611, 0.0799526572227478, 0.11132544279098511, 0.254514217376709, 0.03212583065032959, 0.019502699375152588, 0.0711890459060669, 0.047438740730285645, 0.011103212833404541, 0.2818019986152649, 0.034928321838378906, 0.14896196126937866, 0.0021692514419555664, 0.09690409898757935, 0.005187869071960449, 0.006870567798614502, 0.010379195213317871, 0.00796198844909668, 0.02523130178451538, 0.006354928016662598, 0.003253936767578125, 0.008662819862365723, 0.012850403785705566, 0.3218764066696167, 0.05159395933151245, 0.0031877756118774414, 0.022865891456604004, 0.02731841802597046, 0.054590582847595215, 0.015559792518615723, 0.03794360160827637, 0.07387948036193848, 0.0433049201965332, 0.03213608264923096, 0.014936566352844238, 0.01843118667602539, 0.0876954197883606, 0.12358766794204712, 0.5430879592895508, 0.024166762828826904, 0.008799850940704346, 0.057386815547943115, 0.04111778736114502, 0.030919790267944336, 0.09333670139312744, 0.03105825185775757, 0.42125117778778076, 0.6889633536338806, 0.058577775955200195, 0.025768935680389404, 0.05972319841384888, 0.0665433406829834, 0.08020168542861938, 0.0033348798751831055, 0.03547012805938721, 0.26210713386535645, 0.07714617252349854, 0.03720593452453613, 0.18347477912902832, 0.010013699531555176, 0.007498741149902344, 0.03330308198928833, 0.3305935263633728, 0.015905916690826416, 0.0034155845642089844, 0.07667839527130127, 0.05832529067993164, 0.006423294544219971, 0.007927238941192627, 0.02332162857055664, 0.027661025524139404, 0.029567956924438477, 0.0114479660987854, 0.06191122531890869, 0.09546911716461182, 0.07963800430297852, 0.008349895477294922, 0.0015288591384887695, 0.0634450912475586, 0.05915480852127075, 0.006949067115783691, 0.01425868272781372, 0.015030145645141602, 0.007136225700378418, 0.04443979263305664, 0.11632847785949707, 0.021332621574401855, 0.003012418746948242, 0.13760757446289062, 0.10173511505126953, 0.0007288455963134766, 0.01650524139404297, 0.12489742040634155, 0.13780707120895386, 0.17727911472320557, 0.02546793222427368, 0.03847914934158325, 0.1259021759033203, 0.05430406332015991, 0.18009734153747559, 0.3093876242637634, 0.06794828176498413, 0.008251368999481201, 0.024776697158813477, 0.046288371086120605, 0.050822556018829346, 0.01145869493484497, 0.000588536262512207, 0.04181325435638428, 0.03233456611633301, 0.07799041271209717, 0.04139900207519531, 0.0106009840965271, 0.022033989429473877, 0.5927186608314514, 0.005146443843841553, 0.0021587610244750977, 0.010887742042541504, 0.01642012596130371, 0.6956580877304077, 0.21066159009933472, 1.0923528671264648], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 220, \"name\": 220, \"data\": [0.8094639480113983, 1.0135068446397781, 0.7522338032722473, 0.8821329772472382, 0.7834722250699997, 0.930659294128418, 0.8351200222969055, 0.579373300075531, 0.4791051745414734, 0.6824324876070023, 0.824621245265007, 0.4312714785337448, 0.7823390662670135, 0.3526235967874527, 0.3967354744672775, 0.27112115919589996, 0.5640111416578293, 0.7128186821937561, 0.24236629903316498, 0.39937178790569305, 0.35433341562747955, 0.39497168362140656, 0.4378839209675789, 0.688149943947792, 0.5391060262918472, 0.5191001147031784, 0.3713604360818863, 0.831189826130867, 0.7108675241470337, 0.24627700448036194, 0.27314217388629913, 0.12919862568378448, 0.17961934208869934, 0.8511260151863098, 0.12018632888793945, 0.855359211564064, 0.4177341014146805, 0.4064813107252121, 0.4090001881122589, 0.09221678972244263, 0.29975035786628723, 0.2452194094657898, 0.13924819231033325, 0.49757665395736694, 0.14525264501571655, 0.20714320242404938, 0.3308609127998352, 0.17766307294368744, 0.3282442092895508, 0.15784813463687897, 0.15398401021957397, 0.2744048833847046, 0.22069181501865387, 0.31724001467227936, 0.375786617398262, 0.24818110466003418, 0.23839335143566132, 0.12061598896980286, 0.23144863545894623, 0.34353867173194885, 0.037816762924194336, 0.09776416420936584, 0.09040224552154541, 0.10767850279808044, 0.374203622341156, 0.03819108009338379, 0.36781999468803406, 0.1675013303756714, 0.10136967897415161, 0.06338030099868774, 0.12182310223579407, 0.13241955637931824, 0.1055234968662262, 0.03868475556373596, 0.2869347929954529, 0.16876277327537537, 0.06385478377342224, 0.15218999981880188, 0.07563978433609009, 0.10155072808265686, 0.045637622475624084, 0.0979870855808258, 0.3658275455236435, 0.07749104499816895, 0.17266321182250977, 0.250652015209198, 0.0920136570930481, 0.09448552131652832, 0.08894875645637512, 0.04648396372795105, 0.10806387662887573, 0.06964099407196045, 0.03675106167793274, 0.05954936146736145, 0.07839646935462952, 0.166238933801651, 0.0729493498802185, 0.20139247179031372, 0.07605671882629395, 0.08949205279350281, 0.1644279956817627, 0.02437382936477661, 0.036395758390426636, 0.11374679207801819, 0.03723311424255371, 0.15614011883735657, 0.023149818181991577, 0.035213351249694824, 0.1472572684288025, 0.050667792558670044, 0.039213359355926514, 0.01155310869216919, 0.04066064953804016, 0.24015814065933228, 0.06459084153175354, 0.0924740582704544, 0.06461778283119202, 0.03890499472618103, 0.04353395104408264, 0.0334528386592865, 0.04930785298347473, 0.055357515811920166, 0.014417469501495361, 0.033839643001556396], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 222, \"name\": 222, \"data\": [0.8012077212333679, 0.5869491100311279, 0.5788043141365051, 1.015000820159912, 0.6511281728744507, 0.7701107263565063, 0.5365025997161865, 0.45119524002075195, 0.2914995551109314, 0.12113618850708008, 2.2714452743530273, 1.538375735282898, 0.06323659420013428, 0.7903413772583008, 0.17966198921203613, 0.17816150188446045, 0.5080101490020752, 0.1301025152206421, 0.2228919267654419, 0.9726947546005249, 0.398160457611084, 0.18319737911224365, 2.386465072631836, 0.7673718929290771, 0.3576709032058716, 0.475735068321228, 0.674694299697876, 0.9301713705062866, 0.4918421506881714, 0.5408649444580078, 0.09319794178009033, 0.31037914752960205, 0.8253504037857056, 0.051157236099243164, 0.076393723487854, 0.274979829788208, 0.19147920608520508, 0.6669963598251343, 0.05589556694030762, 1.85343337059021, 0.9619594812393188, 0.7458386421203613, 1.0507317781448364, 0.7295482158660889, 0.19209814071655273, 0.23503684997558594, 0.2604334354400635, 0.3254094123840332, 0.11975312232971191, 0.10243368148803711, 2.1144630908966064, 0.4212925434112549, 0.30566179752349854, 0.6577599048614502, 0.09469306468963623, 0.7139943838119507, 0.231034517288208, 0.0611652135848999, 0.90773606300354, 0.15130925178527832, 0.2833690643310547, 0.1467989683151245, 0.4020965099334717, 0.569582998752594, 0.40067732334136963, 0.2925233840942383, 1.9593744277954102, 0.8943122625350952, 0.48364055156707764, 0.6796031594276428, 0.24825787544250488, 0.32209205627441406, 0.28216660022735596, 0.18144631385803223, 0.4667642116546631, 0.43283748626708984, 1.1277958154678345, 0.5606975555419922, 0.1658782958984375, 0.3004770278930664, 0.48990607261657715, 0.12197017669677734, 0.9053852558135986, 0.1472148895263672, 0.44315969944000244, 0.1503450870513916, 0.3461798429489136, 1.0214380025863647, 0.48080992698669434, 0.8567901253700256, 0.7756088972091675, 1.1699475049972534, 0.32288897037506104, 1.0472095012664795, 1.362015724182129, 0.4716475009918213, 0.5418311953544617, 0.4113881587982178, 0.8507781624794006, 0.39275509119033813, 0.40659648180007935, 3.780641555786133, 0.2816876769065857, 1.191487193107605, 1.4587377309799194, 0.13083040714263916, 2.102292537689209, 0.9599101543426514, 0.39961302280426025, 1.4948498010635376, 0.03608262538909912, 0.7509695887565613, 0.4844409227371216, 0.3387832045555115, 0.41604387760162354, 1.5847160816192627, 0.49183619022369385, 0.08486306667327881, 0.1431087851524353, 1.3262437582015991, 0.740799069404602, 0.18857276439666748, 0.6301060914993286, 0.28521353006362915, 0.09690487384796143, 0.3729839324951172, 0.2954742908477783, 0.27177274227142334, 0.32508301734924316, 0.5392879247665405, 0.44102126359939575, 0.29798275232315063, 0.4284955859184265, 0.10398495197296143, 0.108795166015625, 0.5847491025924683, 0.011586666107177734, 0.06956923007965088, 0.18626165390014648, 0.05087447166442871, 0.08805215358734131, 0.011174201965332031, 0.4818803071975708, 0.007038593292236328, 0.41624581813812256, 0.03233075141906738, 0.0013158321380615234, 0.9153745174407959, 0.2252204418182373, 1.0570032596588135, 0.036420583724975586, 0.06257224082946777, 0.03924691677093506, 0.03931307792663574, 0.05640721321105957, 0.2760469913482666, 1.4155702590942383, 0.018892288208007812, 0.0022897720336914062, 0.09598755836486816, 0.36687493324279785, 0.06941771507263184, 0.03820443153381348, 0.019550561904907227, 1.1113780736923218, 0.13288938999176025, 0.06523466110229492, 0.12342989444732666, 0.007679939270019531, 0.12390637397766113, 0.10586905479431152, 0.15571677684783936, 0.37590742111206055, 0.22358882427215576, 0.2551865577697754, 0.6864500045776367, 0.084023118019104, 0.051637887954711914, 0.0938560962677002, 0.19735181331634521, 0.18997430801391602, 0.10820627212524414, 1.0528957843780518, 0.19479584693908691, 0.10407161712646484, 0.0482792854309082, 0.07527124881744385, 0.5554441809654236, 0.40830230712890625, 0.6106760501861572, 0.5018846988677979, 0.18364596366882324, 0.40832388401031494, 0.04352688789367676, 0.18114900588989258, 0.11052072048187256, 0.07921314239501953, 0.002179861068725586, 0.3498268127441406, 0.2657126188278198, 0.002706766128540039, 0.2111191749572754, 0.5626670122146606, 0.11193966865539551, 0.3033745288848877, 1.3201143741607666, 0.19246959686279297, 0.22207105159759521, 0.6224579215049744, 0.5794570446014404, 1.13555109500885, 3.524289131164551, 4.726747512817383, 0.5434527397155762, 2.2046399116516113, 2.0374882221221924, 0.9453096389770508, 1.769291639328003, 0.5185337066650391, 0.7353661060333252, 0.633089542388916, 0.7218027114868164, 1.073702096939087, 1.170867919921875, 1.1795060634613037, 1.0690476894378662, 1.6134295463562012, 1.2850563526153564, 0.8580653667449951, 1.1902506351470947, 0.6627967357635498, 0.8344135284423828, 1.2756848335266113, 0.9262979030609131, 1.0177562236785889, 0.6813316345214844, 2.5184757709503174, 1.1949119567871094, 0.7486217021942139, 0.8534164428710938, 0.8856713771820068, 0.923515796661377, 1.026376724243164, 0.7684230804443359, 0.770777702331543, 0.7851195335388184, 0.7636852264404297, 0.7968666553497314, 0.7672545909881592, 0.873359203338623, 0.8065540790557861, 0.729398250579834, 0.7461941242218018, 1.1834189891815186, 1.205735206604004, 1.1367216110229492, 1.1340289115905762, 0.9384887218475342, 0.9806010723114014, 0.9512879848480225, 0.9358313083648682, 1.1732125282287598, 0.7724189758300781, 1.2467193603515625, 1.5323333740234375, 1.360482931137085, 0.7477536201477051, 1.3283865451812744, 0.7542867660522461, 0.790454626083374, 0.7818293571472168, 0.7407469749450684, 2.1334075927734375, 0.8742489814758301, 1.252546787261963, 0.800095796585083, 1.0640652179718018, 0.9241898059844971, 0.9449851512908936, 0.8758044242858887, 0.8937969207763672, 0.932917594909668, 0.8471312522888184, 0.7857100963592529, 0.8512327671051025, 0.7006423473358154, 0.727527379989624, 0.6668744087219238, 0.7857527732849121, 1.4021008014678955, 0.6699042320251465, 2.274976968765259, 0.7500011920928955, 0.726635217666626, 0.8870658874511719, 0.9117028713226318, 0.8492014408111572, 1.462515115737915, 0.7119925022125244, 0.7058382034301758, 0.8703982830047607, 0.9154584407806396, 0.8643758296966553, 0.7977766990661621, 0.7526712417602539, 0.7717621326446533, 0.8794612884521484, 1.3227391242980957, 0.9109876155853271, 0.7382714748382568, 0.7507386207580566, 0.6912493705749512, 0.7666089534759521, 1.721527099609375, 1.114546537399292, 0.9136266708374023, 0.6808991432189941, 0.782625675201416, 0.9254763126373291, 0.8349225521087646, 0.8168044090270996, 1.117492914199829, 0.9234251976013184, 0.7550475597381592, 0.7537002563476562, 0.9995973110198975, 0.8879432678222656, 0.7233192920684814, 0.8881480693817139, 1.5100269317626953, 0.8263206481933594, 0.8770096302032471, 0.7476532459259033, 1.702730655670166, 0.6766915321350098, 1.6623446941375732, 0.7153687477111816, 1.4621586799621582, 1.550262689590454, 0.7461042404174805, 0.7386283874511719, 0.7370662689208984, 1.4695978164672852, 0.746518611907959, 0.7500519752502441, 0.815932035446167, 0.786731481552124, 0.836176872253418, 1.0824251174926758, 0.8759474754333496, 0.8908340930938721, 1.0735664367675781, 1.0861585140228271, 1.298081636428833, 1.127349853515625, 0.858973503112793, 1.14833664894104, 0.8022904396057129, 0.8141887187957764, 1.1765666007995605, 0.7423601150512695, 2.4426627159118652, 1.1851975917816162, 0.7902071475982666, 0.7822475433349609, 0.7716355323791504, 0.7804408073425293, 1.2488367557525635, 0.816352367401123, 0.8459911346435547, 0.7323267459869385, 0.8097357749938965, 0.7724919319152832, 0.8623404502868652, 0.8633809089660645, 0.8577406406402588, 0.7840640544891357, 0.8399367332458496, 1.2114841938018799, 1.1772804260253906, 1.1327555179595947, 1.1256632804870605, 0.8377389907836914, 0.8234012126922607, 0.8290584087371826, 0.8524227142333984, 1.128826379776001, 0.8053197860717773, 1.1630210876464844, 1.153557300567627, 1.1723546981811523, 0.745783805847168, 1.251927375793457, 0.8072333335876465, 0.8270649909973145, 0.779409646987915, 0.7146651744842529, 2.306802272796631, 0.7220945358276367, 1.2467105388641357, 0.7958002090454102, 1.1367542743682861, 0.7986600399017334, 0.7782716751098633, 0.8494541645050049, 0.913060188293457, 0.8442726135253906], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 224, \"name\": 224, \"data\": [0.6940784156322479, 0.9248494803905487, 0.94364133477211, 0.8125570714473724, 0.5312688052654266, 0.9024296402931213, 0.6752737462520599, 0.5472540855407715, 0.319705605506897, 1.4573935866355896, 0.28765320777893066, 0.8699797987937927, 0.5351136326789856, 0.32362043857574463, 0.10136640071868896, 0.567095935344696, 0.5154709815979004, 0.08675205707550049, 0.6771892309188843, 0.3805640935897827, 0.44007930159568787, 0.4764440059661865, 0.5130778551101685, 0.28460651636123663, 0.20410364866256714, 0.967726856470108, 0.5481107234954834, 0.1740659475326538, 0.17322784662246704, 1.132639467716217, 0.41165292263031006, 0.18391889333724976, 0.5401164591312408, 0.6195549368858337, 1.2261767387390137, 0.5213496387004852, 0.25083523988723755, 0.11904036998748779, 0.6345123052597046, 0.1561030149459839, 0.5599097311496735, 0.3844224214553833, 0.1924923062324524, 0.9420504570007324, 0.9397557377815247, 0.9151070713996887, 0.5176693797111511, 0.9477264285087585, 0.36071324348449707, 0.3749450445175171, 0.40030598640441895, 0.38485896587371826, 0.283528208732605, 0.5034213960170746, 1.1017848253250122, 0.26724791526794434, 0.19608810544013977, 0.9720462560653687, 0.32786089181900024, 0.4093872308731079, 0.46107178926467896, 0.24848705530166626, 0.49129217863082886, 0.2293134331703186, 0.2191196084022522, 0.5657797455787659, 0.32542815804481506, 0.8951596915721893, 0.129685640335083, 0.14615660905838013, 0.12261819839477539, 0.3263295590877533, 0.10101717710494995, 0.8373720049858093, 0.3050365149974823, 0.255318284034729, 0.22437268495559692, 0.26969265937805176, 0.015976130962371826, 0.00989621877670288, 0.27862364053726196, 0.01921260356903076, 0.8686290383338928, 0.1034245491027832, 0.07756870985031128, 0.44260507822036743, 0.12047016620635986, 0.4053383469581604, 0.28098538517951965, 0.199660062789917, 0.2123684287071228, 0.22071534395217896, 0.03206294775009155, 0.12748169898986816, 0.3563230335712433, 0.2328266203403473, 0.19492414593696594, 0.10234075784683228, 0.05323338508605957, 0.015514373779296875, 0.24765872955322266, 0.0060999393463134766, 0.190729022026062, 0.0555570125579834, 0.0971718430519104, 0.945491373538971, 0.06308799982070923, 0.5088343620300293, 0.5057203769683838, 0.24117517471313477, 0.01924288272857666, 0.15777963399887085, 0.08742433786392212, 0.7876163721084595, 0.03796064853668213, 0.26900073885917664, 0.3208325505256653, 0.5639466047286987, 0.5069135129451752, 0.5433251261711121, 0.47138461470603943, 0.2691258490085602, 0.6680338382720947, 0.23136243224143982, 0.26997676491737366, 0.1107490062713623, 0.22148430347442627, 0.2876788377761841, 1.0480828285217285, 0.787080705165863, 0.11171102523803711, 0.3500160872936249, 0.23340177536010742, 0.2247365117073059, 0.07755935192108154, 0.2328367829322815, 0.3804647922515869, 0.05977463722229004, 0.07152312994003296, 0.2939813733100891, 0.017599105834960938, 0.05811667442321777, 0.0164642333984375, 0.023466944694519043, 0.15621018409729004, 0.01205068826675415, 0.02944314479827881, 0.01750791072845459, 0.03777813911437988, 0.35597318410873413, 0.06787770986557007, 0.004987359046936035, 0.0638776421546936, 0.13574236631393433, 0.02321922779083252, 0.03590160608291626, 0.07714676856994629, 0.46101659536361694, 0.04759758710861206, 0.08300948143005371, 0.006238460540771484, 0.0028520822525024414, 0.3074672818183899, 0.09060007333755493, 0.21481287479400635, 0.007674217224121094, 0.034785568714141846, 0.07930779457092285, 0.06809306144714355, 0.2820098102092743, 0.17033147811889648, 0.03211987018585205, 0.014247536659240723, 0.6038203835487366, 0.026617050170898438, 0.4781724214553833, 0.004691898822784424, 0.015082180500030518, 0.07418221235275269, 0.12440383434295654, 0.0923072099685669, 0.10408258438110352, 0.006030380725860596, 0.02855461835861206, 0.08030921220779419, 0.00041043758392333984, 0.01919776201248169, 0.1788022518157959, 0.4246888756752014, 0.025166630744934082, 0.05562448501586914, 0.21768134832382202, 0.08580607175827026, 0.005449056625366211, 0.012134552001953125, 0.04528367519378662, 0.003322303295135498, 0.10307443141937256, 0.0591886043548584, 0.11427813768386841, 0.0046721696853637695, 0.31153029203414917, 0.02315300703048706, 0.008599996566772461, 0.008517622947692871, 0.07712650299072266, 0.019654452800750732, 0.007430136203765869, 0.014071822166442871, 0.01577627658843994, 0.08380782604217529, 0.0117262601852417, 0.005031585693359375, 0.06554490327835083, 0.009949207305908203, 0.007855415344238281, 0.017253756523132324, 0.023352503776550293, 0.01805788278579712, 0.08137959241867065, 0.016136109828948975, 0.30280226469039917], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 223, \"name\": 223, \"data\": [0.8094639480113983, 0.9736936837434769, 0.7024015337228775, 0.8353763967752457, 0.7609652280807495, 0.8579869419336319, 0.7399982213973999, 0.2045290470123291, 0.5717583149671555, 0.39414308965206146, 0.6549608409404755, 0.23175877332687378, 0.7931201457977295, 0.2735438495874405, 0.2548436373472214, 0.28991660475730896, 0.3506616801023483, 0.6391906440258026, 0.11726966500282288, 0.2624213397502899, 0.36124274134635925, 0.2887173742055893, 0.37288394570350647, 0.7247939109802246, 0.6089345216751099, 0.3345566987991333, 0.3463672399520874, 0.8464425057172775, 0.7578305304050446, 0.2764517366886139, 0.27889516949653625, 0.1247146725654602, 0.14845214784145355, 0.8521626889705658, 0.19608289003372192, 0.11674866080284119, 0.28415457904338837, 0.1784439980983734, 0.10577809810638428, 0.20132769644260406, 0.029565006494522095, 0.09276443719863892, 0.36730268597602844, 0.07314836978912354, 0.07752349972724915, 0.18412303924560547, 0.24403485655784607, 0.09442150592803955, 0.10453465580940247, 0.037044376134872437, 0.10963201522827148, 0.25062793493270874, 0.17983239889144897, 0.7013712823390961, 0.3364938795566559, 0.09599709510803223, 0.050975918769836426, 0.027409076690673828, 0.16880664229393005, 0.21456202864646912, 0.14165964722633362, 0.05054715275764465, 0.0097426176071167, 0.08078870177268982, 0.30620652437210083, 0.05860885977745056, 0.07108747959136963, 0.04099276661872864, 0.017084091901779175, 0.11337846517562866, 0.006810367107391357, 0.031146764755249023, 0.02197498083114624, 0.024283885955810547, 0.10904756188392639, 0.013566672801971436, 0.017998158931732178, 0.009068429470062256, 0.01653587818145752, 0.06474849581718445, 0.005025953054428101, 0.019675016403198242, 0.06017902493476868, 0.020244598388671875, 0.13704898953437805, 0.05501428246498108, 0.010273873805999756, 0.04615405201911926, 0.014218002557754517, 0.022233903408050537, 0.01705765724182129, 0.002805650234222412, 0.01852598786354065, 0.007150918245315552, 0.19618424773216248, 0.04108414053916931, 0.003345698118209839, 0.006989181041717529, 0.02508610486984253, 0.009858846664428711, 0.004354953765869141, 0.0025833845138549805, 0.003450751304626465, 0.011396229267120361, 0.004473328590393066, 0.06062448024749756, 0.0038622617721557617, 0.018446683883666992, 0.0044516921043396, 0.014563590288162231, 0.00551876425743103, 0.003903001546859741, 0.007115960121154785, 0.00932079553604126, 0.01008877158164978, 0.02033296227455139, 0.008978992700576782, 0.004568547010421753, 0.010957151651382446, 0.024907231330871582, 0.014680564403533936, 0.014545023441314697, 0.00224381685256958, 0.011619925498962402], \"mode\": \"lines\", \"stepped\": false}], \"epoch_time\": [{\"run_id\": 208, \"name\": 208, \"data\": [17.574909925460815, 17.46017360687256, 17.48988389968872, 17.415295124053955], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 214, \"name\": 214, \"data\": [16.475717782974243, 16.259063005447388, 16.24798083305359, 16.26915669441223], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 206, \"name\": 206, \"data\": [80.52889895439148, 80.01408529281616], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 207, \"name\": 207, \"data\": [81.77077436447144, 81.70748281478882], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 209, \"name\": 209, \"data\": [82.79566550254822, 82.9411506652832, 82.80448913574219], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 211, \"name\": 211, \"data\": [82.86951446533203, 82.45981454849243, 82.48958897590637], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 210, \"name\": 210, \"data\": [87.64912629127502, 87.59788012504578, 87.35625410079956], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 213, \"name\": 213, \"data\": [79.92867827415466, 79.86816096305847, 79.65678668022156, 79.68190431594849], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 205, \"name\": 205, \"data\": [86.18277359008789, 86.02290439605713, 86.7013897895813, 85.89329862594604], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 212, \"name\": 212, \"data\": [80.49062991142273, 80.29034543037415, 80.40366172790527, 80.29224300384521], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 217, \"name\": 217, \"data\": [16.55463409423828, 16.358238458633423, 16.380149364471436, 16.38364052772522], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 218, \"name\": 218, \"data\": [17.580978393554688, 17.536532402038574, 17.508822679519653, 17.53034996986389], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 215, \"name\": 215, \"data\": [87.10174298286438, 87.22889137268066], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 221, \"name\": 221, \"data\": [18.18412733078003, 18.099773168563843, 17.999746799468994, 17.59343385696411], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 216, \"name\": 216, \"data\": [84.03921222686768, 82.67836260795593, 83.11873388290405, 83.17526721954346], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 220, \"name\": 220, \"data\": [79.76918292045593, 79.53885102272034, 79.50361919403076, 79.46583533287048], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 222, \"name\": 222, \"data\": [87.97214889526367, 87.63434553146362, 87.71718049049377], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 224, \"name\": 224, \"data\": [82.47967505455017, 82.11728930473328, 80.5644633769989], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 223, \"name\": 223, \"data\": [79.73718309402466, 79.56347918510437, 79.32488560676575, 79.20234680175781], \"mode\": \"lines\", \"stepped\": false}], \"val_loss\": [{\"run_id\": 208, \"name\": 208, \"data\": [0.4517747511466344, 0.41446178654829663, 0.43706075350443524, 0.45371869802474973], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 214, \"name\": 214, \"data\": [0.5218349754810333, 0.4816325883070628, 0.634685813387235, 0.7552561978499095], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 206, \"name\": 206, \"data\": [0.7535982847213745, 0.9946286002794902], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 207, \"name\": 207, \"data\": [3.8158863385518393, 1.1344702283541361], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 209, \"name\": 209, \"data\": [0.6437874078750611, 0.47793306509653727, 0.5030475596586863], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 211, \"name\": 211, \"data\": [0.4520814329385757, 0.6209243635336558], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 210, \"name\": 210, \"data\": [0.5314400792121887, 0.48450330396493274, 0.7215488329529762], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 213, \"name\": 213, \"data\": [0.34451807141304014, 0.34779350211222965, 0.38273228878776233, 0.435079567382733], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 205, \"name\": 205, \"data\": [0.3447953090071678, 0.4683617124954859, 0.4515557065606117, 0.4871247986952464], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 212, \"name\": 212, \"data\": [0.40760256747404733, 0.39944262281060217, 0.4111525949090719, 0.5633959598839283], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 217, \"name\": 217, \"data\": [0.44055124521255495, 0.41151560147603355, 0.49621564894914627, 0.520169131954511], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 218, \"name\": 218, \"data\": [0.5413984517256419, 0.7335025548934937, 0.7084611962238948, 0.7140787939230601], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 215, \"name\": 215, \"data\": [1.1182499051094055, 1.0855965455373129], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 221, \"name\": 221, \"data\": [0.5469533582528432, 0.5124016682306926, 0.5613361398379008, 0.5905174116293589], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 216, \"name\": 216, \"data\": [0.4251174688339233, 0.490706080198288, 0.5787086809674898, 0.8955366114775339], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 220, \"name\": 220, \"data\": [0.3719762454430262, 0.37295784056186676, 0.3944741770625114, 0.4045510306954384], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 222, \"name\": 222, \"data\": [0.5453706542650859, 1.0875417669614156, 1.0313016374905903], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 224, \"name\": 224, \"data\": [0.49731663266817727, 0.5261086364587148, 0.5898666209230821], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 223, \"name\": 223, \"data\": [0.3350112020969391, 0.35396125465631484, 0.3722584423919519, 0.37783159390091897], \"mode\": \"lines\", \"stepped\": false}], \"val_acc\": [{\"run_id\": 208, \"name\": 208, \"data\": [0.8392070484581498, 0.8590308370044053, 0.8590308370044053, 0.8590308370044053], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 214, \"name\": 214, \"data\": [0.7819383259911894, 0.801762114537445, 0.8171806167400881, 0.8061674008810573], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 206, \"name\": 206, \"data\": [0.6894273127753304, 0.5242290748898678], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 207, \"name\": 207, \"data\": [0.3964757709251101, 0.4052863436123348], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 209, \"name\": 209, \"data\": [0.7422907488986784, 0.8127753303964758, 0.8414096916299559], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 211, \"name\": 211, \"data\": [0.8259911894273128, 0.7929515418502202], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 210, \"name\": 210, \"data\": [0.8105726872246696, 0.8502202643171806, 0.8127753303964758], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 213, \"name\": 213, \"data\": [0.8832599118942731, 0.8986784140969163, 0.9074889867841409, 0.8986784140969163], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 205, \"name\": 205, \"data\": [0.8788546255506607, 0.8766519823788547, 0.8942731277533039, 0.8942731277533039], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 212, \"name\": 212, \"data\": [0.8414096916299559, 0.8810572687224669, 0.8810572687224669, 0.8612334801762115], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 217, \"name\": 217, \"data\": [0.8392070484581498, 0.8502202643171806, 0.8546255506607929, 0.8612334801762115], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 218, \"name\": 218, \"data\": [0.7819383259911894, 0.7995594713656388, 0.7973568281938326, 0.8105726872246696], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 215, \"name\": 215, \"data\": [0.44493392070484583, 0.40969162995594716], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 221, \"name\": 221, \"data\": [0.7555066079295154, 0.8237885462555066, 0.801762114537445, 0.8215859030837004], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 216, \"name\": 216, \"data\": [0.8414096916299559, 0.8612334801762115, 0.8656387665198237, 0.7929515418502202], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 220, \"name\": 220, \"data\": [0.8634361233480177, 0.8678414096916299, 0.8744493392070485, 0.8810572687224669], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 222, \"name\": 222, \"data\": [0.7907488986784141, 0.44493392070484583, 0.42731277533039647], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 224, \"name\": 224, \"data\": [0.7951541850220264, 0.7995594713656388, 0.8458149779735683], \"mode\": \"lines\", \"stepped\": false}, {\"run_id\": 223, \"name\": 223, \"data\": [0.8876651982378855, 0.9052863436123348, 0.8964757709251101, 0.8964757709251101], \"mode\": \"lines\", \"stepped\": false}]}, \"metricName\": null, \"primaryMetricName\": \"val_acc\", \"showLegend\": true}, \"run_metrics\": [{\"name\": \"best_child_by_primary_metric\", \"run_id\": \"Testing-AML-hyperdrive_1572978287720207\", \"categories\": [0], \"series\": [{\"data\": [{\"metric_name\": [\"val_acc\", \"val_acc\", \"val_acc\", \"val_acc\", \"val_acc\"], \"timestamp\": [\"2019-11-05 18:28:25.765390+00:00\", \"2019-11-05 18:29:58.643250+00:00\", \"2019-11-05 18:31:00.411299+00:00\", \"2019-11-05 18:32:31.724289+00:00\", \"2019-11-05 18:32:31.724289+00:00\"], \"run_id\": [\"Testing-AML-hyperdrive_1572978287720207_0\", \"Testing-AML-hyperdrive_1572978287720207_7\", \"Testing-AML-hyperdrive_1572978287720207_7\", \"Testing-AML-hyperdrive_1572978287720207_7\", \"Testing-AML-hyperdrive_1572978287720207_7\"], \"metric_value\": [0.8590308370044053, 0.8832599118942731, 0.8986784140969163, 0.9074889867841409, 0.9074889867841409], \"final\": [false, false, false, false, true]}]}]}], \"run_logs\": \"[2019-11-05T18:24:47.958689][API][INFO]Experiment created\\r\\n[2019-11-05T18:24:49.060710][GENERATOR][INFO]Successfully sampled '10' jobs, they will soon be submitted to the execution target.\\r\\n[2019-11-05T18:24:50.0781172Z][SCHEDULER][INFO]The execution environment is being prepared. Please be patient as it can take a few minutes.\\r\\n[2019-11-05T18:24:48.695680][GENERATOR][INFO]Trying to sample '10' jobs from the hyperparameter space\\r\\n[2019-11-05T18:25:21.4994512Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_0'\\r\\n[2019-11-05T18:25:21.4989532Z][SCHEDULER][INFO]The execution environment was successfully prepared.\\r\\n[2019-11-05T18:25:21.5012028Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_1'\\r\\n[2019-11-05T18:25:21.5028125Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_2'\\r\\n[2019-11-05T18:25:21.5100678Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_7'\\r\\n[2019-11-05T18:25:21.5194814Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_9'\\r\\n[2019-11-05T18:25:21.5043919Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_3'\\r\\n[2019-11-05T18:25:21.5086461Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_6'\\r\\n[2019-11-05T18:25:21.5072694Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_5'\\r\\n[2019-11-05T18:25:21.5058015Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_4'\\r\\n[2019-11-05T18:25:21.5114439Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_8'\\r\\n[2019-11-05T18:25:22.5329393Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_4'\\r\\n[2019-11-05T18:25:22.5889411Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_5'\\r\\n[2019-11-05T18:25:22.8008375Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_8'\\r\\n[2019-11-05T18:25:23.1571617Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_0'\\r\\n[2019-11-05T18:25:23.1916716Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_6'\\r\\n[2019-11-05T18:25:23.3839821Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_7'\\r\\n[2019-11-05T18:25:23.4164681Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_9'\\r\\n[2019-11-05T18:25:23.9253909Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_3'\\r\\n[2019-11-05T18:25:23.9242659Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_1'\\r\\n[2019-11-05T18:25:24.6361729Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_2'\\r\\n[2019-11-05T18:25:48.271419][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_0', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_1', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_2', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_3', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_4', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_5', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_6', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_7', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_8', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_9', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:26:19.595798][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_0', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_1', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_2', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_3', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_4', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_5', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_6', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_7', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_8', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_9', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:26:50.235636][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_0', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_1', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_2', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_3', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_4', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_5', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_6', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_7', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_8', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_9', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:27:21.840551][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_0', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_1', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_2', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_3', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_4', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_5', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_6', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_7', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_8', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_9', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:27:52.233326][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_0', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_1', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_2', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_3', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_4', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_5', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_6', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_7', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_8', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_9', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:28:22.569374][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_1', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_3', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_4', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_5', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_6', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_7', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_8', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_9', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:28:53.899396][ENFORCER][INFO]Request cancellation of job https://eastus.experiments.azureml.net/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207_2, sequence 2, median running average: 0.8263582966226137, job's best metric: 0.8171806167400881\\r\\n[2019-11-05T18:28:54.289889][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_1', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_3', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_4', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_5', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_6', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_7', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_8', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_9', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:28:54.290010][ENFORCER][INFO]Policy cancelled 1 jobs\\r\\n[2019-11-05T18:29:01.6164287Z][SCHEDULER][INFO]Cancelling job, id='Testing-AML-hyperdrive_1572978287720207_2'\\r\\n[2019-11-05T18:29:04.3111726Z][SCHEDULER][INFO]Updating job statuses to cancelled: [(job id = 'Testing-AML-hyperdrive_1572978287720207_2', previous status = 'RUNNING')]\\r\\n[2019-11-05T18:29:20.185254][GENERATOR][INFO]Trying to sample '2' jobs from the hyperparameter space\\r\\n[2019-11-05T18:29:20.540595][GENERATOR][INFO]Successfully sampled '2' jobs, they will soon be submitted to the execution target.\\r\\n[2019-11-05T18:29:24.510763][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_1', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_3', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_4', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_5', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_6', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_7', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_8', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_9', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:29:35.9658815Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_10'\\r\\n[2019-11-05T18:29:35.9674295Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_11'\\r\\n[2019-11-05T18:29:37.3154092Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_10'\\r\\n[2019-11-05T18:29:38.5930965Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_11'\\r\\n[2019-11-05T18:29:55.446911][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_3', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_10', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_11', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:30:25.874679][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_10', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_11', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:30:56.360420][ENFORCER][INFO]Request cancellation of job https://eastus.experiments.azureml.net/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207_5, sequence 1, median running average: 0.8105726872246696, job's best metric: 0.6894273127753304\\r\\n[2019-11-05T18:30:56.733475][ENFORCER][INFO]Request cancellation of job https://eastus.experiments.azureml.net/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207_8, sequence 1, median running average: 0.8105726872246696, job's best metric: 0.4052863436123348\\r\\n[2019-11-05T18:30:57.119776][ENFORCER][INFO]Policy cancelled 2 jobs\\r\\n[2019-11-05T18:30:57.119697][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_10', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_11', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:31:11.2145256Z][SCHEDULER][INFO]Cancelling job, id='Testing-AML-hyperdrive_1572978287720207_8'\\r\\n[2019-11-05T18:31:11.2137012Z][SCHEDULER][INFO]Cancelling job, id='Testing-AML-hyperdrive_1572978287720207_5'\\r\\n[2019-11-05T18:31:13.4765739Z][SCHEDULER][INFO]Updating job statuses to cancelled: [(job id = 'Testing-AML-hyperdrive_1572978287720207_5', previous status = 'RUNNING'), (job id = 'Testing-AML-hyperdrive_1572978287720207_8', previous status = 'RUNNING')]\\r\\n[2019-11-05T18:31:23.238708][GENERATOR][INFO]Trying to sample '1' jobs from the hyperparameter space\\r\\n[2019-11-05T18:31:23.593553][GENERATOR][INFO]Successfully sampled '1' jobs, they will soon be submitted to the execution target.\\r\\n[2019-11-05T18:31:27.454053][ENFORCER][INFO]Request cancellation of job https://eastus.experiments.azureml.net/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207_6, sequence 1, median running average: 0.8491189427312775, job's best metric: 0.8127753303964758\\r\\n[2019-11-05T18:31:27.843594][ENFORCER][INFO]Request cancellation of job https://eastus.experiments.azureml.net/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207_9, sequence 1, median running average: 0.8491189427312775, job's best metric: 0.8259911894273128\\r\\n[2019-11-05T18:31:28.198259][ENFORCER][INFO]Policy cancelled 2 jobs\\r\\n[2019-11-05T18:31:28.198146][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_10', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_11', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:31:45.2005042Z][SCHEDULER][INFO]Cancelling job, id='Testing-AML-hyperdrive_1572978287720207_9'\\r\\n[2019-11-05T18:31:45.1994635Z][SCHEDULER][INFO]Cancelling job, id='Testing-AML-hyperdrive_1572978287720207_6'\\r\\n[2019-11-05T18:31:46.3978246Z][SCHEDULER][INFO]Updating job statuses to cancelled: [(job id = 'Testing-AML-hyperdrive_1572978287720207_6', previous status = 'RUNNING'), (job id = 'Testing-AML-hyperdrive_1572978287720207_9', previous status = 'RUNNING')]\\r\\n[2019-11-05T18:31:46.7012532Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_12'\\r\\n[2019-11-05T18:31:47.3622153Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_12'\\r\\n[2019-11-05T18:31:53.607090][GENERATOR][INFO]Trying to sample '3' jobs from the hyperparameter space\\r\\n[2019-11-05T18:31:53.730455][GENERATOR][INFO]Successfully sampled '3' jobs, they will soon be submitted to the execution target.\\r\\n[2019-11-05T18:31:58.273054][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_10', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_11', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:32:17.9118790Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_14'\\r\\n[2019-11-05T18:32:17.9101674Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_13'\\r\\n[2019-11-05T18:32:17.9133972Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_15'\\r\\n[2019-11-05T18:32:18.7534266Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_14'\\r\\n[2019-11-05T18:32:18.9869273Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_13'\\r\\n[2019-11-05T18:32:19.6432052Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_15'\\r\\n[2019-11-05T18:32:36.095117][ENFORCER][INFO]Request cancellation of job https://eastus.experiments.azureml.net/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207_1, sequence 1, median running average: 0.8612334801762114, job's best metric: 0.8502202643171806\\r\\n[2019-11-05T18:32:37.211633][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_10', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_11', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_12', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_13', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_14', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_15', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:32:37.211730][ENFORCER][INFO]Policy cancelled 1 jobs\\r\\n[2019-11-05T18:32:50.1611088Z][SCHEDULER][INFO]Cancelling job, id='Testing-AML-hyperdrive_1572978287720207_1'\\r\\n[2019-11-05T18:32:50.8493659Z][SCHEDULER][INFO]Updating job statuses to cancelled: [(job id = 'Testing-AML-hyperdrive_1572978287720207_1', previous status = 'RUNNING')]\\r\\n[2019-11-05T18:33:06.651916][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_10', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_11', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_12', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_13', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_14', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_15', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:33:24.068216][GENERATOR][INFO]Trying to sample '1' jobs from the hyperparameter space\\r\\n[2019-11-05T18:33:24.437982][GENERATOR][INFO]Successfully sampled '1' jobs, they will soon be submitted to the execution target.\\r\\n[2019-11-05T18:33:37.323738][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_10', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_11', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_12', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_13', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_14', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_15', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:33:52.0985305Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_16'\\r\\n[2019-11-05T18:33:54.1546387Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_16'\\r\\n[2019-11-05T18:34:08.266204][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_12', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_13', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_14', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_15', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:34:24.671616][GENERATOR][INFO]Trying to sample '1' jobs from the hyperparameter space\\r\\n[2019-11-05T18:34:24.811240][GENERATOR][INFO]Successfully sampled '1' jobs, they will soon be submitted to the execution target.\\r\\n[2019-11-05T18:34:39.642890][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_13', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_14', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_15', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_16', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:34:54.677534][GENERATOR][INFO]Trying to sample '2' jobs from the hyperparameter space\\r\\n[2019-11-05T18:34:54.795919][GENERATOR][INFO]Successfully sampled '2' jobs, they will soon be submitted to the execution target.\\r\\n[2019-11-05T18:34:55.1156388Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_17'\\r\\n[2019-11-05T18:34:55.1190803Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_19'\\r\\n[2019-11-05T18:34:55.1174492Z][SCHEDULER][INFO]Scheduling job, id='Testing-AML-hyperdrive_1572978287720207_18'\\r\\n[2019-11-05T18:34:56.4230370Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_17'\\r\\n[2019-11-05T18:34:57.1250061Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_18'\\r\\n[2019-11-05T18:34:57.9737277Z][SCHEDULER][INFO]Successfully scheduled a job. Id='Testing-AML-hyperdrive_1572978287720207_19'\\r\\n[2019-11-05T18:35:09.678871][ENFORCER][INFO]Request cancellation of job https://eastus.experiments.azureml.net/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207_14, sequence 1, median running average: 0.8469162995594713, job's best metric: 0.7995594713656388\\r\\n[2019-11-05T18:35:09.815832][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_13', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_15', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_16', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_18', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:35:09.816002][ENFORCER][INFO]Policy cancelled 1 jobs\\r\\n[2019-11-05T18:35:24.561507][GENERATOR][INFO]Max number of jobs '20' reached for experiment.\\r\\n[2019-11-05T18:35:24.686969][GENERATOR][INFO]All jobs generated.\\r\\n[2019-11-05T18:35:29.6712782Z][SCHEDULER][INFO]Cancelling job, id='Testing-AML-hyperdrive_1572978287720207_14'\\r\\n[2019-11-05T18:35:37.4528428Z][SCHEDULER][INFO]Updating job statuses to cancelled: [(job id = 'Testing-AML-hyperdrive_1572978287720207_14', previous status = 'RUNNING')]\\r\\n[2019-11-05T18:35:40.407420][ENFORCER][INFO]Request cancellation of job https://eastus.experiments.azureml.net/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207_10, sequence 1, median running average: 0.8502202643171806, job's best metric: 0.44493392070484583\\r\\n[2019-11-05T18:35:40.792727][ENFORCER][INFO]Request cancellation of job https://eastus.experiments.azureml.net/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207_14, sequence 3, median running average: 0.8526982378854626, job's best metric: 0.8105726872246696\\r\\n[2019-11-05T18:35:41.175262][ENFORCER][INFO]Policy cancelled 2 jobs\\r\\n[2019-11-05T18:35:41.175172][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_15', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_16', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_17', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_18', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_19', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:36:10.539557][ENFORCER][INFO]Request cancellation of job https://eastus.experiments.azureml.net/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207_10, sequence 1, median running average: 0.8513215859030837, job's best metric: 0.44493392070484583\\r\\n[2019-11-05T18:36:10.651465][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_15', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_16', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_17', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_18', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_19', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:36:10.651570][ENFORCER][INFO]Policy cancelled 1 jobs\\r\\n[2019-11-05T18:36:10.6607071Z][SCHEDULER][INFO]Cancelling job, id='Testing-AML-hyperdrive_1572978287720207_10'\\r\\n[2019-11-05T18:36:11.1599276Z][SCHEDULER][INFO]Updating job statuses to cancelled: [(job id = 'Testing-AML-hyperdrive_1572978287720207_10', previous status = 'RUNNING')]\\r\\n[2019-11-05T18:36:41.403344][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_17', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_18', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_19', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:37:13.297649][ENFORCER][INFO]Request cancellation of job https://eastus.experiments.azureml.net/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207_16, sequence 2, median running average: 0.8597650513950073, job's best metric: 0.8237885462555066\\r\\n[2019-11-05T18:37:14.894911][ENFORCER][INFO]Policy cancelled 1 jobs\\r\\n[2019-11-05T18:37:14.894749][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_17', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_18', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_19', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:37:55.049960][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_17', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_18', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_19', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:38:25.507573][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_17', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_18', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_19', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:38:56.451140][ENFORCER][INFO]Jobs [RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_17', data_container_id=None), RunKey(run_scope=RunScope(host='https://eastus.experiments.azureml.net', subscription_id='42ae47bd-b19b-42c1-b0b9-19fd5be9d51b', resource_group='bert-base', workspace_name='SubstrateIntelligenceNLR-WS2', project_name='Testing-AML-hyperdrive'), run_id='Testing-AML-hyperdrive_1572978287720207_18', data_container_id=None)] do not contain any metrics named 'val_acc' at this moment, policy cannot be applied.\\r\\n[2019-11-05T18:40:28.312293][ENFORCER][INFO]Request cancellation of job https://eastus.experiments.azureml.net/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207_17, sequence 1, median running average: 0.8513215859030837, job's best metric: 0.7907488986784141\\r\\n[2019-11-05T18:40:28.751032][ENFORCER][INFO]Request cancellation of job https://eastus.experiments.azureml.net/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207_19, sequence 1, median running average: 0.8513215859030837, job's best metric: 0.7995594713656388\\r\\n[2019-11-05T18:40:29.115841][ENFORCER][INFO]Policy cancelled 2 jobs\\r\\n[2019-11-05T18:41:00.595061][ENFORCER][INFO]Request cancellation of job https://eastus.experiments.azureml.net/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207_17, sequence 1, median running average: 0.8513215859030837, job's best metric: 0.7907488986784141\\r\\n[2019-11-05T18:41:00.709895][ENFORCER][INFO]Request cancellation of job https://eastus.experiments.azureml.net/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207_19, sequence 1, median running average: 0.8513215859030837, job's best metric: 0.7995594713656388\\r\\n[2019-11-05T18:41:00.900773][ENFORCER][INFO]Policy cancelled 2 jobs\\r\\n[2019-11-05T18:41:57.7273432Z][SCHEDULER][INFO]Cancelling job, id='Testing-AML-hyperdrive_1572978287720207_19'\\r\\n[2019-11-05T18:41:57.7264414Z][SCHEDULER][INFO]Cancelling job, id='Testing-AML-hyperdrive_1572978287720207_17'\\r\\n[2019-11-05T18:41:59.1995482Z][SCHEDULER][INFO]Updating job statuses to cancelled: [(job id = 'Testing-AML-hyperdrive_1572978287720207_17', previous status = 'RUNNING'), (job id = 'Testing-AML-hyperdrive_1572978287720207_19', previous status = 'RUNNING')]\\r\\n[2019-11-05T18:42:00.888320][ENFORCER][INFO]Request cancellation of job https://eastus.experiments.azureml.net/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207_17, sequence 2, median running average: 0.8560939794419969, job's best metric: 0.7907488986784141\\r\\n[2019-11-05T18:42:01.230285][ENFORCER][INFO]Request cancellation of job https://eastus.experiments.azureml.net/subscriptions/42ae47bd-b19b-42c1-b0b9-19fd5be9d51b/resourceGroups/bert-base/providers/Microsoft.MachineLearningServices/workspaces/SubstrateIntelligenceNLR-WS2/experiments/Testing-AML-hyperdrive/runs/Testing-AML-hyperdrive_1572978287720207_19, sequence 2, median running average: 0.8560939794419969, job's best metric: 0.8458149779735683\\r\\n[2019-11-05T18:42:01.411120][ENFORCER][INFO]Policy cancelled 2 jobs\\r\\n[2019-11-05T18:44:32.041407][CONTROLLER][INFO]Experiment was 'ExperimentStatus.RUNNING', is 'ExperimentStatus.FINISHED'.\\n\\nRun is completed.\", \"graph\": {}, \"widget_settings\": {\"childWidgetDisplay\": \"popup\", \"send_telemetry\": false, \"log_level\": \"INFO\", \"sdk_version\": \"1.0.72\"}, \"loading\": false}"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "experiment_name = 'Testing-AML-hyperdrive'\n",
    "experiment = Experiment(ws, name=experiment_name)\n",
    "\n",
    "hyperdrive_run = experiment.submit(hyperdrive_run_config)\n",
    "from azureml.widgets import RunDetails\n",
    "RunDetails(hyperdrive_run).show()"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "for run in experiment.get_runs():\n",
    "    if 'Running' == run.status:\n",
    "        hyperdrive_run = run\n",
    "        print(hyperdrive_run)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Best run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'NoneType' object has no attribute 'get_metrics'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-51-8ed75835ef9c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mbest_run\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhyperdrive_run\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_best_run_by_primary_metric\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mbest_run_metrics\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbest_run\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_metrics\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mparameter_values\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbest_run\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_details\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'runDefinition'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'arguments'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Best Run Id: '\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbest_run\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mid\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'get_metrics'"
     ]
    }
   ],
   "source": [
    "best_run = hyperdrive_run.get_best_run_by_primary_metric()\n",
    "best_run_metrics = best_run.get_metrics()\n",
    "parameter_values = best_run.get_details()['runDefinition']['arguments']\n",
    "\n",
    "print('Best Run Id: ', best_run.id)\n",
    "print('\\n Accuracy:', best_run_metrics['val_acc'])\n",
    "print('\\n params:',parameter_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "best_run.get_file_names()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test data as validation file"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}