default_curl_v3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "h6v4c9_UwOmr",
    "outputId": "d5b5140a-cd61-45f6-859d-79986d112476"
   },
   "outputs": [],
   "source": [
    "%load_ext tensorboard\n",
    "\n",
    "import math\n",
    "import gym \n",
    "import plotly.express as px\n",
    "import numpy as np\n",
    "import warnings\n",
    "from plotly.subplots import make_subplots\n",
    "import plotly.graph_objects as go\n",
    "import random\n",
    "from collections import Counter, deque\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "import datetime\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import plotly.figure_factory as ff\n",
    "\n",
    "import os\n",
    "import glob\n",
    "from IPython import display as ipythondisplay\n",
    "from tqdm.notebook import tqdm\n",
    "from gym.wrappers import Monitor\n",
    "from IPython.display import HTML\n",
    "import base64\n",
    "import io\n",
    "import pickle\n",
    "import torch\n",
    "from torch import nn\n",
    "import kornia.augmentation as kaug\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cpu')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "XREqct6awOms"
   },
   "outputs": [],
   "source": [
    "from pyvirtualdisplay import Display\n",
    "display = Display(visible=False, size=(1400, 900))\n",
    "if torch.cuda.is_available():\n",
    "    display.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "T1ACavBgwOmt"
   },
   "outputs": [],
   "source": [
    "import os, sys, copy, argparse, shutil\n",
    "def parse_arguments():\n",
    "    parser = argparse.ArgumentParser(description='Deep Q Network Argument Parser')\n",
    "    parser.add_argument('--seed', dest='seed', type=int, default=1)\n",
    "    parser.add_argument('--env', dest='env', type=str, default='CartPole-v0')\n",
    "    parser.add_argument('--save_interval', type=int, default=50, help='save model every n episodes')\n",
    "    parser.add_argument('--log_interval', type=int, default=10, help='logging every n episodes')\n",
    "    parser.add_argument('--render', help='render', type=int, default=1)\n",
    "    parser.add_argument('--batch_size', help='batch_size', type=int, default=32)\n",
    "    parser.add_argument('--train_freq', help='train_frequency', type=int, default=1)\n",
    "    parser.add_argument('--max_episode', help='maximum episode', type=int, default=None)\n",
    "    parser.add_argument('--max_timesteps', help='maximum timestep', type=int, default=100000000)\n",
    "    parser.add_argument('--lr', dest='lr', type=float, default=0.00025)\n",
    "    parser.add_argument('--lr_decay', action='store_true', help='decay learning rate')\n",
    "    parser.add_argument('--gamma', help='discount_factor', type=float, default=0.99)\n",
    "    parser.add_argument('--warmup_mem', type=int, help='warmup memory size', default=1000)\n",
    "    parser.add_argument('--frame_skip', type=int, help='number of frames to skip for each action', default=3)\n",
    "    parser.add_argument('--frame_stack', type=int, help='number of frames to stack', default=4)\n",
    "    parser.add_argument('--memory', help='memory size', type=int, default=1000000)\n",
    "    parser.add_argument('--initial_epsilon', '-ie', help='initial_epsilon', type=float, default=0.5)\n",
    "    parser.add_argument('--final_epsilon', '-fe', help='final_epsilon', type=float, default=0.05)\n",
    "    parser.add_argument('--max_epsilon_decay_steps', '-eds', help='maximum steps to decay epsilon', type=int, default=100000)\n",
    "    parser.add_argument('--max_grad_norm', type=float, default=None, help='maximum gradient norm')\n",
    "    parser.add_argument('--soft_update', '-su', action='store_true', help='soft update target network')\n",
    "    parser.add_argument('--double_q', '-dq', action='store_true', help='enabling double DQN')\n",
    "    parser.add_argument('--dueling_net', '-dn', action='store_true', help='enabling dueling network')\n",
    "    parser.add_argument('--test', action='store_true', help='test the trained model')\n",
    "    parser.add_argument('--tau', type=float, default=0.01, help='tau for soft target network update')\n",
    "    parser.add_argument('--hard_update_freq', '-huf', type=int, default=2000, help='hard target network update frequency')\n",
    "    parser.add_argument('--save_dir', type=str, default='./data')\n",
    "    parser.add_argument('--resume_step', '-rs', type=int, default=None)\n",
    "    return parser.parse_args()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "X3Rkxe4VwOmu"
   },
   "outputs": [],
   "source": [
    "#@title Set up constants for env and training\n",
    "test = False \n",
    "save_dir = './data'\n",
    "render = False\n",
    "max_episode = None\n",
    "max_timesteps = 100000000\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "539HMtWFwOmv"
   },
   "outputs": [],
   "source": [
    "#@title Augmentations\n",
    "'''\n",
    "color_jitter\n",
    "random_elastic_transform\n",
    "random_fisheye\n",
    "random_color_equalize\n",
    "random_gaussian_blur\n",
    "random_gaussian_noise\n",
    "random_horizontal_flip\n",
    "random_color_invert\n",
    "random_perspective_shift\n",
    "random_shift\n",
    "'''\n",
    "color_jitter = kaug.ColorJitter(\n",
    "        brightness=np.random.random(),\n",
    "        contrast=np.random.random(),\n",
    "        saturation=np.random.random(),\n",
    "        hue=np.random.random(),\n",
    "        p=0.5\n",
    "        )\n",
    "random_elastic_transform = kaug.RandomElasticTransform()\n",
    "random_fisheye = kaug.RandomFisheye(\n",
    "        center_x=torch.tensor([-.3, .3]).to(device),\n",
    "        center_y=torch.tensor([-.3, .3]).to(device),\n",
    "        gamma=torch.tensor([.9, 1.]).to(device),\n",
    "        )\n",
    "# need to divide by 255.0\n",
    "random_color_equalize = lambda obs: kaug.RandomEqualize()(obs / 255.) * 255\n",
    "random_gaussian_blur = kaug.RandomGaussianBlur(\n",
    "        kernel_size=(9, 9),\n",
    "        sigma = (5., 5.)\n",
    "        )\n",
    "random_gaussian_noise = kaug.RandomGaussianNoise()\n",
    "random_horizontal_flip = kaug.RandomHorizontalFlip()\n",
    "random_color_invert = kaug.RandomInvert()\n",
    "random_perspective_shift = kaug.RandomPerspective()\n",
    "get_random_shift = lambda h, w, shift_by: nn.Sequential(kaug.RandomCrop((h - shift_by, w - shift_by)), nn.ReplicationPad2d(20), kaug.RandomCrop((h - shift_by, w - shift_by)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "K78NVCz-oqHc"
   },
   "outputs": [],
   "source": [
    "def tie_weights(src, trg):\n",
    "    assert type(src) == type(trg)\n",
    "    trg.weight = src.weight\n",
    "    trg.bias = src.bias\n",
    "\n",
    "def conv_output_shape(h_w, kernel_size=1, stride=1, pad=0, dilation=1):\n",
    "    from math import floor\n",
    "    if type(kernel_size) is not tuple:\n",
    "        kernel_size = (kernel_size, kernel_size)\n",
    "    h = floor( ((h_w[0] + (2 * pad) - ( dilation * (kernel_size[0] - 1) ) - 1 )/ stride) + 1)\n",
    "    w = floor( ((h_w[1] + (2 * pad) - ( dilation * (kernel_size[1] - 1) ) - 1 )/ stride) + 1)\n",
    "    return h, w\n",
    "\n",
    "# for 84 x 84 inputs\n",
    "OUT_DIM = {2: 39, 4: 35, 6: 31}\n",
    "# for 64 x 64 inputs\n",
    "OUT_DIM_64 = {2: 29, 4: 25, 6: 21}\n",
    "\n",
    "''' TODO change the layer parameters ''' \n",
    "class PixelEncoder(nn.Module):\n",
    "    \"\"\"Convolutional encoder of pixels observations.\"\"\"\n",
    "    def __init__(self, obs_shape, feature_dim=50, num_layers=3, num_filters=64, output_logits=False):\n",
    "        super().__init__()\n",
    "\n",
    "        assert len(obs_shape) == 3\n",
    "        self.obs_shape = (obs_shape[2], obs_shape[0], obs_shape[1])\n",
    "        self.feature_dim = feature_dim\n",
    "        self.num_layers = num_layers\n",
    "        \n",
    "        # 160, 210, 3\n",
    "        self.conv1 = nn.Conv2d(3, 32, kernel_size=5, stride=5)\n",
    "        conv1_shape = conv_output_shape(self.obs_shape[1:], kernel_size=5, stride=5)\n",
    "        # Input to conv2: 32, 42, 32\n",
    "        self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)\n",
    "        conv2_shape = conv_output_shape(conv1_shape, kernel_size=4, stride=2)\n",
    "        # Input to conv3: 15, 20, 64\n",
    "        self.conv3 = nn.Conv2d(64, 64, kernel_size=4, stride=1)\n",
    "        conv3_shape = conv_output_shape(conv2_shape, kernel_size=4, stride=1)\n",
    "        # Output from conv3: 12, 17, 64\n",
    "\n",
    "        # out_dim = OUT_DIM_64[num_layers] if obs_shape[-1] == 64 else OUT_DIM[num_layers]\n",
    "        out_dims = conv3_shape\n",
    "        self.fc = nn.Linear(num_filters * out_dims[0] * out_dims[1], self.feature_dim)\n",
    "        self.ln = nn.LayerNorm(self.feature_dim)\n",
    "\n",
    "        self.outputs = dict()\n",
    "        self.output_logits = output_logits\n",
    "\n",
    "    def reparameterize(self, mu, logstd):\n",
    "        std = torch.exp(logstd)\n",
    "        eps = torch.randn_like(std)\n",
    "        return mu + eps * std\n",
    "\n",
    "    def forward_conv(self, obs):\n",
    "        self.outputs['obs'] = obs\n",
    "        conv1 = torch.relu(self.conv1(obs))\n",
    "        self.outputs['conv1'] = conv1\n",
    "        conv2 = torch.relu(self.conv2(conv1))\n",
    "        self.outputs['conv2'] = conv2\n",
    "        conv3 = torch.relu(self.conv3(conv2))\n",
    "        self.outputs['conv3'] = conv3\n",
    "\n",
    "        h = conv3.reshape(conv3.size(0), -1)\n",
    "        return h\n",
    "\n",
    "    def forward(self, obs, detach=False):\n",
    "        h = self.forward_conv(obs)\n",
    "\n",
    "        if detach:\n",
    "            h = h.detach()\n",
    "\n",
    "        h_fc = self.fc(h)\n",
    "        self.outputs['fc'] = h_fc\n",
    "\n",
    "        h_norm = self.ln(h_fc)\n",
    "        self.outputs['ln'] = h_norm\n",
    "\n",
    "        if self.output_logits:\n",
    "            out = h_norm\n",
    "        else:\n",
    "            out = torch.tanh(h_norm)\n",
    "            self.outputs['tanh'] = out\n",
    "\n",
    "        return out\n",
    "\n",
    "    def copy_conv_weights_from(self, source):\n",
    "        \"\"\"Tie convolutional layers\"\"\"\n",
    "        # only tie conv layers\n",
    "        for i in range(self.num_layers):\n",
    "            tie_weights(src=source.convs[i], trg=self.convs[i])\n",
    "\n",
    "    def log(self, L, step, log_freq):\n",
    "        if step % log_freq != 0:\n",
    "            return\n",
    "\n",
    "        for k, v in self.outputs.items():\n",
    "            L.log_histogram('train_encoder/%s_hist' % k, v, step)\n",
    "            if len(v.shape) > 2:\n",
    "                L.log_image('train_encoder/%s_img' % k, v[0], step)\n",
    "\n",
    "        for i in range(self.num_layers):\n",
    "            L.log_param('train_encoder/conv%s' % (i + 1), self.convs[i], step)\n",
    "        L.log_param('train_encoder/fc', self.fc, step)\n",
    "        L.log_param('train_encoder/ln', self.ln, step)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "0kIzyV1QwOmw"
   },
   "outputs": [],
   "source": [
    "def wrap_env(env, train=True):\n",
    "    suffix = 'train' if train else 'test'\n",
    "    monitor_dir = os.path.join(save_dir, 'monitor_%s' % suffix)\n",
    "    os.makedirs(monitor_dir, exist_ok=True)\n",
    "    if not train:\n",
    "        video_save_interval = 10\n",
    "        env = Monitor(env, directory=monitor_dir,\n",
    "                      video_callable=lambda episode_id: episode_id % video_save_interval == 0,\n",
    "                      force=True)\n",
    "    else:\n",
    "        if render:\n",
    "            if max_episode is not None:\n",
    "                video_save_interval = int(max_episode / 3)\n",
    "            else:\n",
    "                video_save_interval = int(max_timesteps / float(env._max_episode_steps) / 3)\n",
    "            env = Monitor(env, directory=monitor_dir,\n",
    "                          video_callable=lambda episode_id: episode_id % video_save_interval == 0,\n",
    "                          force=True)\n",
    "        else:\n",
    "            env = Monitor(env, directory=monitor_dir, video_callable=False, force=True)\n",
    "    return env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "mZgEGp83wOmw"
   },
   "outputs": [],
   "source": [
    "class ReplayMemory(object):\n",
    "    def __init__(self, max_epi_num=2000, max_epi_len=200, obs_shape=(210, 160)):\n",
    "        # capacity is the maximum number of steps in memory\n",
    "        self.max_epi_num = max_epi_num\n",
    "        self.max_epi_len = max_epi_len\n",
    "        # saves each tuple of (state, action, next state, reward)\n",
    "        self.capacity = 250 # self.max_epi_num * max_epi_len\n",
    "        self.idx = 0\n",
    "        self.obs_memory = np.zeros((self.capacity, *obs_shape, 3)) # deque(maxlen=self.max_epi_num * max_epi_len)\n",
    "        self.next_memory = np.zeros((self.capacity, *obs_shape, 3))\n",
    "        self.act_memory = np.zeros((self.capacity, 1))\n",
    "        self.reward_memory = np.zeros((self.capacity, 1))\n",
    "        self.is_av = False\n",
    "        self.current_epi = 0\n",
    "\n",
    "    def reset(self):\n",
    "        self.current_epi = 0\n",
    "        self.memory.clear()\n",
    "\n",
    "    ''' deprecated for tuple buffer '''\n",
    "    def create_new_epi(self):\n",
    "        pass\n",
    "\n",
    "    def remember(self, state, next_state, action, reward):\n",
    "        idx = self.idx % self.capacity\n",
    "        self.obs_memory[idx] = state.copy()\n",
    "        self.next_memory[idx] = next_state.copy()\n",
    "        self.act_memory[idx] = action\n",
    "        self.reward_memory[idx] = reward\n",
    "        self.idx += 1\n",
    "        \n",
    "        '''\n",
    "        if len(self.memory) < self.capacity:\n",
    "            new_sample = np.array([state, action, reward, next_state])\n",
    "            if len(self.memory) == 0:\n",
    "                self.memory = [new_sample]\n",
    "            else:\n",
    "                length = len(self.memory)\n",
    "                self.memory.append(new_sample)\n",
    "        '''\n",
    "                \n",
    "    # samples batch_size\n",
    "    def sample(self, batch_size):\n",
    "        if batch_size < self.idx:\n",
    "            idx = np.random.randint(0, min(self.idx, self.capacity) - 1, batch_size)\n",
    "            return self.obs_memory[idx], self.next_memory[idx], self.act_memory[idx], self.reward_memory[idx]\n",
    "        return self.obs_memory[:self.idx], self.next_memory[:self.idx], self.act_memory[:self.idx], self.reward_memory[:self.idx]\n",
    "\n",
    "    def size(self):\n",
    "        return self.idx\n",
    "\n",
    "    def is_available(self):\n",
    "        self.is_av = True\n",
    "        if self.idx <= 1:\n",
    "            self.is_av = False\n",
    "        return self.is_av\n",
    "\n",
    "    def print_info(self):\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "FmR5zvSVwOmw"
   },
   "outputs": [],
   "source": [
    "#@title Create a training conv agent\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class DQNetworkConv(nn.Module):\n",
    "    def __init__(self, in_channels, act_dim, dueling=False):\n",
    "        super(DQNetworkConv, self).__init__()\n",
    "        self.act_dim = act_dim\n",
    "        self.dueling = dueling\n",
    "        # 160, 210, 3 \n",
    "        self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=5, stride=5)\n",
    "        # Input to conv2: 32, 42, 32\n",
    "        self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)\n",
    "        # Input to conv3: 15, 20, 64\n",
    "        self.conv3 = nn.Conv2d(64, 64, kernel_size=4, stride=1)\n",
    "        # Output from conv3: 12, 17, 64\n",
    "        if self.dueling:\n",
    "            self.v_fc4 = nn.Linear(12 * 17 * 64, 512)\n",
    "            self.adv_fc4 = nn.Linear(12 * 17 * 64, 512)\n",
    "            self.v_fc5 = nn.Linear(512, 1)\n",
    "            self.adv_fc5 = nn.Linear(512, self.act_dim)\n",
    "        else:\n",
    "            self.fc4 = nn.Linear(12 * 17 * 64, 512)\n",
    "            self.fc5 = nn.Linear(512, self.act_dim)\n",
    "        self.parameters = (list(self.conv1.parameters())) + (list(self.conv2.parameters())) + (list(self.conv3.parameters())) + (list(self.fc4.parameters())) + (list(self.fc5.parameters()))\n",
    "\n",
    "    def forward(self, st):\n",
    "        out = F.relu(self.conv1(st))\n",
    "        out = F.relu(self.conv2(out))\n",
    "        out = F.relu(self.conv3(out))\n",
    "        out = out.view(out.size(0), -1)\n",
    "        if self.dueling:\n",
    "            val = F.relu(self.v_fc4(out))\n",
    "            adv = F.relu(self.adv_fc4(out))\n",
    "            val = self.v_fc5(val)\n",
    "            adv = self.adv_fc5(adv)\n",
    "            out = val.expand_as(adv) + adv - adv.mean(-1, keepdim=True).expand_as(adv)\n",
    "        else:\n",
    "            out = F.relu(self.fc4(out))\n",
    "            out = self.fc5(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "ZGsarcvPzoNm"
   },
   "outputs": [],
   "source": [
    "#@title Create a training FC agent\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class DQNetworkFC(nn.Module):\n",
    "    def __init__(self, z_dim, act_dim, dueling=False):\n",
    "        super(DQNetworkFC, self).__init__()\n",
    "        self.act_dim = act_dim\n",
    "        self.input_dim = z_dim \n",
    "        self.dueling = dueling\n",
    "        if self.dueling:\n",
    "            self.v_fc1 = nn.Linear(z_dim, 512)\n",
    "            self.adv_fc1 = nn.Linear(z_dim, 512)\n",
    "            self.v_fc2 = nn.Linear(512, 1)\n",
    "            self.adv_fc2 = nn.Linear(512, 256)\n",
    "            self.v_fc3 = nn.Linear(256, 1)\n",
    "            self.adv_fc3 = nn.Linear(256, self.act_dim)\n",
    "        else:\n",
    "            self.fc1 = nn.Linear(z_dim, 512)\n",
    "            self.fc2 = nn.Linear(512, 256)\n",
    "            self.fc3 = nn.Linear(256, self.act_dim)\n",
    "\n",
    "    def forward(self, st):\n",
    "        out = F.relu(self.fc1(st))\n",
    "        out = F.relu(self.fc2(out))\n",
    "        ''' Do we need a relu on the last layer if the output is probability over action space? '''\n",
    "        out = F.relu(self.fc3(out))\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "I5dCMf6CH99A"
   },
   "outputs": [],
   "source": [
    "def process_obs(obs, divide=False):\n",
    "    obs = torch.Tensor(obs / 255. if divide else obs)\n",
    "    if len(obs.shape) < 4:\n",
    "        obs = obs.unsqueeze(0)\n",
    "    obs = obs.permute(0, 3, 1, 2)\n",
    "    return obs.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "H6_wThKowOmx"
   },
   "outputs": [],
   "source": [
    "def take_action(env, action):\n",
    "    state, rew, done, _ = env.step(action)\n",
    "    obs = env.render(mode='rgb_array')\n",
    "    return obs, rew, done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "XL9D4MCxwOmx"
   },
   "outputs": [],
   "source": [
    "MAX_STEPS = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "9NKDLpKGJ2lB"
   },
   "outputs": [],
   "source": [
    "class CURL(nn.Module):\n",
    "    \"\"\"\n",
    "    CURL\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, obs_shape, z_dim, batch_size, encoder, output_type=\"continuous\", critic=None, critic_target=None):\n",
    "        super(CURL, self).__init__()\n",
    "        self.obs_shape = obs_shape\n",
    "        self.batch_size = batch_size\n",
    "\n",
    "        # self.encoder = critic.encoder\n",
    "        self.encoder = encoder \n",
    "\n",
    "        # self.encoder_target = critic_target.encoder \n",
    "        self.fc1 = nn.Linear(100, 50)\n",
    "        self.fc2 = nn.Linear(50, 1)\n",
    "\n",
    "        self.W = nn.Parameter(torch.rand(z_dim, z_dim))\n",
    "        self.output_type = output_type\n",
    "\n",
    "    def encode(self, x, detach=False, ema=False):\n",
    "        \"\"\"\n",
    "        Encoder: z_t = e(x_t)\n",
    "        :param x: x_t, x y coordinates\n",
    "        :return: z_t, value in r2\n",
    "        \"\"\"\n",
    "        if ema:\n",
    "            with torch.no_grad():\n",
    "                z_out = self.encoder_target(x)\n",
    "        else:\n",
    "            z_out = self.encoder(x)\n",
    "\n",
    "        if detach:\n",
    "            z_out = z_out.detach()\n",
    "        return z_out\n",
    "\n",
    "    def compute_logits(self, z_a, z_mod):\n",
    "        \"\"\"\n",
    "        Uses logits trick for CURL:\n",
    "        - compute (B,B) matrix z_a (W z_pos.T)\n",
    "        - positives are all diagonal elements\n",
    "        - negatives are all other elements\n",
    "        - to compute loss use multiclass cross entropy with identity matrix for labels\n",
    "        \"\"\"\n",
    "#         Wz = torch.matmul(self.W, z_mod.T)  # (z_dim,B)\n",
    "#         logits = torch.matmul(z_a, Wz)  # (B,B)\n",
    "#         logits = logits - torch.max(logits, 1)[0][:, None]\n",
    "#         return logits\n",
    "        input_zs = torch.cat([z_a, z_mod], 1)\n",
    "        logits = F.relu(self.fc1(input_zs))\n",
    "        logits = F.sigmoid(self.fc2(logits))\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "1wbEY4hTshF6"
   },
   "outputs": [],
   "source": [
    "#@title Generate a batch of negatively labelled examples given observations\n",
    "\n",
    "def generate_negatives(obs):\n",
    "    neg_idx = np.random.randint(len(obs), size=len(obs))\n",
    "    pos_idx = np.arange(len(obs))\n",
    "    resample = (neg_idx == pos_idx)\n",
    "    for (i, r) in enumerate(resample):\n",
    "        if r:\n",
    "            idx = neg_idx[i]\n",
    "        else:\n",
    "            idx = np.random.randint(0, len(obs), 1)[0]\n",
    "            while idx == i:\n",
    "                idx = np.random.randint(0, len(obs), 1)[0]\n",
    "        neg_idx[i] = idx\n",
    "    return (obs[neg_idx]).copy()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "x3KEWmwUwOmx"
   },
   "outputs": [],
   "source": [
    "#@title Create a training agent (wrapper for conv agent)\n",
    "\n",
    "GAMMA = 0.99\n",
    "\n",
    "class Agent(object):\n",
    "    def __init__(self, act_dim, in_channels=3, max_epi_num=50, max_epi_len=300, CURL=None, aug=None, conv_net=False, random_shift=None):\n",
    "        self.N_action = act_dim\n",
    "        self.max_epi_num = max_epi_num\n",
    "        self.max_epi_len = max_epi_len\n",
    "        ''' To decide when to copy weights to the target network '''\n",
    "        self.num_param_updates = 0\n",
    "        self.CURL = CURL\n",
    "        self.aug = aug\n",
    "        self.random_shift = random_shift\n",
    "        if conv_net:\n",
    "            self.conv_net = DQNetworkConv(in_channels, act_dim).to(device)\n",
    "            self.target = DQNetworkConv(in_channels, act_dim).to(device)\n",
    "        else:\n",
    "            ''' if using the encoder head for contrastive loss '''\n",
    "            self.conv_net = DQNetworkFC(self.CURL.encoder.feature_dim, act_dim).to(device)\n",
    "            self.target = DQNetworkFC(self.CURL.encoder.feature_dim, act_dim).to(device)\n",
    "        self.buffer = ReplayMemory(max_epi_num=self.max_epi_num, max_epi_len=self.max_epi_len, obs_shape=CURL.obs_shape[:2])\n",
    "        self.gamma = 0.99\n",
    "        self.loss_fn = torch.nn.MSELoss()\n",
    "        self.optimizer = torch.optim.Adam(\n",
    "            list(self.conv_net.parameters()) + \n",
    "            list(self.CURL.parameters()) + \n",
    "            list(self.CURL.encoder.parameters()), lr=1e-3)\n",
    "\n",
    "    def remember(self, state, action, reward, next_state):\n",
    "        self.buffer.remember(state, next_state, action, reward)\n",
    "\n",
    "    ''' Copy the weights to the target network every 100 updates '''\n",
    "    def train(self, batch_size=32, target_update_freq=100, use_encoder=True):\n",
    "        if self.buffer.is_available():\n",
    "            obs, next_obs, action_list, reward_list = self.buffer.sample(batch_size)\n",
    "            \n",
    "            ''' Pass through the encoder to get encodings\n",
    "             If also training the contrastive loss\n",
    "             include that here! \n",
    "             1. data augmentation to create pos and negative pairs\n",
    "             2. encoder \n",
    "             3. update encoder loss function (using a separate optimizer) or add to the loss computed below\n",
    "            '''\n",
    "\n",
    "            losses = []\n",
    "\n",
    "            # check if obs is a numpy or a torch tensor\n",
    "            if use_encoder:\n",
    "                obs_anchor = self.random_shift(process_obs(obs.copy()))\n",
    "                obs_pos = self.random_shift(process_obs(obs.copy()))\n",
    "                mixed_obs = generate_negatives(obs)\n",
    "                mixed_obs = process_obs(mixed_obs)\n",
    "                obs_neg = self.random_shift(mixed_obs)\n",
    "                if self.aug is not None:\n",
    "                    obs_pos = self.aug(obs_pos)\n",
    "                    obs_neg = self.aug(obs_neg)\n",
    "                z_a = self.CURL.encode(obs_anchor)\n",
    "                z_pos = self.CURL.encode(obs_pos)\n",
    "                # Mix pairs to generate negative labels\n",
    "                z_neg = self.CURL.encode(obs_neg)\n",
    "                next_obs = self.random_shift(process_obs(next_obs.copy()))\n",
    "                z_next = self.CURL.encode(next_obs)\n",
    "\n",
    "                # logits = self.CURL.compute_logits(z_a, z_pos)\n",
    "                # labels = torch.arange(logits.shape[0]).long().to(device)\n",
    "                pos_logits = self.CURL.compute_logits(z_a, z_pos)\n",
    "                neg_logits = self.CURL.compute_logits(z_a, z_neg)\n",
    "                # [32, 32]\n",
    "                pos_labels = torch.ones((pos_logits.shape[0], 1))# .long()\n",
    "                neg_labels = torch.zeros((neg_logits.shape[0], 1))# .long() \n",
    "                # TODO: stack pos and neg logits and labels (double check dim)\n",
    "                logits = torch.cat([pos_logits, neg_logits], 0)\n",
    "                labels = torch.cat([pos_labels, neg_labels], 0).to(device)\n",
    "                \n",
    "                # pass into the loss function\n",
    "                # encoding_loss = nn.CrossEntropyLoss()(logits, labels)\n",
    "                encoding_loss = nn.BCELoss()(logits, labels)\n",
    "\n",
    "                ''' Combine encoding loss with rl loss below '''\n",
    "                losses.append(encoding_loss)\n",
    "\n",
    "                # Then pass that encoding through the conv_net to get Q value estimates\n",
    "                Qs = self.conv_net(z_a)\n",
    "                next_Qs = self.target(z_next).detach().max(1)[0]\n",
    "            \n",
    "            else:\n",
    "                ''' If not using the encoder, pass the obs directly to the CNN '''\n",
    "                obs = process_obs(obs)\n",
    "                # estimate current q values from observations\n",
    "                Qs = self.conv_net(obs)\n",
    "                # find next max q values based on next observations\n",
    "                next_Qs = self.target(next_obs).detach().max(1)[0]\n",
    "            \n",
    "            ''' find target q values ''' \n",
    "            next_Qs = next_Qs.cpu().numpy() \n",
    "            Qs = torch.gather(Qs, dim=1, index=torch.tensor(action_list, dtype=torch.int64).to(device)).float().to(device)\n",
    "            target_Qs = torch.tensor(reward_list.squeeze(-1) + GAMMA * next_Qs).float().to(device)\n",
    "            ''' try to set Qs equal to target_Qs '''\n",
    "            q_loss = self.loss_fn(Qs, target_Qs)\n",
    "            losses.append(q_loss)\n",
    "            \n",
    "            ''' Loss update for q network and encoder head '''\n",
    "            losses = torch.stack(losses).sum()\n",
    "            self.optimizer.zero_grad()\n",
    "            losses.backward()\n",
    "            self.optimizer.step()\n",
    "\n",
    "            self.num_param_updates += 1\n",
    "            if self.num_param_updates % target_update_freq == 0:\n",
    "                self.target.load_state_dict(self.conv_net.state_dict())\n",
    "\n",
    "    # TODO: check the sizes of inputs and outputs\n",
    "    def get_action(self, obs, epsilon, use_encoding=True):\n",
    "        ''' \n",
    "         If using an encoder, need to pass that thorugh the encoder\n",
    "         then use the encoding to pass through self.conv_net\n",
    "        '''\n",
    "        # obs = torch.tensor(obs)\n",
    "        if use_encoding:\n",
    "            obs = self.random_shift(process_obs(obs.copy()))\n",
    "            obs = self.CURL.encode(obs, detach=True)\n",
    "\n",
    "        # Dividing obs by 255 is handled in encoder forward (only needed for use_encoding=False)\n",
    "        if len(obs.shape) == 1:\n",
    "            obs = obs.unsqueeze(0)\n",
    "\n",
    "        # epsilon greedy for selecting which action to take\n",
    "        if random.random() > epsilon:\n",
    "            qs = self.conv_net(obs)\n",
    "            action = qs[0].argmax().data.item()\n",
    "        else:\n",
    "            action = random.randint(0, self.N_action-1)\n",
    "\n",
    "        return action\n",
    "\n",
    "def get_decay(epi_iter):\n",
    "    decay = math.pow(0.999, epi_iter)\n",
    "    if decay < 0.05:\n",
    "        decay = 0.05\n",
    "    return decay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "1WWwX8wewOmy"
   },
   "outputs": [],
   "source": [
    "def main(aug=None, train_curve_filename_prefix=\"default\"):\n",
    "    env = gym.make('CartPole-v1')\n",
    "    env.reset()\n",
    "    max_epi_iter = 1000\n",
    "    max_MC_iter = 200\n",
    "    obs = env.render(mode='rgb_array')\n",
    "    obs_shape = obs.shape\n",
    "    shift_by = 20\n",
    "    random_shift = get_random_shift(*obs_shape[:2], shift_by)\n",
    "    cropped_obs_shape = (obs_shape[0] - shift_by, obs_shape[1] - shift_by, obs_shape[2])\n",
    "    encoder = PixelEncoder(obs_shape=cropped_obs_shape, feature_dim=50, num_layers=2, num_filters=64, output_logits=False).to(device)\n",
    "    CURL_encoder = CURL(obs_shape=obs_shape, z_dim=50, batch_size=1, encoder=encoder, output_type=\"continuous\").to(device)\n",
    "    agent = Agent(act_dim=env.action_space.n, max_epi_num=max_epi_iter, max_epi_len=max_MC_iter, CURL=CURL_encoder, aug=aug, random_shift=random_shift)\n",
    "    train_curve = []\n",
    "    steps_curve = []\n",
    "    for epi_iter in range(max_epi_iter):\n",
    "        random.seed()\n",
    "        env.reset()\n",
    "        obs = env.render(mode='rgb_array')\n",
    "        returns = 0.0\n",
    "        steps = 1\n",
    "        for MC_iter in range(max_MC_iter):\n",
    "            action = agent.get_action(obs, get_decay(epi_iter))\n",
    "            next_obs, reward, done = take_action(env, action)\n",
    "            returns += reward * agent.gamma ** (MC_iter)\n",
    "            agent.remember(obs, action, reward, next_obs)\n",
    "            obs = next_obs.copy()\n",
    "            if done or MC_iter >= max_MC_iter-1:\n",
    "                agent.buffer.create_new_epi()\n",
    "                steps = MC_iter\n",
    "                break\n",
    "        print('Episode', epi_iter, 'returns', returns, 'after', steps, 'timesteps')\n",
    "        if epi_iter % 1 == 0:\n",
    "            train_curve.append(returns)\n",
    "            steps_curve.append(steps)\n",
    "        if epi_iter % 100 == 0:\n",
    "            print(f\"Saving at episode {epi_iter}\")\n",
    "            np.save(f'{train_curve_filename_prefix}_curl_cartpole_{max_MC_iter}MC_{max_epi_iter}E_returns', np.array(train_curve))\n",
    "            np.save(f'{train_curve_filename_prefix}_curl_cartpole_{max_MC_iter}MC_{max_epi_iter}E_steps', np.array(steps_curve))\n",
    "        if agent.buffer.is_available():\n",
    "            for _ in range(1):\n",
    "                agent.train()\n",
    "    env.close()\n",
    "    np.save(f'{train_curve_filename_prefix}_curl_cartpole_{max_MC_iter}MC_{max_epi_iter}E_returns', np.array(train_curve))\n",
    "    np.save(f'{train_curve_filename_prefix}_curl_cartpole_{max_MC_iter}MC_{max_epi_iter}E_steps', np.array(steps_curve))\n",
    "    print(train_curve)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 0 returns 34.434077942585596 after 41 timesteps\n",
      "Saving at episode 0\n",
      "Episode 1 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 2 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 3 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 4 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 5 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 6 returns 26.029962661171947 after 29 timesteps\n",
      "Episode 7 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 8 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 9 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 10 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 11 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 12 returns 31.744540498961264 after 37 timesteps\n",
      "Episode 13 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 14 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 15 returns 31.055091413092185 after 36 timesteps\n",
      "Episode 16 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 17 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 18 returns 33.10282414303193 after 39 timesteps\n",
      "Episode 19 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 20 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 21 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 22 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 23 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 24 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 25 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 26 returns 28.94467727277075 after 33 timesteps\n",
      "Episode 27 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 28 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 29 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 30 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 31 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 32 returns 32.42709509397165 after 38 timesteps\n",
      "Episode 33 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 34 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 35 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 36 returns 27.501966404214624 after 31 timesteps\n",
      "Episode 37 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 38 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 39 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 40 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 41 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 42 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 43 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 44 returns 39.499393286246324 after 49 timesteps\n",
      "Episode 45 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 46 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 47 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 48 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 49 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 50 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 51 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 52 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 53 returns 27.501966404214624 after 31 timesteps\n",
      "Episode 54 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 55 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 56 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 57 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 58 returns 30.35867819504261 after 35 timesteps\n",
      "Episode 59 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 60 returns 31.744540498961264 after 37 timesteps\n",
      "Episode 61 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 62 returns 35.73883979152814 after 43 timesteps\n",
      "Episode 63 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 64 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 65 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 66 returns 38.27098590577117 after 47 timesteps\n",
      "Episode 67 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 68 returns 27.501966404214624 after 31 timesteps\n",
      "Episode 69 returns 22.995685419484463 after 25 timesteps\n",
      "Episode 70 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 71 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 72 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 73 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 74 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 75 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 76 returns 38.27098590577117 after 47 timesteps\n",
      "Episode 77 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 78 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 79 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 80 returns 28.226946740172476 after 32 timesteps\n",
      "Episode 81 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 82 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 83 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 84 returns 26.029962661171947 after 29 timesteps\n",
      "Episode 85 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 86 returns 41.29632180625152 after 52 timesteps\n",
      "Episode 87 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 88 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 89 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 90 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 91 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 92 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 93 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 94 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 95 returns 26.029962661171947 after 29 timesteps\n",
      "Episode 96 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 97 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 98 returns 49.51141112129299 after 67 timesteps\n",
      "Episode 99 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 100 returns 35.08973716315974 after 42 timesteps\n",
      "Saving at episode 100\n",
      "Episode 101 returns 22.995685419484463 after 25 timesteps\n",
      "Episode 102 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 103 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 104 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 105 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 106 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 107 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 108 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 109 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 110 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 111 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 112 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 113 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 114 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 115 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 116 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 117 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 118 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 119 returns 28.226946740172476 after 32 timesteps\n",
      "Episode 120 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 121 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 122 returns 33.771795901601614 after 40 timesteps\n",
      "Episode 123 returns 33.10282414303193 after 39 timesteps\n",
      "Episode 124 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 125 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 126 returns 28.94467727277075 after 33 timesteps\n",
      "Episode 127 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 128 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 129 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 130 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 131 returns 36.38145139361286 after 44 timesteps\n",
      "Episode 132 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 133 returns 47.96594773496931 after 64 timesteps\n",
      "Episode 134 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 135 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 136 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 137 returns 22.995685419484463 after 25 timesteps\n",
      "Episode 138 returns 22.995685419484463 after 25 timesteps\n",
      "Episode 139 returns 22.995685419484463 after 25 timesteps\n",
      "Episode 140 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 141 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 142 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 143 returns 14.854222890512437 after 15 timesteps\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 144 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 145 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 146 returns 28.94467727277075 after 33 timesteps\n",
      "Episode 147 returns 26.029962661171947 after 29 timesteps\n",
      "Episode 148 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 149 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 150 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 151 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 152 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 153 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 154 returns 30.35867819504261 after 35 timesteps\n",
      "Episode 155 returns 30.35867819504261 after 35 timesteps\n",
      "Episode 156 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 157 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 158 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 159 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 160 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 161 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 162 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 163 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 164 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 165 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 166 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 167 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 168 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 169 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 170 returns 43.039879752284044 after 55 timesteps\n",
      "Episode 171 returns 32.42709509397165 after 38 timesteps\n",
      "Episode 172 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 173 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 174 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 175 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 176 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 177 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 178 returns 22.995685419484463 after 25 timesteps\n",
      "Episode 179 returns 28.226946740172476 after 32 timesteps\n",
      "Episode 180 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 181 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 182 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 183 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 184 returns 27.501966404214624 after 31 timesteps\n",
      "Episode 185 returns 26.769663034560228 after 30 timesteps\n",
      "Episode 186 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 187 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 188 returns 28.226946740172476 after 32 timesteps\n",
      "Episode 189 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 190 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 191 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 192 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 193 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 194 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 195 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 196 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 197 returns 45.28433576092384 after 59 timesteps\n",
      "Episode 198 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 199 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 200 returns 13.12541872310217 after 13 timesteps\n",
      "Saving at episode 200\n",
      "Episode 201 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 202 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 203 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 204 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 205 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 206 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 207 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 208 returns 37.017636879676736 after 45 timesteps\n",
      "Episode 209 returns 33.10282414303193 after 39 timesteps\n",
      "Episode 210 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 211 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 212 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 213 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 214 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 215 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 216 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 217 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 218 returns 25.28279056684035 after 28 timesteps\n",
      "Episode 219 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 220 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 221 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 222 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 223 returns 31.744540498961264 after 37 timesteps\n",
      "Episode 224 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 225 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 226 returns 26.769663034560228 after 30 timesteps\n",
      "Episode 227 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 228 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 229 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 230 returns 28.226946740172476 after 32 timesteps\n",
      "Episode 231 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 232 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 233 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 234 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 235 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 236 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 237 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 238 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 239 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 240 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 241 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 242 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 243 returns 30.35867819504261 after 35 timesteps\n",
      "Episode 244 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 245 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 246 returns 31.744540498961264 after 37 timesteps\n",
      "Episode 247 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 248 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 249 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 250 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 251 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 252 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 253 returns 28.226946740172476 after 32 timesteps\n",
      "Episode 254 returns 27.501966404214624 after 31 timesteps\n",
      "Episode 255 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 256 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 257 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 258 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 259 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 260 returns 31.055091413092185 after 36 timesteps\n",
      "Episode 261 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 262 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 263 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 264 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 265 returns 27.501966404214624 after 31 timesteps\n",
      "Episode 266 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 267 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 268 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 269 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 270 returns 26.029962661171947 after 29 timesteps\n",
      "Episode 271 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 272 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 273 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 274 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 275 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 276 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 277 returns 31.744540498961264 after 37 timesteps\n",
      "Episode 278 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 279 returns 26.769663034560228 after 30 timesteps\n",
      "Episode 280 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 281 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 282 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 283 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 284 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 285 returns 12.247897700103202 after 12 timesteps\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 286 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 287 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 288 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 289 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 290 returns 33.10282414303193 after 39 timesteps\n",
      "Episode 291 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 292 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 293 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 294 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 295 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 296 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 297 returns 47.96594773496931 after 64 timesteps\n",
      "Episode 298 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 299 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 300 returns 23.765728565289617 after 26 timesteps\n",
      "Saving at episode 300\n",
      "Episode 301 returns 22.995685419484463 after 25 timesteps\n",
      "Episode 302 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 303 returns 31.744540498961264 after 37 timesteps\n",
      "Episode 304 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 305 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 306 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 307 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 308 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 309 returns 22.995685419484463 after 25 timesteps\n",
      "Episode 310 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 311 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 312 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 313 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 314 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 315 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 316 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 317 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 318 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 319 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 320 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 321 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 322 returns 28.226946740172476 after 32 timesteps\n",
      "Episode 323 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 324 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 325 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 326 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 327 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 328 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 329 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 330 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 331 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 332 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 333 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 334 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 335 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 336 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 337 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 338 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 339 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 340 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 341 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 342 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 343 returns 27.501966404214624 after 31 timesteps\n",
      "Episode 344 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 345 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 346 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 347 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 348 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 349 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 350 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 351 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 352 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 353 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 354 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 355 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 356 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 357 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 358 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 359 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 360 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 361 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 362 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 363 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 364 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 365 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 366 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 367 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 368 returns 25.28279056684035 after 28 timesteps\n",
      "Episode 369 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 370 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 371 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 372 returns 30.35867819504261 after 35 timesteps\n",
      "Episode 373 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 374 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 375 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 376 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 377 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 378 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 379 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 380 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 381 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 382 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 383 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 384 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 385 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 386 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 387 returns 29.655230500043043 after 34 timesteps\n",
      "Episode 388 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 389 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 390 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 391 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 392 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 393 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 394 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 395 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 396 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 397 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 398 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 399 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 400 returns 13.12541872310217 after 13 timesteps\n",
      "Saving at episode 400\n",
      "Episode 401 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 402 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 403 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 404 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 405 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 406 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 407 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 408 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 409 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 410 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 411 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 412 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 413 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 414 returns 28.94467727277075 after 33 timesteps\n",
      "Episode 415 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 416 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 417 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 418 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 419 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 420 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 421 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 422 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 423 returns 27.501966404214624 after 31 timesteps\n",
      "Episode 424 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 425 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 426 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 427 returns 12.247897700103202 after 12 timesteps\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 428 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 429 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 430 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 431 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 432 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 433 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 434 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 435 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 436 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 437 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 438 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 439 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 440 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 441 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 442 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 443 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 444 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 445 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 446 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 447 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 448 returns 25.28279056684035 after 28 timesteps\n",
      "Episode 449 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 450 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 451 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 452 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 453 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 454 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 455 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 456 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 457 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 458 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 459 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 460 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 461 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 462 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 463 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 464 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 465 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 466 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 467 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 468 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 469 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 470 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 471 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 472 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 473 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 474 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 475 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 476 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 477 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 478 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 479 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 480 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 481 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 482 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 483 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 484 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 485 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 486 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 487 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 488 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 489 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 490 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 491 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 492 returns 22.995685419484463 after 25 timesteps\n",
      "Episode 493 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 494 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 495 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 496 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 497 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 498 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 499 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 500 returns 19.02721317787414 after 20 timesteps\n",
      "Saving at episode 500\n",
      "Episode 501 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 502 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 503 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 504 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 505 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 506 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 507 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 508 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 509 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 510 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 511 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 512 returns 26.029962661171947 after 29 timesteps\n",
      "Episode 513 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 514 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 515 returns 31.744540498961264 after 37 timesteps\n",
      "Episode 516 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 517 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 518 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 519 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 520 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 521 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 522 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 523 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 524 returns 31.744540498961264 after 37 timesteps\n",
      "Episode 525 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 526 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 527 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 528 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 529 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 530 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 531 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 532 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 533 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 534 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 535 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 536 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 537 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 538 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 539 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 540 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 541 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 542 returns 28.226946740172476 after 32 timesteps\n",
      "Episode 543 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 544 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 545 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 546 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 547 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 548 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 549 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 550 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 551 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 552 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 553 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 554 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 555 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 556 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 557 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 558 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 559 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 560 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 561 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 562 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 563 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 564 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 565 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 566 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 567 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 568 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 569 returns 14.854222890512437 after 15 timesteps\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 570 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 571 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 572 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 573 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 574 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 575 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 576 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 577 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 578 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 579 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 580 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 581 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 582 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 583 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 584 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 585 returns 28.94467727277075 after 33 timesteps\n",
      "Episode 586 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 587 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 588 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 589 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 590 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 591 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 592 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 593 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 594 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 595 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 596 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 597 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 598 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 599 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 600 returns 11.361512828387072 after 11 timesteps\n",
      "Saving at episode 600\n",
      "Episode 601 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 602 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 603 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 604 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 605 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 606 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 607 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 608 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 609 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 610 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 611 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 612 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 613 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 614 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 615 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 616 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 617 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 618 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 619 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 620 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 621 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 622 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 623 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 624 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 625 returns 31.744540498961264 after 37 timesteps\n",
      "Episode 626 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 627 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 628 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 629 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 630 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 631 returns 27.501966404214624 after 31 timesteps\n",
      "Episode 632 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 633 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 634 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 635 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 636 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 637 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 638 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 639 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 640 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 641 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 642 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 643 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 644 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 645 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 646 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 647 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 648 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 649 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 650 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 651 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 652 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 653 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 654 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 655 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 656 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 657 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 658 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 659 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 660 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 661 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 662 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 663 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 664 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 665 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 666 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 667 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 668 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 669 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 670 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 671 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 672 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 673 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 674 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 675 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 676 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 677 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 678 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 679 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 680 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 681 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 682 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 683 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 684 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 685 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 686 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 687 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 688 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 689 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 690 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 691 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 692 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 693 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 694 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 695 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 696 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 697 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 698 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 699 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 700 returns 10.466174574128356 after 10 timesteps\n",
      "Saving at episode 700\n",
      "Episode 701 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 702 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 703 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 704 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 705 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 706 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 707 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 708 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 709 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 710 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 711 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 712 returns 33.771795901601614 after 40 timesteps\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 713 returns 26.769663034560228 after 30 timesteps\n",
      "Episode 714 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 715 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 716 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 717 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 718 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 719 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 720 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 721 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 722 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 723 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 724 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 725 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 726 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 727 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 728 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 729 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 730 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 731 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 732 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 733 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 734 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 735 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 736 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 737 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 738 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 739 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 740 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 741 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 742 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 743 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 744 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 745 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 746 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 747 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 748 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 749 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 750 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 751 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 752 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 753 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 754 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 755 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 756 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 757 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 758 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 759 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 760 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 761 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 762 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 763 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 764 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 765 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 766 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 767 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 768 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 769 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 770 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 771 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 772 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 773 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 774 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 775 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 776 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 777 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 778 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 779 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 780 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 781 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 782 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 783 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 784 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 785 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 786 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 787 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 788 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 789 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 790 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 791 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 792 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 793 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 794 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 795 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 796 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 797 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 798 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 799 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 800 returns 9.561792499119552 after 9 timesteps\n",
      "Saving at episode 800\n",
      "Episode 801 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 802 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 803 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 804 returns 28.94467727277075 after 33 timesteps\n",
      "Episode 805 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 806 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 807 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 808 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 809 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 810 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 811 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 812 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 813 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 814 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 815 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 816 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 817 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 818 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 819 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 820 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 821 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 822 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 823 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 824 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 825 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 826 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 827 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 828 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 829 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 830 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 831 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 832 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 833 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 834 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 835 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 836 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 837 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 838 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 839 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 840 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 841 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 842 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 843 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 844 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 845 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 846 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 847 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 848 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 849 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 850 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 851 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 852 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 853 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 854 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 855 returns 12.247897700103202 after 12 timesteps\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 856 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 857 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 858 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 859 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 860 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 861 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 862 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 863 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 864 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 865 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 866 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 867 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 868 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 869 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 870 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 871 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 872 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 873 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 874 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 875 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 876 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 877 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 878 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 879 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 880 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 881 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 882 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 883 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 884 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 885 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 886 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 887 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 888 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 889 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 890 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 891 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 892 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 893 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 894 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 895 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 896 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 897 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 898 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 899 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 900 returns 9.561792499119552 after 9 timesteps\n",
      "Saving at episode 900\n",
      "Episode 901 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 902 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 903 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 904 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 905 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 906 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 907 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 908 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 909 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 910 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 911 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 912 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 913 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 914 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 915 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 916 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 917 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 918 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 919 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 920 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 921 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 922 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 923 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 924 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 925 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 926 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 927 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 928 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 929 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 930 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 931 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 932 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 933 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 934 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 935 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 936 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 937 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 938 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 939 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 940 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 941 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 942 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 943 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 944 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 945 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 946 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 947 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 948 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 949 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 950 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 951 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 952 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 953 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 954 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 955 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 956 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 957 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 958 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 959 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 960 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 961 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 962 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 963 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 964 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 965 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 966 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 967 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 968 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 969 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 970 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 971 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 972 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 973 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 974 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 975 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 976 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 977 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 978 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 979 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 980 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 981 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 982 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 983 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 984 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 985 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 986 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 987 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 988 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 989 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 990 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 991 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 992 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 993 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 994 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 995 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 996 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 997 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 998 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 999 returns 10.466174574128356 after 10 timesteps\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[34.434077942585596, 10.466174574128356, 17.383137616441324, 20.63857163563444, 19.836941046095397, 18.20930624027691, 26.029962661171947, 10.466174574128356, 20.63857163563444, 14.854222890512437, 10.466174574128356, 22.217864060085315, 31.744540498961264, 19.02721317787414, 17.383137616441324, 31.055091413092185, 18.20930624027691, 12.247897700103202, 33.10282414303193, 19.836941046095397, 13.12541872310217, 19.02721317787414, 7.72553055720799, 19.836941046095397, 11.361512828387072, 22.217864060085315, 28.94467727277075, 14.854222890512437, 13.12541872310217, 10.466174574128356, 12.247897700103202, 19.836941046095397, 32.42709509397165, 24.52807127963672, 16.548623854991238, 10.466174574128356, 27.501966404214624, 15.705680661607312, 11.361512828387072, 13.12541872310217, 18.20930624027691, 14.854222890512437, 16.548623854991238, 12.247897700103202, 39.499393286246324, 14.854222890512437, 15.705680661607312, 13.994164535871148, 19.02721317787414, 17.383137616441324, 11.361512828387072, 20.63857163563444, 17.383137616441324, 27.501966404214624, 13.994164535871148, 19.02721317787414, 7.72553055720799, 15.705680661607312, 30.35867819504261, 15.705680661607312, 31.744540498961264, 19.02721317787414, 35.73883979152814, 20.63857163563444, 19.836941046095397, 10.466174574128356, 38.27098590577117, 15.705680661607312, 27.501966404214624, 22.995685419484463, 16.548623854991238, 13.994164535871148, 13.12541872310217, 12.247897700103202, 12.247897700103202, 13.994164535871148, 38.27098590577117, 13.12541872310217, 17.383137616441324, 13.12541872310217, 28.226946740172476, 15.705680661607312, 17.383137616441324, 12.247897700103202, 26.029962661171947, 10.466174574128356, 41.29632180625152, 15.705680661607312, 12.247897700103202, 18.20930624027691, 13.994164535871148, 12.247897700103202, 10.466174574128356, 12.247897700103202, 12.247897700103202, 26.029962661171947, 21.432185919278098, 13.12541872310217, 49.51141112129299, 12.247897700103202, 35.08973716315974, 22.995685419484463, 10.466174574128356, 10.466174574128356, 13.12541872310217, 20.63857163563444, 18.20930624027691, 17.383137616441324, 16.548623854991238, 22.217864060085315, 17.383137616441324, 9.561792499119552, 18.20930624027691, 12.247897700103202, 15.705680661607312, 21.432185919278098, 13.994164535871148, 13.994164535871148, 15.705680661607312, 28.226946740172476, 11.361512828387072, 19.02721317787414, 33.771795901601614, 33.10282414303193, 11.361512828387072, 17.383137616441324, 28.94467727277075, 19.02721317787414, 18.20930624027691, 19.836941046095397, 23.765728565289617, 36.38145139361286, 15.705680661607312, 47.96594773496931, 13.12541872310217, 19.02721317787414, 17.383137616441324, 22.995685419484463, 22.995685419484463, 22.995685419484463, 20.63857163563444, 13.12541872310217, 8.64827525163591, 14.854222890512437, 19.836941046095397, 9.561792499119552, 28.94467727277075, 26.029962661171947, 10.466174574128356, 13.994164535871148, 12.247897700103202, 10.466174574128356, 13.994164535871148, 11.361512828387072, 30.35867819504261, 30.35867819504261, 16.548623854991238, 14.854222890512437, 10.466174574128356, 22.217864060085315, 21.432185919278098, 13.994164535871148, 16.548623854991238, 15.705680661607312, 20.63857163563444, 15.705680661607312, 17.383137616441324, 18.20930624027691, 12.247897700103202, 10.466174574128356, 43.039879752284044, 32.42709509397165, 10.466174574128356, 22.217864060085315, 14.854222890512437, 13.994164535871148, 23.765728565289617, 11.361512828387072, 22.995685419484463, 28.226946740172476, 23.765728565289617, 13.12541872310217, 10.466174574128356, 8.64827525163591, 27.501966404214624, 26.769663034560228, 13.12541872310217, 19.02721317787414, 28.226946740172476, 22.217864060085315, 17.383137616441324, 9.561792499119552, 15.705680661607312, 19.02721317787414, 14.854222890512437, 10.466174574128356, 13.12541872310217, 45.28433576092384, 12.247897700103202, 21.432185919278098, 13.12541872310217, 8.64827525163591, 15.705680661607312, 11.361512828387072, 23.765728565289617, 19.02721317787414, 19.836941046095397, 19.02721317787414, 37.017636879676736, 33.10282414303193, 17.383137616441324, 10.466174574128356, 8.64827525163591, 22.217864060085315, 15.705680661607312, 13.994164535871148, 13.994164535871148, 14.854222890512437, 25.28279056684035, 15.705680661607312, 10.466174574128356, 8.64827525163591, 13.12541872310217, 31.744540498961264, 11.361512828387072, 8.64827525163591, 26.769663034560228, 15.705680661607312, 21.432185919278098, 20.63857163563444, 28.226946740172476, 17.383137616441324, 17.383137616441324, 20.63857163563444, 24.52807127963672, 15.705680661607312, 19.02721317787414, 19.02721317787414, 24.52807127963672, 19.836941046095397, 21.432185919278098, 9.561792499119552, 19.02721317787414, 30.35867819504261, 11.361512828387072, 12.247897700103202, 31.744540498961264, 23.765728565289617, 10.466174574128356, 13.994164535871148, 19.836941046095397, 22.217864060085315, 13.12541872310217, 28.226946740172476, 27.501966404214624, 24.52807127963672, 16.548623854991238, 11.361512828387072, 17.383137616441324, 14.854222890512437, 31.055091413092185, 19.02721317787414, 13.994164535871148, 10.466174574128356, 22.217864060085315, 27.501966404214624, 14.854222890512437, 13.12541872310217, 24.52807127963672, 10.466174574128356, 26.029962661171947, 10.466174574128356, 13.12541872310217, 13.994164535871148, 16.548623854991238, 12.247897700103202, 16.548623854991238, 31.744540498961264, 23.765728565289617, 26.769663034560228, 14.854222890512437, 19.02721317787414, 19.02721317787414, 21.432185919278098, 23.765728565289617, 12.247897700103202, 13.994164535871148, 13.994164535871148, 19.02721317787414, 10.466174574128356, 33.10282414303193, 13.994164535871148, 15.705680661607312, 13.12541872310217, 9.561792499119552, 13.994164535871148, 21.432185919278098, 47.96594773496931, 19.836941046095397, 9.561792499119552, 23.765728565289617, 22.995685419484463, 14.854222890512437, 31.744540498961264, 12.247897700103202, 17.383137616441324, 12.247897700103202, 11.361512828387072, 22.217864060085315, 22.995685419484463, 11.361512828387072, 8.64827525163591, 13.12541872310217, 19.836941046095397, 9.561792499119552, 16.548623854991238, 19.02721317787414, 13.994164535871148, 12.247897700103202, 20.63857163563444, 11.361512828387072, 13.12541872310217, 28.226946740172476, 20.63857163563444, 8.64827525163591, 19.02721317787414, 13.12541872310217, 13.12541872310217, 12.247897700103202, 12.247897700103202, 12.247897700103202, 14.854222890512437, 11.361512828387072, 14.854222890512437, 7.72553055720799, 10.466174574128356, 22.217864060085315, 15.705680661607312, 14.854222890512437, 22.217864060085315, 12.247897700103202, 13.12541872310217, 13.12541872310217, 27.501966404214624, 14.854222890512437, 14.854222890512437, 14.854222890512437, 16.548623854991238, 15.705680661607312, 14.854222890512437, 19.02721317787414, 10.466174574128356, 15.705680661607312, 17.383137616441324, 10.466174574128356, 13.994164535871148, 19.02721317787414, 17.383137616441324, 19.836941046095397, 19.02721317787414, 23.765728565289617, 19.02721317787414, 10.466174574128356, 10.466174574128356, 10.466174574128356, 14.854222890512437, 9.561792499119552, 11.361512828387072, 25.28279056684035, 18.20930624027691, 11.361512828387072, 10.466174574128356, 30.35867819504261, 12.247897700103202, 13.12541872310217, 18.20930624027691, 16.548623854991238, 8.64827525163591, 13.12541872310217, 22.217864060085315, 11.361512828387072, 13.994164535871148, 9.561792499119552, 10.466174574128356, 8.64827525163591, 12.247897700103202, 13.12541872310217, 29.655230500043043, 15.705680661607312, 10.466174574128356, 22.217864060085315, 12.247897700103202, 19.02721317787414, 11.361512828387072, 22.217864060085315, 13.994164535871148, 8.64827525163591, 9.561792499119552, 8.64827525163591, 16.548623854991238, 13.12541872310217, 18.20930624027691, 12.247897700103202, 21.432185919278098, 13.994164535871148, 17.383137616441324, 9.561792499119552, 12.247897700103202, 10.466174574128356, 16.548623854991238, 17.383137616441324, 10.466174574128356, 10.466174574128356, 11.361512828387072, 28.94467727277075, 8.64827525163591, 15.705680661607312, 9.561792499119552, 15.705680661607312, 13.994164535871148, 13.994164535871148, 16.548623854991238, 18.20930624027691, 27.501966404214624, 13.12541872310217, 9.561792499119552, 10.466174574128356, 12.247897700103202, 12.247897700103202, 21.432185919278098, 9.561792499119552, 9.561792499119552, 7.72553055720799, 11.361512828387072, 17.383137616441324, 15.705680661607312, 21.432185919278098, 19.836941046095397, 14.854222890512437, 10.466174574128356, 19.02721317787414, 17.383137616441324, 21.432185919278098, 13.12541872310217, 17.383137616441324, 12.247897700103202, 22.217864060085315, 19.02721317787414, 25.28279056684035, 12.247897700103202, 12.247897700103202, 9.561792499119552, 11.361512828387072, 22.217864060085315, 12.247897700103202, 11.361512828387072, 11.361512828387072, 15.705680661607312, 12.247897700103202, 10.466174574128356, 22.217864060085315, 16.548623854991238, 9.561792499119552, 13.994164535871148, 17.383137616441324, 14.854222890512437, 11.361512828387072, 12.247897700103202, 14.854222890512437, 11.361512828387072, 10.466174574128356, 12.247897700103202, 11.361512828387072, 13.12541872310217, 16.548623854991238, 13.12541872310217, 8.64827525163591, 9.561792499119552, 12.247897700103202, 10.466174574128356, 15.705680661607312, 13.994164535871148, 13.994164535871148, 16.548623854991238, 9.561792499119552, 13.12541872310217, 13.12541872310217, 12.247897700103202, 13.12541872310217, 9.561792499119552, 19.836941046095397, 10.466174574128356, 22.995685419484463, 15.705680661607312, 16.548623854991238, 15.705680661607312, 18.20930624027691, 19.02721317787414, 13.994164535871148, 10.466174574128356, 19.02721317787414, 15.705680661607312, 10.466174574128356, 11.361512828387072, 19.836941046095397, 8.64827525163591, 19.836941046095397, 13.12541872310217, 21.432185919278098, 14.854222890512437, 9.561792499119552, 10.466174574128356, 26.029962661171947, 24.52807127963672, 10.466174574128356, 31.744540498961264, 9.561792499119552, 13.12541872310217, 11.361512828387072, 15.705680661607312, 15.705680661607312, 10.466174574128356, 9.561792499119552, 10.466174574128356, 31.744540498961264, 13.994164535871148, 9.561792499119552, 16.548623854991238, 13.12541872310217, 10.466174574128356, 13.994164535871148, 15.705680661607312, 13.12541872310217, 9.561792499119552, 11.361512828387072, 12.247897700103202, 11.361512828387072, 12.247897700103202, 11.361512828387072, 18.20930624027691, 9.561792499119552, 12.247897700103202, 28.226946740172476, 11.361512828387072, 19.836941046095397, 14.854222890512437, 13.12541872310217, 12.247897700103202, 19.836941046095397, 11.361512828387072, 13.994164535871148, 10.466174574128356, 15.705680661607312, 12.247897700103202, 11.361512828387072, 18.20930624027691, 13.12541872310217, 12.247897700103202, 14.854222890512437, 11.361512828387072, 12.247897700103202, 21.432185919278098, 11.361512828387072, 18.20930624027691, 15.705680661607312, 9.561792499119552, 19.02721317787414, 9.561792499119552, 9.561792499119552, 14.854222890512437, 15.705680661607312, 12.247897700103202, 14.854222890512437, 10.466174574128356, 13.12541872310217, 24.52807127963672, 11.361512828387072, 11.361512828387072, 10.466174574128356, 15.705680661607312, 9.561792499119552, 19.02721317787414, 20.63857163563444, 10.466174574128356, 9.561792499119552, 28.94467727277075, 10.466174574128356, 11.361512828387072, 12.247897700103202, 10.466174574128356, 9.561792499119552, 21.432185919278098, 11.361512828387072, 20.63857163563444, 9.561792499119552, 17.383137616441324, 18.20930624027691, 10.466174574128356, 12.247897700103202, 12.247897700103202, 11.361512828387072, 15.705680661607312, 12.247897700103202, 13.994164535871148, 17.383137616441324, 10.466174574128356, 14.854222890512437, 16.548623854991238, 14.854222890512437, 11.361512828387072, 13.12541872310217, 22.217864060085315, 9.561792499119552, 12.247897700103202, 8.64827525163591, 9.561792499119552, 18.20930624027691, 9.561792499119552, 16.548623854991238, 10.466174574128356, 10.466174574128356, 11.361512828387072, 10.466174574128356, 20.63857163563444, 13.994164535871148, 31.744540498961264, 18.20930624027691, 13.994164535871148, 9.561792499119552, 16.548623854991238, 12.247897700103202, 27.501966404214624, 8.64827525163591, 10.466174574128356, 9.561792499119552, 11.361512828387072, 10.466174574128356, 12.247897700103202, 10.466174574128356, 9.561792499119552, 11.361512828387072, 14.854222890512437, 10.466174574128356, 9.561792499119552, 8.64827525163591, 14.854222890512437, 10.466174574128356, 12.247897700103202, 10.466174574128356, 9.561792499119552, 8.64827525163591, 11.361512828387072, 13.994164535871148, 9.561792499119552, 10.466174574128356, 8.64827525163591, 12.247897700103202, 11.361512828387072, 8.64827525163591, 10.466174574128356, 10.466174574128356, 9.561792499119552, 12.247897700103202, 12.247897700103202, 8.64827525163591, 13.994164535871148, 10.466174574128356, 13.994164535871148, 14.854222890512437, 10.466174574128356, 16.548623854991238, 15.705680661607312, 13.12541872310217, 12.247897700103202, 9.561792499119552, 18.20930624027691, 15.705680661607312, 18.20930624027691, 18.20930624027691, 11.361512828387072, 9.561792499119552, 12.247897700103202, 13.994164535871148, 14.854222890512437, 13.994164535871148, 9.561792499119552, 7.72553055720799, 10.466174574128356, 13.12541872310217, 10.466174574128356, 12.247897700103202, 10.466174574128356, 8.64827525163591, 14.854222890512437, 23.765728565289617, 10.466174574128356, 11.361512828387072, 7.72553055720799, 10.466174574128356, 11.361512828387072, 10.466174574128356, 19.02721317787414, 11.361512828387072, 8.64827525163591, 13.12541872310217, 11.361512828387072, 19.836941046095397, 9.561792499119552, 11.361512828387072, 11.361512828387072, 9.561792499119552, 13.12541872310217, 33.771795901601614, 26.769663034560228, 10.466174574128356, 10.466174574128356, 9.561792499119552, 8.64827525163591, 8.64827525163591, 21.432185919278098, 13.994164535871148, 8.64827525163591, 10.466174574128356, 9.561792499119552, 19.02721317787414, 17.383137616441324, 9.561792499119552, 11.361512828387072, 9.561792499119552, 14.854222890512437, 18.20930624027691, 17.383137616441324, 11.361512828387072, 8.64827525163591, 9.561792499119552, 8.64827525163591, 13.994164535871148, 12.247897700103202, 14.854222890512437, 10.466174574128356, 23.765728565289617, 8.64827525163591, 12.247897700103202, 10.466174574128356, 8.64827525163591, 14.854222890512437, 12.247897700103202, 13.994164535871148, 10.466174574128356, 12.247897700103202, 13.12541872310217, 12.247897700103202, 11.361512828387072, 12.247897700103202, 12.247897700103202, 8.64827525163591, 8.64827525163591, 13.994164535871148, 8.64827525163591, 8.64827525163591, 13.994164535871148, 13.12541872310217, 9.561792499119552, 13.994164535871148, 13.12541872310217, 10.466174574128356, 10.466174574128356, 10.466174574128356, 11.361512828387072, 18.20930624027691, 12.247897700103202, 10.466174574128356, 11.361512828387072, 19.02721317787414, 21.432185919278098, 14.854222890512437, 9.561792499119552, 10.466174574128356, 22.217864060085315, 16.548623854991238, 10.466174574128356, 9.561792499119552, 10.466174574128356, 13.12541872310217, 18.20930624027691, 10.466174574128356, 11.361512828387072, 9.561792499119552, 13.994164535871148, 13.12541872310217, 14.854222890512437, 11.361512828387072, 13.12541872310217, 9.561792499119552, 10.466174574128356, 10.466174574128356, 11.361512828387072, 13.994164535871148, 11.361512828387072, 14.854222890512437, 9.561792499119552, 10.466174574128356, 13.12541872310217, 14.854222890512437, 28.94467727277075, 10.466174574128356, 7.72553055720799, 9.561792499119552, 7.72553055720799, 7.72553055720799, 15.705680661607312, 11.361512828387072, 14.854222890512437, 13.994164535871148, 11.361512828387072, 10.466174574128356, 13.994164535871148, 10.466174574128356, 9.561792499119552, 10.466174574128356, 13.12541872310217, 11.361512828387072, 9.561792499119552, 17.383137616441324, 12.247897700103202, 17.383137616441324, 13.12541872310217, 9.561792499119552, 8.64827525163591, 10.466174574128356, 10.466174574128356, 8.64827525163591, 11.361512828387072, 9.561792499119552, 9.561792499119552, 11.361512828387072, 8.64827525163591, 12.247897700103202, 13.12541872310217, 8.64827525163591, 12.247897700103202, 10.466174574128356, 13.994164535871148, 13.994164535871148, 12.247897700103202, 9.561792499119552, 13.12541872310217, 10.466174574128356, 9.561792499119552, 17.383137616441324, 12.247897700103202, 10.466174574128356, 10.466174574128356, 16.548623854991238, 9.561792499119552, 12.247897700103202, 18.20930624027691, 15.705680661607312, 12.247897700103202, 17.383137616441324, 11.361512828387072, 11.361512828387072, 12.247897700103202, 10.466174574128356, 12.247897700103202, 9.561792499119552, 14.854222890512437, 10.466174574128356, 8.64827525163591, 9.561792499119552, 13.994164535871148, 10.466174574128356, 9.561792499119552, 13.994164535871148, 8.64827525163591, 12.247897700103202, 11.361512828387072, 19.02721317787414, 8.64827525163591, 10.466174574128356, 13.12541872310217, 12.247897700103202, 11.361512828387072, 11.361512828387072, 9.561792499119552, 12.247897700103202, 13.12541872310217, 10.466174574128356, 9.561792499119552, 11.361512828387072, 13.994164535871148, 9.561792499119552, 10.466174574128356, 7.72553055720799, 15.705680661607312, 13.12541872310217, 11.361512828387072, 12.247897700103202, 10.466174574128356, 13.12541872310217, 9.561792499119552, 10.466174574128356, 14.854222890512437, 19.02721317787414, 11.361512828387072, 11.361512828387072, 10.466174574128356, 15.705680661607312, 10.466174574128356, 9.561792499119552, 14.854222890512437, 11.361512828387072, 13.994164535871148, 9.561792499119552, 11.361512828387072, 12.247897700103202, 8.64827525163591, 9.561792499119552, 9.561792499119552, 8.64827525163591, 10.466174574128356, 11.361512828387072, 12.247897700103202, 13.12541872310217, 13.12541872310217, 10.466174574128356, 10.466174574128356, 7.72553055720799, 7.72553055720799, 11.361512828387072, 12.247897700103202, 9.561792499119552, 13.994164535871148, 10.466174574128356, 9.561792499119552, 13.994164535871148, 9.561792499119552, 11.361512828387072, 13.12541872310217, 10.466174574128356, 13.12541872310217, 11.361512828387072, 8.64827525163591, 9.561792499119552, 13.12541872310217, 20.63857163563444, 15.705680661607312, 16.548623854991238, 9.561792499119552, 8.64827525163591, 8.64827525163591, 14.854222890512437, 10.466174574128356, 8.64827525163591, 11.361512828387072, 13.994164535871148, 7.72553055720799, 10.466174574128356, 13.12541872310217, 11.361512828387072, 8.64827525163591, 13.12541872310217, 15.705680661607312, 8.64827525163591, 9.561792499119552, 9.561792499119552, 12.247897700103202, 8.64827525163591, 10.466174574128356, 13.12541872310217, 13.994164535871148, 11.361512828387072, 8.64827525163591, 8.64827525163591, 9.561792499119552, 10.466174574128356, 8.64827525163591, 13.12541872310217, 8.64827525163591, 9.561792499119552, 10.466174574128356, 13.994164535871148, 7.72553055720799, 10.466174574128356, 9.561792499119552, 11.361512828387072, 13.12541872310217, 14.854222890512437, 10.466174574128356, 7.72553055720799, 8.64827525163591, 9.561792499119552, 13.12541872310217, 13.994164535871148, 9.561792499119552, 9.561792499119552, 8.64827525163591, 13.994164535871148, 9.561792499119552, 10.466174574128356]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 0 returns 16.548623854991238 after 17 timesteps\n",
      "Saving at episode 0\n",
      "Episode 1 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 2 returns 46.37317747928145 after 61 timesteps\n",
      "Episode 3 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 4 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 5 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 6 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 7 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 8 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 9 returns 28.226946740172476 after 32 timesteps\n",
      "Episode 10 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 11 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 12 returns 32.42709509397165 after 38 timesteps\n",
      "Episode 13 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 14 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 15 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 16 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 17 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 18 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 19 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 20 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 21 returns 33.10282414303193 after 39 timesteps\n",
      "Episode 22 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 23 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 24 returns 39.499393286246324 after 49 timesteps\n",
      "Episode 25 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 26 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 27 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 28 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 29 returns 22.995685419484463 after 25 timesteps\n",
      "Episode 30 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 31 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 32 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 33 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 34 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 35 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 36 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 37 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 38 returns 28.94467727277075 after 33 timesteps\n",
      "Episode 39 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 40 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 41 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 42 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 43 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 44 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 45 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 46 returns 37.017636879676736 after 45 timesteps\n",
      "Episode 47 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 48 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 49 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 50 returns 27.501966404214624 after 31 timesteps\n",
      "Episode 51 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 52 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 53 returns 27.501966404214624 after 31 timesteps\n",
      "Episode 54 returns 39.499393286246324 after 49 timesteps\n",
      "Episode 55 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 56 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 57 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 58 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 59 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 60 returns 40.10439935338386 after 50 timesteps\n",
      "Episode 61 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 62 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 63 returns 37.64746051087997 after 46 timesteps\n",
      "Episode 64 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 65 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 66 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 67 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 68 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 69 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 70 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 71 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 72 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 73 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 74 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 75 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 76 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 77 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 78 returns 28.226946740172476 after 32 timesteps\n",
      "Episode 79 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 80 returns 31.055091413092185 after 36 timesteps\n",
      "Episode 81 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 82 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 83 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 84 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 85 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 86 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 87 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 88 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 89 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 90 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 91 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 92 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 93 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 94 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 95 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 96 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 97 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 98 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 99 returns 25.28279056684035 after 28 timesteps\n",
      "Episode 100 returns 15.705680661607312 after 16 timesteps\n",
      "Saving at episode 100\n",
      "Episode 101 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 102 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 103 returns 25.28279056684035 after 28 timesteps\n",
      "Episode 104 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 105 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 106 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 107 returns 28.226946740172476 after 32 timesteps\n",
      "Episode 108 returns 25.28279056684035 after 28 timesteps\n",
      "Episode 109 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 110 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 111 returns 39.499393286246324 after 49 timesteps\n",
      "Episode 112 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 113 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 114 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 115 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 116 returns 26.769663034560228 after 30 timesteps\n",
      "Episode 117 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 118 returns 28.94467727277075 after 33 timesteps\n",
      "Episode 119 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 120 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 121 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 122 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 123 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 124 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 125 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 126 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 127 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 128 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 129 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 130 returns 34.434077942585596 after 41 timesteps\n",
      "Episode 131 returns 30.35867819504261 after 35 timesteps\n",
      "Episode 132 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 133 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 134 returns 42.46452500230711 after 54 timesteps\n",
      "Episode 135 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 136 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 137 returns 27.501966404214624 after 31 timesteps\n",
      "Episode 138 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 139 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 140 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 141 returns 25.28279056684035 after 28 timesteps\n",
      "Episode 142 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 143 returns 11.361512828387072 after 11 timesteps\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 144 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 145 returns 25.28279056684035 after 28 timesteps\n",
      "Episode 146 returns 31.744540498961264 after 37 timesteps\n",
      "Episode 147 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 148 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 149 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 150 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 151 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 152 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 153 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 154 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 155 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 156 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 157 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 158 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 159 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 160 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 161 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 162 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 163 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 164 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 165 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 166 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 167 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 168 returns 51.985854342857834 after 72 timesteps\n",
      "Episode 169 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 170 returns 33.10282414303193 after 39 timesteps\n",
      "Episode 171 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 172 returns 29.655230500043043 after 34 timesteps\n",
      "Episode 173 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 174 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 175 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 176 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 177 returns 22.995685419484463 after 25 timesteps\n",
      "Episode 178 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 179 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 180 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 181 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 182 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 183 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 184 returns 33.771795901601614 after 40 timesteps\n",
      "Episode 185 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 186 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 187 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 188 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 189 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 190 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 191 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 192 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 193 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 194 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 195 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 196 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 197 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 198 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 199 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 200 returns 10.466174574128356 after 10 timesteps\n",
      "Saving at episode 200\n",
      "Episode 201 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 202 returns 29.655230500043043 after 34 timesteps\n",
      "Episode 203 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 204 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 205 returns 30.35867819504261 after 35 timesteps\n",
      "Episode 206 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 207 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 208 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 209 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 210 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 211 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 212 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 213 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 214 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 215 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 216 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 217 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 218 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 219 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 220 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 221 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 222 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 223 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 224 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 225 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 226 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 227 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 228 returns 33.771795901601614 after 40 timesteps\n",
      "Episode 229 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 230 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 231 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 232 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 233 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 234 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 235 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 236 returns 28.94467727277075 after 33 timesteps\n",
      "Episode 237 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 238 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 239 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 240 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 241 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 242 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 243 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 244 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 245 returns 26.029962661171947 after 29 timesteps\n",
      "Episode 246 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 247 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 248 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 249 returns 28.226946740172476 after 32 timesteps\n",
      "Episode 250 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 251 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 252 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 253 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 254 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 255 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 256 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 257 returns 32.42709509397165 after 38 timesteps\n",
      "Episode 258 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 259 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 260 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 261 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 262 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 263 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 264 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 265 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 266 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 267 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 268 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 269 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 270 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 271 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 272 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 273 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 274 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 275 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 276 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 277 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 278 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 279 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 280 returns 22.995685419484463 after 25 timesteps\n",
      "Episode 281 returns 36.38145139361286 after 44 timesteps\n",
      "Episode 282 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 283 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 284 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 285 returns 13.12541872310217 after 13 timesteps\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 286 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 287 returns 25.28279056684035 after 28 timesteps\n",
      "Episode 288 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 289 returns 31.744540498961264 after 37 timesteps\n",
      "Episode 290 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 291 returns 22.995685419484463 after 25 timesteps\n",
      "Episode 292 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 293 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 294 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 295 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 296 returns 25.28279056684035 after 28 timesteps\n",
      "Episode 297 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 298 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 299 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 300 returns 10.466174574128356 after 10 timesteps\n",
      "Saving at episode 300\n",
      "Episode 301 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 302 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 303 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 304 returns 31.744540498961264 after 37 timesteps\n",
      "Episode 305 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 306 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 307 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 308 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 309 returns 22.995685419484463 after 25 timesteps\n",
      "Episode 310 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 311 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 312 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 313 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 314 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 315 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 316 returns 26.029962661171947 after 29 timesteps\n",
      "Episode 317 returns 26.769663034560228 after 30 timesteps\n",
      "Episode 318 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 319 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 320 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 321 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 322 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 323 returns 26.029962661171947 after 29 timesteps\n",
      "Episode 324 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 325 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 326 returns 29.655230500043043 after 34 timesteps\n",
      "Episode 327 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 328 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 329 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 330 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 331 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 332 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 333 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 334 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 335 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 336 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 337 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 338 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 339 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 340 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 341 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 342 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 343 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 344 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 345 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 346 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 347 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 348 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 349 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 350 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 351 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 352 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 353 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 354 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 355 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 356 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 357 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 358 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 359 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 360 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 361 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 362 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 363 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 364 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 365 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 366 returns 31.055091413092185 after 36 timesteps\n",
      "Episode 367 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 368 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 369 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 370 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 371 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 372 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 373 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 374 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 375 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 376 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 377 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 378 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 379 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 380 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 381 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 382 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 383 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 384 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 385 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 386 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 387 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 388 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 389 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 390 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 391 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 392 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 393 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 394 returns 27.501966404214624 after 31 timesteps\n",
      "Episode 395 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 396 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 397 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 398 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 399 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 400 returns 10.466174574128356 after 10 timesteps\n",
      "Saving at episode 400\n",
      "Episode 401 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 402 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 403 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 404 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 405 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 406 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 407 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 408 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 409 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 410 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 411 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 412 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 413 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 414 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 415 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 416 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 417 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 418 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 419 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 420 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 421 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 422 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 423 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 424 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 425 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 426 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 427 returns 9.561792499119552 after 9 timesteps\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 428 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 429 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 430 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 431 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 432 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 433 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 434 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 435 returns 22.995685419484463 after 25 timesteps\n",
      "Episode 436 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 437 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 438 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 439 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 440 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 441 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 442 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 443 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 444 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 445 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 446 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 447 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 448 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 449 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 450 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 451 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 452 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 453 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 454 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 455 returns 29.655230500043043 after 34 timesteps\n",
      "Episode 456 returns 28.226946740172476 after 32 timesteps\n",
      "Episode 457 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 458 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 459 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 460 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 461 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 462 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 463 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 464 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 465 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 466 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 467 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 468 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 469 returns 26.029962661171947 after 29 timesteps\n",
      "Episode 470 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 471 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 472 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 473 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 474 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 475 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 476 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 477 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 478 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 479 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 480 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 481 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 482 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 483 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 484 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 485 returns 35.08973716315974 after 42 timesteps\n",
      "Episode 486 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 487 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 488 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 489 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 490 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 491 returns 31.055091413092185 after 36 timesteps\n",
      "Episode 492 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 493 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 494 returns 33.10282414303193 after 39 timesteps\n",
      "Episode 495 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 496 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 497 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 498 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 499 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 500 returns 9.561792499119552 after 9 timesteps\n",
      "Saving at episode 500\n",
      "Episode 501 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 502 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 503 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 504 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 505 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 506 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 507 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 508 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 509 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 510 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 511 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 512 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 513 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 514 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 515 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 516 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 517 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 518 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 519 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 520 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 521 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 522 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 523 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 524 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 525 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 526 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 527 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 528 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 529 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 530 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 531 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 532 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 533 returns 35.08973716315974 after 42 timesteps\n",
      "Episode 534 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 535 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 536 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 537 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 538 returns 30.35867819504261 after 35 timesteps\n",
      "Episode 539 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 540 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 541 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 542 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 543 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 544 returns 26.029962661171947 after 29 timesteps\n",
      "Episode 545 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 546 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 547 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 548 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 549 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 550 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 551 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 552 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 553 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 554 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 555 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 556 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 557 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 558 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 559 returns 24.52807127963672 after 27 timesteps\n",
      "Episode 560 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 561 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 562 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 563 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 564 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 565 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 566 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 567 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 568 returns 22.995685419484463 after 25 timesteps\n",
      "Episode 569 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 570 returns 22.217864060085315 after 24 timesteps\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 571 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 572 returns 38.27098590577117 after 47 timesteps\n",
      "Episode 573 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 574 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 575 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 576 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 577 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 578 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 579 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 580 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 581 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 582 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 583 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 584 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 585 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 586 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 587 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 588 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 589 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 590 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 591 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 592 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 593 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 594 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 595 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 596 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 597 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 598 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 599 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 600 returns 16.548623854991238 after 17 timesteps\n",
      "Saving at episode 600\n",
      "Episode 601 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 602 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 603 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 604 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 605 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 606 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 607 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 608 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 609 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 610 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 611 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 612 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 613 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 614 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 615 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 616 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 617 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 618 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 619 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 620 returns 31.744540498961264 after 37 timesteps\n",
      "Episode 621 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 622 returns 26.769663034560228 after 30 timesteps\n",
      "Episode 623 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 624 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 625 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 626 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 627 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 628 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 629 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 630 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 631 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 632 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 633 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 634 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 635 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 636 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 637 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 638 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 639 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 640 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 641 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 642 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 643 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 644 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 645 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 646 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 647 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 648 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 649 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 650 returns 28.94467727277075 after 33 timesteps\n",
      "Episode 651 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 652 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 653 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 654 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 655 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 656 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 657 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 658 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 659 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 660 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 661 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 662 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 663 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 664 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 665 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 666 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 667 returns 28.94467727277075 after 33 timesteps\n",
      "Episode 668 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 669 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 670 returns 31.055091413092185 after 36 timesteps\n",
      "Episode 671 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 672 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 673 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 674 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 675 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 676 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 677 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 678 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 679 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 680 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 681 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 682 returns 20.63857163563444 after 22 timesteps\n",
      "Episode 683 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 684 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 685 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 686 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 687 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 688 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 689 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 690 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 691 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 692 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 693 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 694 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 695 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 696 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 697 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 698 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 699 returns 26.769663034560228 after 30 timesteps\n",
      "Episode 700 returns 9.561792499119552 after 9 timesteps\n",
      "Saving at episode 700\n",
      "Episode 701 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 702 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 703 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 704 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 705 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 706 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 707 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 708 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 709 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 710 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 711 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 712 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 713 returns 13.994164535871148 after 14 timesteps\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 714 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 715 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 716 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 717 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 718 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 719 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 720 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 721 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 722 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 723 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 724 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 725 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 726 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 727 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 728 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 729 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 730 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 731 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 732 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 733 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 734 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 735 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 736 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 737 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 738 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 739 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 740 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 741 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 742 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 743 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 744 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 745 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 746 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 747 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 748 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 749 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 750 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 751 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 752 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 753 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 754 returns 26.029962661171947 after 29 timesteps\n",
      "Episode 755 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 756 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 757 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 758 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 759 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 760 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 761 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 762 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 763 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 764 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 765 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 766 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 767 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 768 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 769 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 770 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 771 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 772 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 773 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 774 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 775 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 776 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 777 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 778 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 779 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 780 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 781 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 782 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 783 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 784 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 785 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 786 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 787 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 788 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 789 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 790 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 791 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 792 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 793 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 794 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 795 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 796 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 797 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 798 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 799 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 800 returns 13.994164535871148 after 14 timesteps\n",
      "Saving at episode 800\n",
      "Episode 801 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 802 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 803 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 804 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 805 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 806 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 807 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 808 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 809 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 810 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 811 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 812 returns 19.836941046095397 after 21 timesteps\n",
      "Episode 813 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 814 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 815 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 816 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 817 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 818 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 819 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 820 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 821 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 822 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 823 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 824 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 825 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 826 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 827 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 828 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 829 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 830 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 831 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 832 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 833 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 834 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 835 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 836 returns 21.432185919278098 after 23 timesteps\n",
      "Episode 837 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 838 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 839 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 840 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 841 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 842 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 843 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 844 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 845 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 846 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 847 returns 22.217864060085315 after 24 timesteps\n",
      "Episode 848 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 849 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 850 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 851 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 852 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 853 returns 18.20930624027691 after 19 timesteps\n",
      "Episode 854 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 855 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 856 returns 8.64827525163591 after 8 timesteps\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 857 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 858 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 859 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 860 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 861 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 862 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 863 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 864 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 865 returns 17.383137616441324 after 18 timesteps\n",
      "Episode 866 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 867 returns 28.226946740172476 after 32 timesteps\n",
      "Episode 868 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 869 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 870 returns 19.02721317787414 after 20 timesteps\n",
      "Episode 871 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 872 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 873 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 874 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 875 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 876 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 877 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 878 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 879 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 880 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 881 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 882 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 883 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 884 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 885 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 886 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 887 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 888 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 889 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 890 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 891 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 892 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 893 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 894 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 895 returns 7.72553055720799 after 7 timesteps\n",
      "Episode 896 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 897 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 898 returns 23.765728565289617 after 26 timesteps\n",
      "Episode 899 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 900 returns 15.705680661607312 after 16 timesteps\n",
      "Saving at episode 900\n",
      "Episode 901 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 902 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 903 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 904 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 905 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 906 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 907 returns 13.12541872310217 after 13 timesteps\n",
      "Episode 908 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 909 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 910 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 911 returns 15.705680661607312 after 16 timesteps\n",
      "Episode 912 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 913 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 914 returns 14.854222890512437 after 15 timesteps\n",
      "Episode 915 returns 8.64827525163591 after 8 timesteps\n",
      "Episode 916 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 917 returns 16.548623854991238 after 17 timesteps\n",
      "Episode 918 returns 9.561792499119552 after 9 timesteps\n",
      "Episode 919 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 920 returns 13.994164535871148 after 14 timesteps\n",
      "Episode 921 returns 10.466174574128356 after 10 timesteps\n",
      "Episode 922 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 923 returns 12.247897700103202 after 12 timesteps\n",
      "Episode 924 returns 11.361512828387072 after 11 timesteps\n",
      "Episode 925 returns 12.247897700103202 after 12 timesteps\n"
     ]
    }
   ],
   "source": [
    "main()\n",
    "main(color_jitter, \"color_jitter\")\n",
    "main(random_elastic_transform, \"random_elastic_transform\")\n",
    "main(random_fisheye, \"random_fisheye\")\n",
    "main(random_color_equalize, \"random_color_equalize\")\n",
    "main(random_gaussian_blur, \"random_gaussian_blur\")\n",
    "main(random_gaussian_noise, \"random_gaussian_noise\")\n",
    "main(random_horizontal_flip, \"random_horizontal_flip\")\n",
    "main(random_color_invert, \"random_color_invert\")\n",
    "main(random_perspective_shift, \"random_perspective_shift\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UkCfk2GjBNRl"
   },
   "source": [
    "Code references for DQN:\n",
    "\n",
    "https://github.com/taochenshh/dqn-pytorch\n",
    "\n",
    "https://github.com/transedward/pytorch-dqn (for sampling from replay buffer)\n",
    "\n",
    "CURL code: https://github.com/MishaLaskin/curl"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "sample_gym_env_v3.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python [conda env:cs230proj] *",
   "language": "python",
   "name": "conda-env-cs230proj-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}