policy_gradients_torch2019RLProject.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DQN FUNCTION\n",
    "import torch as T\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "\n",
    "class PolicyNetwork(nn.Module):\n",
    "    def __init__( self, lr, state_dim, num_layers, hidden_dim, num_actions):\n",
    "        #self, state_dim, action_dim, *, num_layers=3, hidden_dim=256\n",
    "        super(PolicyNetwork, self).__init__() #super constructor\n",
    "        self.state_dim=state_dim\n",
    "        self.lr=lr\n",
    "        self.num_layers = num_layers\n",
    "        self.hidden_dim = hidden_dim\n",
    "\n",
    "        \n",
    "        self.layerss = nn.ModuleList().append(nn.Linear(*state_dim, hidden_dim))\n",
    "        #self.linears.extend([nn.Linear(hidden_dim, hidden_dim) \n",
    "        for i in range(1, self.num_layers-1):\n",
    "            self.layerss.append(nn.Linear(hidden_dim, hidden_dim))\n",
    "        self.layerss.append(nn.Linear(hidden_dim, num_actions))        \n",
    "        \n",
    "        self.optimizer=optim.Adam(self.parameters(),lr=lr)\n",
    "        \n",
    "        #self.device=T.device(CPU:0)\n",
    "        #self.to(self.device)\n",
    "        \n",
    "    def forward(self, observation):\n",
    "    \n",
    "        state1=torch.Tensor(observation)#.to(self.device)\n",
    "        x= nn.functional.relu(self.layerss[0](state1))\n",
    "        for i in range(1, self.num_layers-1):\n",
    "            x= nn.functional.relu(self.layerss[i](x))\n",
    "        x= self.layerss[i+1](x)\n",
    "        return x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent(object):\n",
    "    def __init__(self, lr, state_dim, gamma, num_layers, hidden_dim, num_actions):\n",
    "        self.gamma=gamma\n",
    "        self.action_memory=[]\n",
    "        self.reward_memory=[]\n",
    "        self.lr=lr\n",
    "        self.state_dim=state_dim\n",
    "        self.num_layers=num_layers\n",
    "        self.hidden_dim=hidden_dim\n",
    "        self.num_actions=num_actions\n",
    "        self.policy = PolicyNetwork( self.lr, self.state_dim, self.num_layers, self.hidden_dim, self.num_actions)\n",
    "        #self, lr, state_dim, *, num_layers=3, hidden_dim=256, num_actions\n",
    "    \n",
    "    def choose_action(self,observation):\n",
    "        probabilities=F.softmax(self.policy.forward(observation))\n",
    "        action_probabilities=T.distributions.Categorical(probabilities)\n",
    "        action=action_probabilities.sample()\n",
    "        log_probs=action_probabilities.log_prob(action)\n",
    "        self.action_memory.append(log_probs)\n",
    "        \n",
    "        return action.item()\n",
    "    \n",
    "    #def store_rewards(self, reward):\n",
    "    #    self.reward_memory.append(reward)\n",
    "        \n",
    "    def learn(self):\n",
    "        self.policy.optimizer.zero_grad()\n",
    "        G=np.zeros_like(self.reward_memory, dtype=np.float64)\n",
    "        for t in range(len(self.reward_memory)):\n",
    "            G_sum=0\n",
    "            discount=1\n",
    "            for k in range(t, len(self.reward_memory)):\n",
    "                G_sum += self.reward_memory[k]*discount\n",
    "                discount *= self.gamma\n",
    "            G[t]=G_sum# at end of step store reward in G array\n",
    "        \n",
    "        mean=np.mean(G)\n",
    "        \n",
    "        if np.std(G)>0:\n",
    "            std=np.std(G) \n",
    "        else:\n",
    "            std=1\n",
    "            \n",
    "        G=T.tensor((G-mean)/std, dtype=T.float) #std could be zero so above line\n",
    "        \n",
    "     \n",
    "        loss=0 #loss calculation\n",
    "        for g, logprob in zip(G,self.action_memory):   #log probability=action_memory\n",
    "            loss+= -g*logprob\n",
    "            \n",
    "        loss.backward()\n",
    "        self.policy.optimizer.step()\n",
    "        \n",
    "        self.action_memory=[]\n",
    "        self.reward_memory=[]\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:  0 score 0.000\n",
      "episode:  1 score 18.000\n",
      "episode:  2 score 18.000\n",
      "episode:  3 score 12.000\n",
      "episode:  4 score 14.000\n",
      "episode:  5 score 17.000\n",
      "episode:  6 score 14.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:15: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:  7 score 14.000\n",
      "episode:  8 score 14.000\n",
      "episode:  9 score 18.000\n",
      "episode:  10 score 14.000\n",
      "episode:  11 score 25.000\n",
      "episode:  12 score 19.000\n",
      "episode:  13 score 18.000\n",
      "episode:  14 score 22.000\n",
      "episode:  15 score 19.000\n",
      "episode:  16 score 18.000\n",
      "episode:  17 score 11.000\n",
      "episode:  18 score 14.000\n",
      "episode:  19 score 20.000\n",
      "episode:  20 score 11.000\n",
      "episode:  21 score 17.000\n",
      "episode:  22 score 28.000\n",
      "episode:  23 score 19.000\n",
      "episode:  24 score 12.000\n",
      "episode:  25 score 8.000\n",
      "episode:  26 score 21.000\n",
      "episode:  27 score 50.000\n",
      "episode:  28 score 17.000\n",
      "episode:  29 score 18.000\n",
      "episode:  30 score 35.000\n",
      "episode:  31 score 15.000\n",
      "episode:  32 score 16.000\n",
      "episode:  33 score 14.000\n",
      "episode:  34 score 25.000\n",
      "episode:  35 score 12.000\n",
      "episode:  36 score 13.000\n",
      "episode:  37 score 41.000\n",
      "episode:  38 score 21.000\n",
      "episode:  39 score 31.000\n",
      "episode:  40 score 30.000\n",
      "episode:  41 score 17.000\n",
      "episode:  42 score 14.000\n",
      "episode:  43 score 28.000\n",
      "episode:  44 score 104.000\n",
      "episode:  45 score 12.000\n",
      "episode:  46 score 102.000\n",
      "episode:  47 score 33.000\n",
      "episode:  48 score 23.000\n",
      "episode:  49 score 44.000\n",
      "episode:  50 score 30.000\n",
      "episode:  51 score 17.000\n",
      "episode:  52 score 45.000\n",
      "episode:  53 score 94.000\n",
      "episode:  54 score 20.000\n",
      "episode:  55 score 27.000\n",
      "episode:  56 score 31.000\n",
      "episode:  57 score 52.000\n",
      "episode:  58 score 44.000\n",
      "episode:  59 score 24.000\n",
      "episode:  60 score 28.000\n",
      "episode:  61 score 103.000\n",
      "episode:  62 score 99.000\n",
      "episode:  63 score 63.000\n",
      "episode:  64 score 44.000\n",
      "episode:  65 score 25.000\n",
      "episode:  66 score 71.000\n",
      "episode:  67 score 32.000\n",
      "episode:  68 score 149.000\n",
      "episode:  69 score 52.000\n",
      "episode:  70 score 111.000\n",
      "episode:  71 score 83.000\n",
      "episode:  72 score 33.000\n",
      "episode:  73 score 93.000\n",
      "episode:  74 score 17.000\n",
      "episode:  75 score 60.000\n",
      "episode:  76 score 72.000\n",
      "episode:  77 score 151.000\n",
      "episode:  78 score 235.000\n",
      "episode:  79 score 161.000\n",
      "episode:  80 score 59.000\n",
      "episode:  81 score 119.000\n",
      "episode:  82 score 36.000\n",
      "episode:  83 score 75.000\n",
      "episode:  84 score 69.000\n",
      "episode:  85 score 158.000\n",
      "episode:  86 score 79.000\n",
      "episode:  87 score 199.000\n",
      "episode:  88 score 97.000\n",
      "episode:  89 score 39.000\n",
      "episode:  90 score 86.000\n",
      "episode:  91 score 69.000\n",
      "episode:  92 score 46.000\n",
      "episode:  93 score 133.000\n",
      "episode:  94 score 70.000\n",
      "episode:  95 score 63.000\n",
      "episode:  96 score 90.000\n",
      "episode:  97 score 114.000\n",
      "episode:  98 score 193.000\n",
      "episode:  99 score 208.000\n",
      "episode:  100 score 76.000\n",
      "episode:  101 score 98.000\n",
      "episode:  102 score 200.000\n",
      "episode:  103 score 83.000\n",
      "episode:  104 score 107.000\n",
      "episode:  105 score 46.000\n",
      "episode:  106 score 48.000\n",
      "episode:  107 score 91.000\n",
      "episode:  108 score 57.000\n",
      "episode:  109 score 27.000\n",
      "episode:  110 score 148.000\n",
      "episode:  111 score 143.000\n",
      "episode:  112 score 119.000\n",
      "episode:  113 score 137.000\n",
      "episode:  114 score 104.000\n",
      "episode:  115 score 177.000\n",
      "episode:  116 score 382.000\n",
      "episode:  117 score 232.000\n",
      "episode:  118 score 160.000\n",
      "episode:  119 score 224.000\n",
      "episode:  120 score 143.000\n",
      "episode:  121 score 27.000\n",
      "episode:  122 score 111.000\n",
      "episode:  123 score 64.000\n",
      "episode:  124 score 47.000\n",
      "episode:  125 score 52.000\n",
      "episode:  126 score 52.000\n",
      "episode:  127 score 35.000\n",
      "episode:  128 score 29.000\n",
      "episode:  129 score 38.000\n",
      "episode:  130 score 29.000\n",
      "episode:  131 score 29.000\n",
      "episode:  132 score 35.000\n",
      "episode:  133 score 26.000\n",
      "episode:  134 score 24.000\n",
      "episode:  135 score 28.000\n",
      "episode:  136 score 32.000\n",
      "episode:  137 score 24.000\n",
      "episode:  138 score 23.000\n",
      "episode:  139 score 35.000\n",
      "episode:  140 score 28.000\n",
      "episode:  141 score 24.000\n",
      "episode:  142 score 20.000\n",
      "episode:  143 score 24.000\n",
      "episode:  144 score 29.000\n",
      "episode:  145 score 30.000\n",
      "episode:  146 score 24.000\n",
      "episode:  147 score 23.000\n",
      "episode:  148 score 17.000\n",
      "episode:  149 score 24.000\n",
      "episode:  150 score 11.000\n",
      "episode:  151 score 22.000\n",
      "episode:  152 score 16.000\n",
      "episode:  153 score 27.000\n",
      "episode:  154 score 15.000\n",
      "episode:  155 score 17.000\n",
      "episode:  156 score 18.000\n",
      "episode:  157 score 34.000\n",
      "episode:  158 score 21.000\n",
      "episode:  159 score 19.000\n",
      "episode:  160 score 20.000\n",
      "episode:  161 score 26.000\n",
      "episode:  162 score 23.000\n",
      "episode:  163 score 26.000\n",
      "episode:  164 score 25.000\n",
      "episode:  165 score 35.000\n",
      "episode:  166 score 43.000\n",
      "episode:  167 score 24.000\n",
      "episode:  168 score 29.000\n",
      "episode:  169 score 39.000\n",
      "episode:  170 score 31.000\n",
      "episode:  171 score 23.000\n",
      "episode:  172 score 34.000\n",
      "episode:  173 score 28.000\n",
      "episode:  174 score 32.000\n",
      "episode:  175 score 44.000\n",
      "episode:  176 score 25.000\n",
      "episode:  177 score 38.000\n",
      "episode:  178 score 28.000\n",
      "episode:  179 score 30.000\n",
      "episode:  180 score 27.000\n",
      "episode:  181 score 34.000\n",
      "episode:  182 score 53.000\n",
      "episode:  183 score 49.000\n",
      "episode:  184 score 41.000\n",
      "episode:  185 score 37.000\n",
      "episode:  186 score 33.000\n",
      "episode:  187 score 59.000\n",
      "episode:  188 score 48.000\n",
      "episode:  189 score 66.000\n",
      "episode:  190 score 43.000\n",
      "episode:  191 score 71.000\n",
      "episode:  192 score 57.000\n",
      "episode:  193 score 56.000\n",
      "episode:  194 score 50.000\n",
      "episode:  195 score 243.000\n",
      "episode:  196 score 74.000\n",
      "episode:  197 score 126.000\n",
      "episode:  198 score 127.000\n",
      "episode:  199 score 128.000\n",
      "episode:  200 score 192.000\n",
      "episode:  201 score 181.000\n",
      "episode:  202 score 195.000\n",
      "episode:  203 score 209.000\n",
      "episode:  204 score 83.000\n",
      "episode:  205 score 80.000\n",
      "episode:  206 score 113.000\n",
      "episode:  207 score 118.000\n",
      "episode:  208 score 168.000\n",
      "episode:  209 score 80.000\n",
      "episode:  210 score 69.000\n",
      "episode:  211 score 75.000\n",
      "episode:  212 score 92.000\n",
      "episode:  213 score 146.000\n",
      "episode:  214 score 72.000\n",
      "episode:  215 score 145.000\n",
      "episode:  216 score 151.000\n",
      "episode:  217 score 146.000\n",
      "episode:  218 score 157.000\n",
      "episode:  219 score 131.000\n",
      "episode:  220 score 212.000\n",
      "episode:  221 score 399.000\n",
      "episode:  222 score 159.000\n",
      "episode:  223 score 175.000\n",
      "episode:  224 score 99.000\n",
      "episode:  225 score 193.000\n",
      "episode:  226 score 151.000\n",
      "episode:  227 score 120.000\n",
      "episode:  228 score 251.000\n",
      "episode:  229 score 180.000\n",
      "episode:  230 score 204.000\n",
      "episode:  231 score 171.000\n",
      "episode:  232 score 160.000\n",
      "episode:  233 score 154.000\n",
      "episode:  234 score 183.000\n",
      "episode:  235 score 198.000\n",
      "episode:  236 score 141.000\n",
      "episode:  237 score 174.000\n",
      "episode:  238 score 123.000\n",
      "episode:  239 score 116.000\n",
      "episode:  240 score 140.000\n",
      "episode:  241 score 125.000\n",
      "episode:  242 score 111.000\n",
      "episode:  243 score 88.000\n",
      "episode:  244 score 118.000\n",
      "episode:  245 score 78.000\n",
      "episode:  246 score 103.000\n",
      "episode:  247 score 99.000\n",
      "episode:  248 score 89.000\n",
      "episode:  249 score 108.000\n",
      "episode:  250 score 129.000\n",
      "episode:  251 score 133.000\n",
      "episode:  252 score 153.000\n",
      "episode:  253 score 165.000\n",
      "episode:  254 score 134.000\n",
      "episode:  255 score 196.000\n",
      "episode:  256 score 141.000\n",
      "episode:  257 score 157.000\n",
      "episode:  258 score 288.000\n",
      "episode:  259 score 196.000\n",
      "episode:  260 score 167.000\n",
      "episode:  261 score 143.000\n",
      "episode:  262 score 234.000\n",
      "episode:  263 score 200.000\n",
      "episode:  264 score 153.000\n",
      "episode:  265 score 234.000\n",
      "episode:  266 score 299.000\n",
      "episode:  267 score 311.000\n",
      "episode:  268 score 169.000\n",
      "episode:  269 score 233.000\n",
      "episode:  270 score 187.000\n",
      "episode:  271 score 275.000\n",
      "episode:  272 score 283.000\n",
      "episode:  273 score 275.000\n",
      "episode:  274 score 160.000\n",
      "episode:  275 score 172.000\n",
      "episode:  276 score 162.000\n",
      "episode:  277 score 133.000\n",
      "episode:  278 score 136.000\n",
      "episode:  279 score 127.000\n",
      "episode:  280 score 134.000\n",
      "episode:  281 score 235.000\n",
      "episode:  282 score 134.000\n",
      "episode:  283 score 136.000\n",
      "episode:  284 score 120.000\n",
      "episode:  285 score 161.000\n",
      "episode:  286 score 140.000\n",
      "episode:  287 score 130.000\n",
      "episode:  288 score 110.000\n",
      "episode:  289 score 116.000\n",
      "episode:  290 score 78.000\n",
      "episode:  291 score 54.000\n",
      "episode:  292 score 91.000\n",
      "episode:  293 score 76.000\n",
      "episode:  294 score 88.000\n",
      "episode:  295 score 69.000\n",
      "episode:  296 score 69.000\n",
      "episode:  297 score 77.000\n",
      "episode:  298 score 59.000\n",
      "episode:  299 score 54.000\n",
      "episode:  300 score 79.000\n",
      "episode:  301 score 91.000\n",
      "episode:  302 score 78.000\n",
      "episode:  303 score 55.000\n",
      "episode:  304 score 74.000\n",
      "episode:  305 score 70.000\n",
      "episode:  306 score 101.000\n",
      "episode:  307 score 90.000\n",
      "episode:  308 score 82.000\n",
      "episode:  309 score 88.000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:  310 score 85.000\n",
      "episode:  311 score 116.000\n",
      "episode:  312 score 77.000\n",
      "episode:  313 score 115.000\n",
      "episode:  314 score 110.000\n",
      "episode:  315 score 82.000\n",
      "episode:  316 score 109.000\n",
      "episode:  317 score 102.000\n",
      "episode:  318 score 136.000\n",
      "episode:  319 score 144.000\n",
      "episode:  320 score 123.000\n",
      "episode:  321 score 159.000\n",
      "episode:  322 score 500.000\n",
      "episode:  323 score 158.000\n",
      "episode:  324 score 122.000\n",
      "episode:  325 score 121.000\n",
      "episode:  326 score 353.000\n",
      "episode:  327 score 126.000\n",
      "episode:  328 score 246.000\n",
      "episode:  329 score 162.000\n",
      "episode:  330 score 226.000\n",
      "episode:  331 score 225.000\n",
      "episode:  332 score 231.000\n",
      "episode:  333 score 235.000\n",
      "episode:  334 score 208.000\n",
      "episode:  335 score 281.000\n",
      "episode:  336 score 299.000\n",
      "episode:  337 score 353.000\n",
      "episode:  338 score 332.000\n",
      "episode:  339 score 500.000\n",
      "episode:  340 score 289.000\n",
      "episode:  341 score 458.000\n",
      "episode:  342 score 500.000\n",
      "episode:  343 score 278.000\n",
      "episode:  344 score 201.000\n",
      "episode:  345 score 310.000\n",
      "episode:  346 score 140.000\n",
      "episode:  347 score 133.000\n",
      "episode:  348 score 144.000\n",
      "episode:  349 score 100.000\n",
      "episode:  350 score 126.000\n",
      "episode:  351 score 129.000\n",
      "episode:  352 score 112.000\n",
      "episode:  353 score 103.000\n",
      "episode:  354 score 137.000\n",
      "episode:  355 score 91.000\n",
      "episode:  356 score 70.000\n",
      "episode:  357 score 113.000\n",
      "episode:  358 score 80.000\n",
      "episode:  359 score 68.000\n",
      "episode:  360 score 77.000\n",
      "episode:  361 score 82.000\n",
      "episode:  362 score 76.000\n",
      "episode:  363 score 70.000\n",
      "episode:  364 score 85.000\n",
      "episode:  365 score 60.000\n",
      "episode:  366 score 79.000\n",
      "episode:  367 score 73.000\n",
      "episode:  368 score 55.000\n",
      "episode:  369 score 75.000\n",
      "episode:  370 score 68.000\n",
      "episode:  371 score 78.000\n",
      "episode:  372 score 94.000\n",
      "episode:  373 score 64.000\n",
      "episode:  374 score 64.000\n",
      "episode:  375 score 82.000\n",
      "episode:  376 score 54.000\n",
      "episode:  377 score 106.000\n",
      "episode:  378 score 43.000\n",
      "episode:  379 score 102.000\n",
      "episode:  380 score 76.000\n",
      "episode:  381 score 122.000\n",
      "episode:  382 score 91.000\n",
      "episode:  383 score 119.000\n",
      "episode:  384 score 90.000\n",
      "episode:  385 score 114.000\n",
      "episode:  386 score 82.000\n",
      "episode:  387 score 78.000\n",
      "episode:  388 score 87.000\n",
      "episode:  389 score 125.000\n",
      "episode:  390 score 149.000\n",
      "episode:  391 score 100.000\n",
      "episode:  392 score 92.000\n",
      "episode:  393 score 174.000\n",
      "episode:  394 score 152.000\n",
      "episode:  395 score 195.000\n",
      "episode:  396 score 205.000\n",
      "episode:  397 score 144.000\n",
      "episode:  398 score 149.000\n",
      "episode:  399 score 418.000\n",
      "episode:  400 score 204.000\n",
      "episode:  401 score 500.000\n",
      "episode:  402 score 126.000\n",
      "episode:  403 score 500.000\n",
      "episode:  404 score 170.000\n",
      "episode:  405 score 299.000\n",
      "episode:  406 score 319.000\n",
      "episode:  407 score 381.000\n",
      "episode:  408 score 302.000\n",
      "episode:  409 score 414.000\n",
      "episode:  410 score 500.000\n",
      "episode:  411 score 500.000\n",
      "episode:  412 score 500.000\n",
      "episode:  413 score 366.000\n",
      "episode:  414 score 294.000\n",
      "episode:  415 score 450.000\n",
      "episode:  416 score 500.000\n",
      "episode:  417 score 403.000\n",
      "episode:  418 score 222.000\n",
      "episode:  419 score 498.000\n",
      "episode:  420 score 331.000\n",
      "episode:  421 score 316.000\n",
      "episode:  422 score 267.000\n",
      "episode:  423 score 252.000\n",
      "episode:  424 score 342.000\n",
      "episode:  425 score 500.000\n",
      "episode:  426 score 359.000\n",
      "episode:  427 score 470.000\n",
      "episode:  428 score 500.000\n",
      "episode:  429 score 500.000\n",
      "episode:  430 score 242.000\n",
      "episode:  431 score 469.000\n",
      "episode:  432 score 462.000\n",
      "episode:  433 score 500.000\n",
      "episode:  434 score 216.000\n",
      "episode:  435 score 250.000\n",
      "episode:  436 score 457.000\n",
      "episode:  437 score 295.000\n",
      "episode:  438 score 354.000\n",
      "episode:  439 score 497.000\n",
      "episode:  440 score 259.000\n",
      "episode:  441 score 500.000\n",
      "episode:  442 score 471.000\n",
      "episode:  443 score 500.000\n",
      "episode:  444 score 149.000\n",
      "episode:  445 score 406.000\n",
      "episode:  446 score 361.000\n",
      "episode:  447 score 500.000\n",
      "episode:  448 score 304.000\n",
      "episode:  449 score 500.000\n",
      "episode:  450 score 457.000\n",
      "episode:  451 score 500.000\n",
      "episode:  452 score 167.000\n",
      "episode:  453 score 500.000\n",
      "episode:  454 score 309.000\n",
      "episode:  455 score 268.000\n",
      "episode:  456 score 500.000\n",
      "episode:  457 score 335.000\n",
      "episode:  458 score 178.000\n",
      "episode:  459 score 402.000\n",
      "episode:  460 score 500.000\n",
      "episode:  461 score 408.000\n",
      "episode:  462 score 163.000\n",
      "episode:  463 score 500.000\n",
      "episode:  464 score 291.000\n",
      "episode:  465 score 274.000\n",
      "episode:  466 score 215.000\n",
      "episode:  467 score 132.000\n",
      "episode:  468 score 295.000\n",
      "episode:  469 score 172.000\n",
      "episode:  470 score 500.000\n",
      "episode:  471 score 172.000\n",
      "episode:  472 score 500.000\n",
      "episode:  473 score 204.000\n",
      "episode:  474 score 500.000\n",
      "episode:  475 score 500.000\n",
      "episode:  476 score 500.000\n",
      "episode:  477 score 247.000\n",
      "episode:  478 score 291.000\n",
      "episode:  479 score 181.000\n",
      "episode:  480 score 178.000\n",
      "episode:  481 score 182.000\n",
      "episode:  482 score 182.000\n",
      "episode:  483 score 121.000\n",
      "episode:  484 score 148.000\n",
      "episode:  485 score 154.000\n",
      "episode:  486 score 129.000\n",
      "episode:  487 score 126.000\n",
      "episode:  488 score 156.000\n",
      "episode:  489 score 136.000\n",
      "episode:  490 score 121.000\n",
      "episode:  491 score 134.000\n",
      "episode:  492 score 134.000\n",
      "episode:  493 score 129.000\n",
      "episode:  494 score 131.000\n",
      "episode:  495 score 165.000\n",
      "episode:  496 score 110.000\n",
      "episode:  497 score 123.000\n",
      "episode:  498 score 140.000\n",
      "episode:  499 score 125.000\n",
      "episode:  500 score 127.000\n",
      "episode:  501 score 125.000\n",
      "episode:  502 score 102.000\n",
      "episode:  503 score 118.000\n",
      "episode:  504 score 109.000\n",
      "episode:  505 score 127.000\n",
      "episode:  506 score 145.000\n",
      "episode:  507 score 133.000\n",
      "episode:  508 score 123.000\n",
      "episode:  509 score 182.000\n",
      "episode:  510 score 140.000\n",
      "episode:  511 score 148.000\n",
      "episode:  512 score 181.000\n",
      "episode:  513 score 166.000\n",
      "episode:  514 score 258.000\n",
      "episode:  515 score 231.000\n",
      "episode:  516 score 214.000\n",
      "episode:  517 score 317.000\n",
      "episode:  518 score 438.000\n",
      "episode:  519 score 500.000\n",
      "episode:  520 score 500.000\n",
      "episode:  521 score 500.000\n",
      "episode:  522 score 500.000\n",
      "episode:  523 score 500.000\n",
      "episode:  524 score 500.000\n",
      "episode:  525 score 500.000\n",
      "episode:  526 score 500.000\n",
      "episode:  527 score 500.000\n",
      "episode:  528 score 500.000\n",
      "episode:  529 score 500.000\n",
      "episode:  530 score 500.000\n",
      "episode:  531 score 124.000\n",
      "episode:  532 score 500.000\n",
      "episode:  533 score 500.000\n",
      "episode:  534 score 500.000\n",
      "episode:  535 score 500.000\n",
      "episode:  536 score 500.000\n",
      "episode:  537 score 500.000\n",
      "episode:  538 score 330.000\n",
      "episode:  539 score 500.000\n",
      "episode:  540 score 500.000\n",
      "episode:  541 score 324.000\n",
      "episode:  542 score 500.000\n",
      "episode:  543 score 500.000\n",
      "episode:  544 score 500.000\n",
      "episode:  545 score 500.000\n",
      "episode:  546 score 500.000\n",
      "episode:  547 score 500.000\n",
      "episode:  548 score 500.000\n",
      "episode:  549 score 399.000\n",
      "episode:  550 score 322.000\n",
      "episode:  551 score 372.000\n",
      "episode:  552 score 293.000\n",
      "episode:  553 score 394.000\n",
      "episode:  554 score 303.000\n",
      "episode:  555 score 485.000\n",
      "episode:  556 score 500.000\n",
      "episode:  557 score 402.000\n",
      "episode:  558 score 500.000\n",
      "episode:  559 score 468.000\n",
      "episode:  560 score 317.000\n",
      "episode:  561 score 486.000\n",
      "episode:  562 score 500.000\n",
      "episode:  563 score 500.000\n",
      "episode:  564 score 500.000\n",
      "episode:  565 score 500.000\n",
      "episode:  566 score 500.000\n",
      "episode:  567 score 500.000\n",
      "episode:  568 score 500.000\n",
      "episode:  569 score 500.000\n",
      "episode:  570 score 500.000\n",
      "episode:  571 score 500.000\n",
      "episode:  572 score 500.000\n",
      "episode:  573 score 500.000\n",
      "episode:  574 score 441.000\n",
      "episode:  575 score 133.000\n",
      "episode:  576 score 322.000\n",
      "episode:  577 score 197.000\n",
      "episode:  578 score 315.000\n",
      "episode:  579 score 500.000\n",
      "episode:  580 score 327.000\n",
      "episode:  581 score 403.000\n",
      "episode:  582 score 387.000\n",
      "episode:  583 score 309.000\n",
      "episode:  584 score 149.000\n",
      "episode:  585 score 335.000\n",
      "episode:  586 score 263.000\n",
      "episode:  587 score 213.000\n",
      "episode:  588 score 97.000\n",
      "episode:  589 score 357.000\n",
      "episode:  590 score 157.000\n",
      "episode:  591 score 211.000\n",
      "episode:  592 score 228.000\n",
      "episode:  593 score 96.000\n",
      "episode:  594 score 101.000\n",
      "episode:  595 score 225.000\n",
      "episode:  596 score 226.000\n",
      "episode:  597 score 108.000\n",
      "episode:  598 score 93.000\n",
      "episode:  599 score 112.000\n",
      "episode:  600 score 91.000\n",
      "episode:  601 score 110.000\n",
      "episode:  602 score 167.000\n",
      "episode:  603 score 90.000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:  604 score 104.000\n",
      "episode:  605 score 106.000\n",
      "episode:  606 score 166.000\n",
      "episode:  607 score 109.000\n",
      "episode:  608 score 114.000\n",
      "episode:  609 score 105.000\n",
      "episode:  610 score 160.000\n",
      "episode:  611 score 237.000\n",
      "episode:  612 score 249.000\n",
      "episode:  613 score 97.000\n",
      "episode:  614 score 236.000\n",
      "episode:  615 score 231.000\n",
      "episode:  616 score 251.000\n",
      "episode:  617 score 273.000\n",
      "episode:  618 score 252.000\n",
      "episode:  619 score 320.000\n",
      "episode:  620 score 409.000\n",
      "episode:  621 score 430.000\n",
      "episode:  622 score 500.000\n",
      "episode:  623 score 327.000\n",
      "episode:  624 score 323.000\n",
      "episode:  625 score 496.000\n",
      "episode:  626 score 290.000\n",
      "episode:  627 score 500.000\n",
      "episode:  628 score 500.000\n",
      "episode:  629 score 500.000\n",
      "episode:  630 score 500.000\n",
      "episode:  631 score 500.000\n",
      "episode:  632 score 500.000\n",
      "episode:  633 score 500.000\n",
      "episode:  634 score 500.000\n",
      "episode:  635 score 500.000\n",
      "episode:  636 score 500.000\n",
      "episode:  637 score 500.000\n",
      "episode:  638 score 500.000\n",
      "episode:  639 score 500.000\n",
      "episode:  640 score 500.000\n",
      "episode:  641 score 500.000\n",
      "episode:  642 score 500.000\n",
      "episode:  643 score 500.000\n",
      "episode:  644 score 500.000\n",
      "episode:  645 score 500.000\n",
      "episode:  646 score 500.000\n",
      "episode:  647 score 500.000\n",
      "episode:  648 score 500.000\n",
      "episode:  649 score 500.000\n",
      "episode:  650 score 500.000\n",
      "episode:  651 score 500.000\n",
      "episode:  652 score 500.000\n",
      "episode:  653 score 500.000\n",
      "episode:  654 score 500.000\n",
      "episode:  655 score 500.000\n",
      "episode:  656 score 500.000\n",
      "episode:  657 score 500.000\n",
      "episode:  658 score 500.000\n",
      "episode:  659 score 500.000\n",
      "episode:  660 score 425.000\n",
      "episode:  661 score 500.000\n",
      "episode:  662 score 500.000\n",
      "episode:  663 score 500.000\n",
      "episode:  664 score 396.000\n",
      "episode:  665 score 500.000\n",
      "episode:  666 score 308.000\n",
      "episode:  667 score 422.000\n",
      "episode:  668 score 500.000\n",
      "episode:  669 score 409.000\n",
      "episode:  670 score 235.000\n",
      "episode:  671 score 324.000\n",
      "episode:  672 score 469.000\n",
      "episode:  673 score 500.000\n",
      "episode:  674 score 376.000\n",
      "episode:  675 score 500.000\n",
      "episode:  676 score 500.000\n",
      "episode:  677 score 455.000\n",
      "episode:  678 score 500.000\n",
      "episode:  679 score 500.000\n",
      "episode:  680 score 500.000\n",
      "episode:  681 score 280.000\n",
      "episode:  682 score 246.000\n",
      "episode:  683 score 215.000\n",
      "episode:  684 score 197.000\n",
      "episode:  685 score 157.000\n",
      "episode:  686 score 154.000\n",
      "episode:  687 score 139.000\n",
      "episode:  688 score 26.000\n",
      "episode:  689 score 136.000\n",
      "episode:  690 score 138.000\n",
      "episode:  691 score 126.000\n",
      "episode:  692 score 19.000\n",
      "episode:  693 score 17.000\n",
      "episode:  694 score 24.000\n",
      "episode:  695 score 111.000\n",
      "episode:  696 score 19.000\n",
      "episode:  697 score 20.000\n",
      "episode:  698 score 22.000\n",
      "episode:  699 score 15.000\n",
      "episode:  700 score 19.000\n",
      "episode:  701 score 23.000\n",
      "episode:  702 score 19.000\n",
      "episode:  703 score 21.000\n",
      "episode:  704 score 92.000\n",
      "episode:  705 score 21.000\n",
      "episode:  706 score 22.000\n",
      "episode:  707 score 25.000\n",
      "episode:  708 score 111.000\n",
      "episode:  709 score 117.000\n",
      "episode:  710 score 25.000\n",
      "episode:  711 score 127.000\n",
      "episode:  712 score 132.000\n",
      "episode:  713 score 132.000\n",
      "episode:  714 score 142.000\n",
      "episode:  715 score 151.000\n",
      "episode:  716 score 169.000\n",
      "episode:  717 score 183.000\n",
      "episode:  718 score 173.000\n",
      "episode:  719 score 214.000\n",
      "episode:  720 score 202.000\n",
      "episode:  721 score 215.000\n",
      "episode:  722 score 236.000\n",
      "episode:  723 score 255.000\n",
      "episode:  724 score 304.000\n",
      "episode:  725 score 349.000\n",
      "episode:  726 score 386.000\n",
      "episode:  727 score 377.000\n",
      "episode:  728 score 425.000\n",
      "episode:  729 score 424.000\n",
      "episode:  730 score 416.000\n",
      "episode:  731 score 354.000\n",
      "episode:  732 score 328.000\n",
      "episode:  733 score 355.000\n",
      "episode:  734 score 316.000\n",
      "episode:  735 score 348.000\n",
      "episode:  736 score 331.000\n",
      "episode:  737 score 334.000\n",
      "episode:  738 score 367.000\n",
      "episode:  739 score 393.000\n",
      "episode:  740 score 443.000\n",
      "episode:  741 score 500.000\n",
      "episode:  742 score 500.000\n",
      "episode:  743 score 500.000\n",
      "episode:  744 score 500.000\n",
      "episode:  745 score 500.000\n",
      "episode:  746 score 478.000\n",
      "episode:  747 score 470.000\n",
      "episode:  748 score 453.000\n",
      "episode:  749 score 467.000\n",
      "episode:  750 score 448.000\n",
      "episode:  751 score 457.000\n",
      "episode:  752 score 461.000\n",
      "episode:  753 score 500.000\n",
      "episode:  754 score 440.000\n",
      "episode:  755 score 484.000\n",
      "episode:  756 score 418.000\n",
      "episode:  757 score 355.000\n",
      "episode:  758 score 475.000\n",
      "episode:  759 score 500.000\n",
      "episode:  760 score 461.000\n",
      "episode:  761 score 500.000\n",
      "episode:  762 score 500.000\n",
      "episode:  763 score 500.000\n",
      "episode:  764 score 500.000\n",
      "episode:  765 score 500.000\n",
      "episode:  766 score 500.000\n",
      "episode:  767 score 500.000\n",
      "episode:  768 score 500.000\n",
      "episode:  769 score 500.000\n",
      "episode:  770 score 500.000\n",
      "episode:  771 score 500.000\n",
      "episode:  772 score 500.000\n",
      "episode:  773 score 500.000\n",
      "episode:  774 score 500.000\n",
      "episode:  775 score 500.000\n",
      "episode:  776 score 500.000\n",
      "episode:  777 score 500.000\n",
      "episode:  778 score 500.000\n",
      "episode:  779 score 500.000\n",
      "episode:  780 score 500.000\n",
      "episode:  781 score 500.000\n",
      "episode:  782 score 500.000\n",
      "episode:  783 score 500.000\n",
      "episode:  784 score 500.000\n",
      "episode:  785 score 500.000\n",
      "episode:  786 score 500.000\n",
      "episode:  787 score 500.000\n",
      "episode:  788 score 500.000\n",
      "episode:  789 score 500.000\n",
      "episode:  790 score 500.000\n",
      "episode:  791 score 500.000\n",
      "episode:  792 score 500.000\n",
      "episode:  793 score 500.000\n",
      "episode:  794 score 500.000\n",
      "episode:  795 score 500.000\n",
      "episode:  796 score 500.000\n",
      "episode:  797 score 500.000\n",
      "episode:  798 score 500.000\n",
      "episode:  799 score 500.000\n",
      "episode:  800 score 500.000\n",
      "episode:  801 score 500.000\n",
      "episode:  802 score 500.000\n",
      "episode:  803 score 500.000\n",
      "episode:  804 score 500.000\n",
      "episode:  805 score 500.000\n",
      "episode:  806 score 500.000\n",
      "episode:  807 score 500.000\n",
      "episode:  808 score 500.000\n",
      "episode:  809 score 500.000\n",
      "episode:  810 score 500.000\n",
      "episode:  811 score 500.000\n",
      "episode:  812 score 500.000\n",
      "episode:  813 score 500.000\n",
      "episode:  814 score 500.000\n",
      "episode:  815 score 500.000\n",
      "episode:  816 score 500.000\n",
      "episode:  817 score 500.000\n",
      "episode:  818 score 500.000\n",
      "episode:  819 score 500.000\n",
      "episode:  820 score 500.000\n",
      "episode:  821 score 500.000\n",
      "episode:  822 score 500.000\n",
      "episode:  823 score 500.000\n",
      "episode:  824 score 500.000\n",
      "episode:  825 score 500.000\n",
      "episode:  826 score 500.000\n",
      "episode:  827 score 500.000\n",
      "episode:  828 score 500.000\n",
      "episode:  829 score 500.000\n",
      "episode:  830 score 500.000\n",
      "episode:  831 score 500.000\n",
      "episode:  832 score 500.000\n",
      "episode:  833 score 500.000\n",
      "episode:  834 score 500.000\n",
      "episode:  835 score 500.000\n",
      "episode:  836 score 500.000\n",
      "episode:  837 score 500.000\n",
      "episode:  838 score 500.000\n",
      "episode:  839 score 500.000\n",
      "episode:  840 score 500.000\n",
      "episode:  841 score 500.000\n",
      "episode:  842 score 500.000\n",
      "episode:  843 score 500.000\n",
      "episode:  844 score 500.000\n",
      "episode:  845 score 500.000\n",
      "episode:  846 score 500.000\n",
      "episode:  847 score 500.000\n",
      "episode:  848 score 500.000\n",
      "episode:  849 score 500.000\n",
      "episode:  850 score 500.000\n",
      "episode:  851 score 500.000\n",
      "episode:  852 score 500.000\n",
      "episode:  853 score 500.000\n",
      "episode:  854 score 500.000\n",
      "episode:  855 score 500.000\n",
      "episode:  856 score 500.000\n",
      "episode:  857 score 500.000\n",
      "episode:  858 score 500.000\n",
      "episode:  859 score 500.000\n",
      "episode:  860 score 500.000\n",
      "episode:  861 score 500.000\n",
      "episode:  862 score 500.000\n",
      "episode:  863 score 500.000\n",
      "episode:  864 score 500.000\n",
      "episode:  865 score 500.000\n",
      "episode:  866 score 500.000\n",
      "episode:  867 score 500.000\n",
      "episode:  868 score 500.000\n",
      "episode:  869 score 500.000\n",
      "episode:  870 score 500.000\n",
      "episode:  871 score 500.000\n",
      "episode:  872 score 500.000\n",
      "episode:  873 score 500.000\n",
      "episode:  874 score 500.000\n",
      "episode:  875 score 500.000\n",
      "episode:  876 score 500.000\n",
      "episode:  877 score 500.000\n",
      "episode:  878 score 500.000\n",
      "episode:  879 score 500.000\n",
      "episode:  880 score 500.000\n",
      "episode:  881 score 500.000\n",
      "episode:  882 score 500.000\n",
      "episode:  883 score 500.000\n",
      "episode:  884 score 500.000\n",
      "episode:  885 score 500.000\n",
      "episode:  886 score 500.000\n",
      "episode:  887 score 500.000\n",
      "episode:  888 score 500.000\n",
      "episode:  889 score 500.000\n",
      "episode:  890 score 500.000\n",
      "episode:  891 score 500.000\n",
      "episode:  892 score 500.000\n",
      "episode:  893 score 500.000\n",
      "episode:  894 score 500.000\n",
      "episode:  895 score 500.000\n",
      "episode:  896 score 500.000\n",
      "episode:  897 score 500.000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:  898 score 500.000\n",
      "episode:  899 score 500.000\n",
      "episode:  900 score 500.000\n",
      "episode:  901 score 500.000\n",
      "episode:  902 score 500.000\n",
      "episode:  903 score 500.000\n",
      "episode:  904 score 500.000\n",
      "episode:  905 score 500.000\n",
      "episode:  906 score 500.000\n",
      "episode:  907 score 500.000\n",
      "episode:  908 score 500.000\n",
      "episode:  909 score 500.000\n",
      "episode:  910 score 500.000\n",
      "episode:  911 score 500.000\n",
      "episode:  912 score 500.000\n",
      "episode:  913 score 500.000\n",
      "episode:  914 score 500.000\n",
      "episode:  915 score 500.000\n",
      "episode:  916 score 500.000\n",
      "episode:  917 score 500.000\n",
      "episode:  918 score 500.000\n",
      "episode:  919 score 500.000\n",
      "episode:  920 score 500.000\n",
      "episode:  921 score 500.000\n",
      "episode:  922 score 500.000\n",
      "episode:  923 score 500.000\n",
      "episode:  924 score 500.000\n",
      "episode:  925 score 500.000\n",
      "episode:  926 score 500.000\n",
      "episode:  927 score 500.000\n",
      "episode:  928 score 500.000\n",
      "episode:  929 score 500.000\n",
      "episode:  930 score 500.000\n",
      "episode:  931 score 500.000\n",
      "episode:  932 score 500.000\n",
      "episode:  933 score 500.000\n",
      "episode:  934 score 500.000\n",
      "episode:  935 score 500.000\n",
      "episode:  936 score 500.000\n",
      "episode:  937 score 500.000\n",
      "episode:  938 score 500.000\n",
      "episode:  939 score 500.000\n",
      "episode:  940 score 500.000\n",
      "episode:  941 score 500.000\n",
      "episode:  942 score 500.000\n",
      "episode:  943 score 500.000\n",
      "episode:  944 score 500.000\n",
      "episode:  945 score 500.000\n",
      "episode:  946 score 500.000\n",
      "episode:  947 score 500.000\n",
      "episode:  948 score 500.000\n",
      "episode:  949 score 500.000\n",
      "episode:  950 score 500.000\n",
      "episode:  951 score 500.000\n",
      "episode:  952 score 500.000\n",
      "episode:  953 score 500.000\n",
      "episode:  954 score 500.000\n",
      "episode:  955 score 500.000\n",
      "episode:  956 score 500.000\n",
      "episode:  957 score 500.000\n",
      "episode:  958 score 500.000\n",
      "episode:  959 score 500.000\n",
      "episode:  960 score 500.000\n",
      "episode:  961 score 500.000\n",
      "episode:  962 score 500.000\n",
      "episode:  963 score 500.000\n",
      "episode:  964 score 500.000\n",
      "episode:  965 score 500.000\n",
      "episode:  966 score 500.000\n",
      "episode:  967 score 500.000\n",
      "episode:  968 score 500.000\n",
      "episode:  969 score 500.000\n",
      "episode:  970 score 500.000\n",
      "episode:  971 score 500.000\n",
      "episode:  972 score 500.000\n",
      "episode:  973 score 500.000\n",
      "episode:  974 score 500.000\n",
      "episode:  975 score 500.000\n",
      "episode:  976 score 500.000\n",
      "episode:  977 score 500.000\n",
      "episode:  978 score 500.000\n",
      "episode:  979 score 500.000\n",
      "episode:  980 score 500.000\n",
      "episode:  981 score 500.000\n",
      "episode:  982 score 500.000\n",
      "episode:  983 score 500.000\n",
      "episode:  984 score 500.000\n",
      "episode:  985 score 500.000\n",
      "episode:  986 score 500.000\n",
      "episode:  987 score 500.000\n",
      "episode:  988 score 500.000\n",
      "episode:  989 score 500.000\n",
      "episode:  990 score 500.000\n",
      "episode:  991 score 500.000\n",
      "episode:  992 score 500.000\n",
      "episode:  993 score 500.000\n",
      "episode:  994 score 500.000\n",
      "episode:  995 score 500.000\n",
      "episode:  996 score 500.000\n",
      "episode:  997 score 500.000\n",
      "episode:  998 score 500.000\n",
      "episode:  999 score 500.000\n",
      "episode:  1000 score 500.000\n",
      "episode:  1001 score 500.000\n",
      "episode:  1002 score 500.000\n",
      "episode:  1003 score 500.000\n",
      "episode:  1004 score 500.000\n",
      "episode:  1005 score 500.000\n",
      "episode:  1006 score 500.000\n",
      "episode:  1007 score 500.000\n",
      "episode:  1008 score 500.000\n",
      "episode:  1009 score 500.000\n",
      "episode:  1010 score 500.000\n",
      "episode:  1011 score 500.000\n",
      "episode:  1012 score 500.000\n",
      "episode:  1013 score 500.000\n",
      "episode:  1014 score 500.000\n",
      "episode:  1015 score 500.000\n",
      "episode:  1016 score 500.000\n",
      "episode:  1017 score 500.000\n",
      "episode:  1018 score 500.000\n",
      "episode:  1019 score 500.000\n",
      "episode:  1020 score 500.000\n",
      "episode:  1021 score 500.000\n",
      "episode:  1022 score 500.000\n",
      "episode:  1023 score 500.000\n",
      "episode:  1024 score 500.000\n",
      "episode:  1025 score 500.000\n",
      "episode:  1026 score 500.000\n",
      "episode:  1027 score 500.000\n",
      "episode:  1028 score 500.000\n",
      "episode:  1029 score 500.000\n",
      "episode:  1030 score 500.000\n",
      "episode:  1031 score 500.000\n",
      "episode:  1032 score 500.000\n",
      "episode:  1033 score 500.000\n",
      "episode:  1034 score 500.000\n",
      "episode:  1035 score 500.000\n",
      "episode:  1036 score 500.000\n",
      "episode:  1037 score 500.000\n",
      "episode:  1038 score 500.000\n",
      "episode:  1039 score 500.000\n",
      "episode:  1040 score 500.000\n",
      "episode:  1041 score 500.000\n",
      "episode:  1042 score 500.000\n",
      "episode:  1043 score 500.000\n",
      "episode:  1044 score 500.000\n",
      "episode:  1045 score 500.000\n",
      "episode:  1046 score 500.000\n",
      "episode:  1047 score 500.000\n",
      "episode:  1048 score 500.000\n",
      "episode:  1049 score 500.000\n",
      "episode:  1050 score 500.000\n",
      "episode:  1051 score 500.000\n",
      "episode:  1052 score 500.000\n",
      "episode:  1053 score 500.000\n",
      "episode:  1054 score 500.000\n",
      "episode:  1055 score 500.000\n",
      "episode:  1056 score 500.000\n",
      "episode:  1057 score 500.000\n",
      "episode:  1058 score 500.000\n",
      "episode:  1059 score 500.000\n",
      "episode:  1060 score 500.000\n",
      "episode:  1061 score 500.000\n",
      "episode:  1062 score 500.000\n",
      "episode:  1063 score 500.000\n",
      "episode:  1064 score 500.000\n",
      "episode:  1065 score 500.000\n",
      "episode:  1066 score 500.000\n",
      "episode:  1067 score 500.000\n",
      "episode:  1068 score 500.000\n",
      "episode:  1069 score 500.000\n",
      "episode:  1070 score 500.000\n",
      "episode:  1071 score 500.000\n",
      "episode:  1072 score 500.000\n",
      "episode:  1073 score 500.000\n",
      "episode:  1074 score 500.000\n",
      "episode:  1075 score 500.000\n",
      "episode:  1076 score 500.000\n",
      "episode:  1077 score 500.000\n",
      "episode:  1078 score 500.000\n",
      "episode:  1079 score 500.000\n",
      "episode:  1080 score 500.000\n",
      "episode:  1081 score 500.000\n",
      "episode:  1082 score 500.000\n",
      "episode:  1083 score 500.000\n",
      "episode:  1084 score 500.000\n",
      "episode:  1085 score 500.000\n",
      "episode:  1086 score 500.000\n",
      "episode:  1087 score 500.000\n",
      "episode:  1088 score 500.000\n",
      "episode:  1089 score 500.000\n",
      "episode:  1090 score 500.000\n",
      "episode:  1091 score 500.000\n",
      "episode:  1092 score 500.000\n",
      "episode:  1093 score 500.000\n",
      "episode:  1094 score 500.000\n",
      "episode:  1095 score 500.000\n",
      "episode:  1096 score 500.000\n",
      "episode:  1097 score 500.000\n",
      "episode:  1098 score 500.000\n",
      "episode:  1099 score 500.000\n",
      "episode:  1100 score 500.000\n",
      "episode:  1101 score 500.000\n",
      "episode:  1102 score 500.000\n",
      "episode:  1103 score 500.000\n",
      "episode:  1104 score 500.000\n",
      "episode:  1105 score 500.000\n",
      "episode:  1106 score 500.000\n",
      "episode:  1107 score 500.000\n",
      "episode:  1108 score 500.000\n",
      "episode:  1109 score 500.000\n",
      "episode:  1110 score 500.000\n",
      "episode:  1111 score 500.000\n",
      "episode:  1112 score 500.000\n",
      "episode:  1113 score 500.000\n",
      "episode:  1114 score 500.000\n",
      "episode:  1115 score 500.000\n",
      "episode:  1116 score 500.000\n",
      "episode:  1117 score 500.000\n",
      "episode:  1118 score 500.000\n",
      "episode:  1119 score 500.000\n",
      "episode:  1120 score 500.000\n",
      "episode:  1121 score 500.000\n",
      "episode:  1122 score 500.000\n",
      "episode:  1123 score 500.000\n",
      "episode:  1124 score 500.000\n",
      "episode:  1125 score 500.000\n",
      "episode:  1126 score 500.000\n",
      "episode:  1127 score 500.000\n",
      "episode:  1128 score 500.000\n",
      "episode:  1129 score 500.000\n",
      "episode:  1130 score 500.000\n",
      "episode:  1131 score 500.000\n",
      "episode:  1132 score 500.000\n",
      "episode:  1133 score 500.000\n",
      "episode:  1134 score 500.000\n",
      "episode:  1135 score 500.000\n",
      "episode:  1136 score 500.000\n",
      "episode:  1137 score 500.000\n",
      "episode:  1138 score 500.000\n",
      "episode:  1139 score 500.000\n",
      "episode:  1140 score 500.000\n",
      "episode:  1141 score 500.000\n",
      "episode:  1142 score 500.000\n",
      "episode:  1143 score 500.000\n",
      "episode:  1144 score 500.000\n",
      "episode:  1145 score 500.000\n",
      "episode:  1146 score 500.000\n",
      "episode:  1147 score 500.000\n",
      "episode:  1148 score 500.000\n",
      "episode:  1149 score 500.000\n",
      "episode:  1150 score 500.000\n",
      "episode:  1151 score 500.000\n",
      "episode:  1152 score 500.000\n",
      "episode:  1153 score 489.000\n",
      "episode:  1154 score 500.000\n",
      "episode:  1155 score 500.000\n",
      "episode:  1156 score 500.000\n",
      "episode:  1157 score 500.000\n",
      "episode:  1158 score 157.000\n",
      "episode:  1159 score 500.000\n",
      "episode:  1160 score 500.000\n",
      "episode:  1161 score 500.000\n",
      "episode:  1162 score 309.000\n",
      "episode:  1163 score 500.000\n",
      "episode:  1164 score 500.000\n",
      "episode:  1165 score 500.000\n",
      "episode:  1166 score 500.000\n",
      "episode:  1167 score 500.000\n",
      "episode:  1168 score 500.000\n",
      "episode:  1169 score 500.000\n",
      "episode:  1170 score 500.000\n",
      "episode:  1171 score 500.000\n",
      "episode:  1172 score 273.000\n",
      "episode:  1173 score 500.000\n",
      "episode:  1174 score 296.000\n",
      "episode:  1175 score 500.000\n",
      "episode:  1176 score 500.000\n",
      "episode:  1177 score 442.000\n",
      "episode:  1178 score 500.000\n",
      "episode:  1179 score 415.000\n",
      "episode:  1180 score 274.000\n",
      "episode:  1181 score 339.000\n",
      "episode:  1182 score 398.000\n",
      "episode:  1183 score 388.000\n",
      "episode:  1184 score 362.000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:  1185 score 500.000\n",
      "episode:  1186 score 224.000\n",
      "episode:  1187 score 217.000\n",
      "episode:  1188 score 500.000\n",
      "episode:  1189 score 464.000\n",
      "episode:  1190 score 500.000\n",
      "episode:  1191 score 496.000\n",
      "episode:  1192 score 500.000\n",
      "episode:  1193 score 496.000\n",
      "episode:  1194 score 500.000\n",
      "episode:  1195 score 351.000\n",
      "episode:  1196 score 500.000\n",
      "episode:  1197 score 496.000\n",
      "episode:  1198 score 500.000\n",
      "episode:  1199 score 500.000\n",
      "episode:  1200 score 500.000\n",
      "episode:  1201 score 500.000\n",
      "episode:  1202 score 500.000\n",
      "episode:  1203 score 500.000\n",
      "episode:  1204 score 500.000\n",
      "episode:  1205 score 500.000\n",
      "episode:  1206 score 500.000\n",
      "episode:  1207 score 500.000\n",
      "episode:  1208 score 500.000\n",
      "episode:  1209 score 500.000\n",
      "episode:  1210 score 500.000\n",
      "episode:  1211 score 500.000\n",
      "episode:  1212 score 500.000\n",
      "episode:  1213 score 500.000\n",
      "episode:  1214 score 500.000\n",
      "episode:  1215 score 500.000\n",
      "episode:  1216 score 500.000\n",
      "episode:  1217 score 500.000\n",
      "episode:  1218 score 500.000\n",
      "episode:  1219 score 500.000\n",
      "episode:  1220 score 500.000\n",
      "episode:  1221 score 500.000\n",
      "episode:  1222 score 500.000\n",
      "episode:  1223 score 500.000\n",
      "episode:  1224 score 500.000\n",
      "episode:  1225 score 500.000\n",
      "episode:  1226 score 500.000\n",
      "episode:  1227 score 500.000\n",
      "episode:  1228 score 500.000\n",
      "episode:  1229 score 500.000\n",
      "episode:  1230 score 500.000\n",
      "episode:  1231 score 500.000\n",
      "episode:  1232 score 500.000\n",
      "episode:  1233 score 500.000\n",
      "episode:  1234 score 500.000\n",
      "episode:  1235 score 500.000\n",
      "episode:  1236 score 500.000\n",
      "episode:  1237 score 500.000\n",
      "episode:  1238 score 500.000\n",
      "episode:  1239 score 500.000\n",
      "episode:  1240 score 500.000\n",
      "episode:  1241 score 500.000\n",
      "episode:  1242 score 500.000\n",
      "episode:  1243 score 500.000\n",
      "episode:  1244 score 500.000\n",
      "episode:  1245 score 500.000\n",
      "episode:  1246 score 500.000\n",
      "episode:  1247 score 500.000\n",
      "episode:  1248 score 500.000\n",
      "episode:  1249 score 500.000\n",
      "episode:  1250 score 500.000\n",
      "episode:  1251 score 500.000\n",
      "episode:  1252 score 500.000\n",
      "episode:  1253 score 500.000\n",
      "episode:  1254 score 500.000\n",
      "episode:  1255 score 500.000\n",
      "episode:  1256 score 500.000\n",
      "episode:  1257 score 500.000\n",
      "episode:  1258 score 500.000\n",
      "episode:  1259 score 500.000\n",
      "episode:  1260 score 500.000\n",
      "episode:  1261 score 500.000\n",
      "episode:  1262 score 500.000\n",
      "episode:  1263 score 500.000\n",
      "episode:  1264 score 500.000\n",
      "episode:  1265 score 500.000\n",
      "episode:  1266 score 500.000\n",
      "episode:  1267 score 500.000\n",
      "episode:  1268 score 500.000\n",
      "episode:  1269 score 500.000\n",
      "episode:  1270 score 500.000\n",
      "episode:  1271 score 500.000\n",
      "episode:  1272 score 500.000\n",
      "episode:  1273 score 500.000\n",
      "episode:  1274 score 500.000\n",
      "episode:  1275 score 500.000\n",
      "episode:  1276 score 500.000\n",
      "episode:  1277 score 500.000\n",
      "episode:  1278 score 500.000\n",
      "episode:  1279 score 500.000\n",
      "episode:  1280 score 500.000\n",
      "episode:  1281 score 500.000\n",
      "episode:  1282 score 500.000\n",
      "episode:  1283 score 500.000\n",
      "episode:  1284 score 500.000\n",
      "episode:  1285 score 500.000\n",
      "episode:  1286 score 500.000\n",
      "episode:  1287 score 500.000\n",
      "episode:  1288 score 500.000\n",
      "episode:  1289 score 500.000\n",
      "episode:  1290 score 500.000\n",
      "episode:  1291 score 500.000\n",
      "episode:  1292 score 500.000\n",
      "episode:  1293 score 500.000\n",
      "episode:  1294 score 500.000\n",
      "episode:  1295 score 500.000\n",
      "episode:  1296 score 500.000\n",
      "episode:  1297 score 500.000\n",
      "episode:  1298 score 500.000\n",
      "episode:  1299 score 500.000\n",
      "episode:  1300 score 500.000\n",
      "episode:  1301 score 500.000\n",
      "episode:  1302 score 500.000\n",
      "episode:  1303 score 500.000\n",
      "episode:  1304 score 500.000\n",
      "episode:  1305 score 500.000\n",
      "episode:  1306 score 500.000\n",
      "episode:  1307 score 500.000\n",
      "episode:  1308 score 500.000\n",
      "episode:  1309 score 500.000\n",
      "episode:  1310 score 500.000\n",
      "episode:  1311 score 500.000\n",
      "episode:  1312 score 500.000\n",
      "episode:  1313 score 500.000\n",
      "episode:  1314 score 500.000\n",
      "episode:  1315 score 500.000\n",
      "episode:  1316 score 500.000\n",
      "episode:  1317 score 500.000\n",
      "episode:  1318 score 500.000\n",
      "episode:  1319 score 500.000\n",
      "episode:  1320 score 500.000\n",
      "episode:  1321 score 500.000\n",
      "episode:  1322 score 500.000\n",
      "episode:  1323 score 500.000\n",
      "episode:  1324 score 500.000\n",
      "episode:  1325 score 500.000\n",
      "episode:  1326 score 500.000\n",
      "episode:  1327 score 500.000\n",
      "episode:  1328 score 500.000\n",
      "episode:  1329 score 500.000\n",
      "episode:  1330 score 500.000\n",
      "episode:  1331 score 500.000\n",
      "episode:  1332 score 500.000\n",
      "episode:  1333 score 500.000\n",
      "episode:  1334 score 500.000\n",
      "episode:  1335 score 500.000\n",
      "episode:  1336 score 500.000\n",
      "episode:  1337 score 500.000\n",
      "episode:  1338 score 500.000\n",
      "episode:  1339 score 500.000\n",
      "episode:  1340 score 500.000\n",
      "episode:  1341 score 500.000\n",
      "episode:  1342 score 500.000\n",
      "episode:  1343 score 500.000\n",
      "episode:  1344 score 500.000\n",
      "episode:  1345 score 500.000\n",
      "episode:  1346 score 500.000\n",
      "episode:  1347 score 500.000\n",
      "episode:  1348 score 500.000\n",
      "episode:  1349 score 500.000\n",
      "episode:  1350 score 500.000\n",
      "episode:  1351 score 500.000\n",
      "episode:  1352 score 500.000\n",
      "episode:  1353 score 500.000\n",
      "episode:  1354 score 500.000\n",
      "episode:  1355 score 500.000\n",
      "episode:  1356 score 500.000\n",
      "episode:  1357 score 500.000\n",
      "episode:  1358 score 500.000\n",
      "episode:  1359 score 500.000\n",
      "episode:  1360 score 500.000\n",
      "episode:  1361 score 500.000\n",
      "episode:  1362 score 500.000\n",
      "episode:  1363 score 500.000\n",
      "episode:  1364 score 500.000\n",
      "episode:  1365 score 500.000\n",
      "episode:  1366 score 500.000\n",
      "episode:  1367 score 500.000\n",
      "episode:  1368 score 500.000\n",
      "episode:  1369 score 500.000\n",
      "episode:  1370 score 500.000\n",
      "episode:  1371 score 500.000\n",
      "episode:  1372 score 500.000\n",
      "episode:  1373 score 500.000\n",
      "episode:  1374 score 500.000\n",
      "episode:  1375 score 500.000\n",
      "episode:  1376 score 500.000\n",
      "episode:  1377 score 500.000\n",
      "episode:  1378 score 500.000\n",
      "episode:  1379 score 500.000\n",
      "episode:  1380 score 500.000\n",
      "episode:  1381 score 500.000\n",
      "episode:  1382 score 500.000\n",
      "episode:  1383 score 500.000\n",
      "episode:  1384 score 500.000\n",
      "episode:  1385 score 500.000\n",
      "episode:  1386 score 500.000\n",
      "episode:  1387 score 500.000\n",
      "episode:  1388 score 375.000\n",
      "episode:  1389 score 500.000\n",
      "episode:  1390 score 500.000\n",
      "episode:  1391 score 500.000\n",
      "episode:  1392 score 500.000\n",
      "episode:  1393 score 403.000\n",
      "episode:  1394 score 500.000\n",
      "episode:  1395 score 500.000\n",
      "episode:  1396 score 500.000\n",
      "episode:  1397 score 500.000\n",
      "episode:  1398 score 500.000\n",
      "episode:  1399 score 500.000\n",
      "episode:  1400 score 457.000\n",
      "episode:  1401 score 500.000\n",
      "episode:  1402 score 500.000\n",
      "episode:  1403 score 491.000\n",
      "episode:  1404 score 500.000\n",
      "episode:  1405 score 500.000\n",
      "episode:  1406 score 500.000\n",
      "episode:  1407 score 500.000\n",
      "episode:  1408 score 500.000\n",
      "episode:  1409 score 500.000\n",
      "episode:  1410 score 500.000\n",
      "episode:  1411 score 500.000\n",
      "episode:  1412 score 500.000\n",
      "episode:  1413 score 500.000\n",
      "episode:  1414 score 500.000\n",
      "episode:  1415 score 500.000\n",
      "episode:  1416 score 500.000\n",
      "episode:  1417 score 500.000\n",
      "episode:  1418 score 500.000\n",
      "episode:  1419 score 500.000\n",
      "episode:  1420 score 500.000\n",
      "episode:  1421 score 500.000\n",
      "episode:  1422 score 500.000\n",
      "episode:  1423 score 500.000\n",
      "episode:  1424 score 500.000\n",
      "episode:  1425 score 500.000\n",
      "episode:  1426 score 500.000\n",
      "episode:  1427 score 500.000\n",
      "episode:  1428 score 500.000\n",
      "episode:  1429 score 500.000\n",
      "episode:  1430 score 500.000\n",
      "episode:  1431 score 500.000\n",
      "episode:  1432 score 500.000\n",
      "episode:  1433 score 500.000\n",
      "episode:  1434 score 500.000\n",
      "episode:  1435 score 333.000\n",
      "episode:  1436 score 500.000\n",
      "episode:  1437 score 500.000\n",
      "episode:  1438 score 500.000\n",
      "episode:  1439 score 500.000\n",
      "episode:  1440 score 500.000\n",
      "episode:  1441 score 500.000\n",
      "episode:  1442 score 500.000\n",
      "episode:  1443 score 500.000\n",
      "episode:  1444 score 500.000\n",
      "episode:  1445 score 500.000\n",
      "episode:  1446 score 500.000\n",
      "episode:  1447 score 500.000\n",
      "episode:  1448 score 500.000\n",
      "episode:  1449 score 500.000\n",
      "episode:  1450 score 500.000\n",
      "episode:  1451 score 500.000\n",
      "episode:  1452 score 391.000\n",
      "episode:  1453 score 500.000\n",
      "episode:  1454 score 500.000\n",
      "episode:  1455 score 500.000\n",
      "episode:  1456 score 500.000\n",
      "episode:  1457 score 500.000\n",
      "episode:  1458 score 500.000\n",
      "episode:  1459 score 500.000\n",
      "episode:  1460 score 500.000\n",
      "episode:  1461 score 500.000\n",
      "episode:  1462 score 500.000\n",
      "episode:  1463 score 500.000\n",
      "episode:  1464 score 500.000\n",
      "episode:  1465 score 414.000\n",
      "episode:  1466 score 500.000\n",
      "episode:  1467 score 500.000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:  1468 score 500.000\n",
      "episode:  1469 score 500.000\n",
      "episode:  1470 score 500.000\n",
      "episode:  1471 score 500.000\n",
      "episode:  1472 score 500.000\n",
      "episode:  1473 score 500.000\n",
      "episode:  1474 score 500.000\n",
      "episode:  1475 score 500.000\n",
      "episode:  1476 score 500.000\n",
      "episode:  1477 score 500.000\n",
      "episode:  1478 score 500.000\n",
      "episode:  1479 score 500.000\n",
      "episode:  1480 score 500.000\n",
      "episode:  1481 score 500.000\n",
      "episode:  1482 score 500.000\n",
      "episode:  1483 score 500.000\n",
      "episode:  1484 score 500.000\n",
      "episode:  1485 score 500.000\n",
      "episode:  1486 score 500.000\n",
      "episode:  1487 score 500.000\n",
      "episode:  1488 score 500.000\n",
      "episode:  1489 score 500.000\n",
      "episode:  1490 score 500.000\n",
      "episode:  1491 score 500.000\n",
      "episode:  1492 score 500.000\n",
      "episode:  1493 score 500.000\n",
      "episode:  1494 score 500.000\n",
      "episode:  1495 score 500.000\n",
      "episode:  1496 score 500.000\n",
      "episode:  1497 score 500.000\n",
      "episode:  1498 score 500.000\n",
      "episode:  1499 score 500.000\n",
      "episode:  1500 score 500.000\n",
      "episode:  1501 score 500.000\n",
      "episode:  1502 score 500.000\n",
      "episode:  1503 score 500.000\n",
      "episode:  1504 score 500.000\n",
      "episode:  1505 score 500.000\n",
      "episode:  1506 score 500.000\n",
      "episode:  1507 score 500.000\n",
      "episode:  1508 score 500.000\n",
      "episode:  1509 score 500.000\n",
      "episode:  1510 score 500.000\n",
      "episode:  1511 score 500.000\n",
      "episode:  1512 score 500.000\n",
      "episode:  1513 score 500.000\n",
      "episode:  1514 score 500.000\n",
      "episode:  1515 score 500.000\n",
      "episode:  1516 score 500.000\n",
      "episode:  1517 score 500.000\n",
      "episode:  1518 score 500.000\n",
      "episode:  1519 score 500.000\n",
      "episode:  1520 score 500.000\n",
      "episode:  1521 score 500.000\n",
      "episode:  1522 score 500.000\n",
      "episode:  1523 score 500.000\n",
      "episode:  1524 score 500.000\n",
      "episode:  1525 score 500.000\n",
      "episode:  1526 score 500.000\n",
      "episode:  1527 score 500.000\n",
      "episode:  1528 score 500.000\n",
      "episode:  1529 score 500.000\n",
      "episode:  1530 score 500.000\n",
      "episode:  1531 score 500.000\n",
      "episode:  1532 score 500.000\n",
      "episode:  1533 score 500.000\n",
      "episode:  1534 score 500.000\n",
      "episode:  1535 score 500.000\n",
      "episode:  1536 score 500.000\n",
      "episode:  1537 score 500.000\n",
      "episode:  1538 score 500.000\n",
      "episode:  1539 score 500.000\n",
      "episode:  1540 score 500.000\n",
      "episode:  1541 score 500.000\n",
      "episode:  1542 score 500.000\n",
      "episode:  1543 score 500.000\n",
      "episode:  1544 score 500.000\n",
      "episode:  1545 score 500.000\n",
      "episode:  1546 score 500.000\n",
      "episode:  1547 score 500.000\n",
      "episode:  1548 score 500.000\n",
      "episode:  1549 score 500.000\n",
      "episode:  1550 score 500.000\n",
      "episode:  1551 score 500.000\n",
      "episode:  1552 score 500.000\n",
      "episode:  1553 score 500.000\n",
      "episode:  1554 score 500.000\n",
      "episode:  1555 score 500.000\n",
      "episode:  1556 score 500.000\n",
      "episode:  1557 score 500.000\n",
      "episode:  1558 score 500.000\n",
      "episode:  1559 score 500.000\n",
      "episode:  1560 score 500.000\n",
      "episode:  1561 score 500.000\n",
      "episode:  1562 score 500.000\n",
      "episode:  1563 score 500.000\n",
      "episode:  1564 score 500.000\n",
      "episode:  1565 score 500.000\n",
      "episode:  1566 score 500.000\n",
      "episode:  1567 score 500.000\n",
      "episode:  1568 score 500.000\n",
      "episode:  1569 score 500.000\n",
      "episode:  1570 score 500.000\n",
      "episode:  1571 score 500.000\n",
      "episode:  1572 score 500.000\n",
      "episode:  1573 score 500.000\n",
      "episode:  1574 score 500.000\n",
      "episode:  1575 score 500.000\n",
      "episode:  1576 score 500.000\n",
      "episode:  1577 score 500.000\n",
      "episode:  1578 score 500.000\n",
      "episode:  1579 score 500.000\n",
      "episode:  1580 score 500.000\n",
      "episode:  1581 score 500.000\n",
      "episode:  1582 score 500.000\n",
      "episode:  1583 score 500.000\n",
      "episode:  1584 score 500.000\n",
      "episode:  1585 score 500.000\n",
      "episode:  1586 score 500.000\n",
      "episode:  1587 score 500.000\n",
      "episode:  1588 score 500.000\n",
      "episode:  1589 score 500.000\n",
      "episode:  1590 score 500.000\n",
      "episode:  1591 score 500.000\n",
      "episode:  1592 score 500.000\n",
      "episode:  1593 score 500.000\n",
      "episode:  1594 score 500.000\n",
      "episode:  1595 score 500.000\n",
      "episode:  1596 score 500.000\n",
      "episode:  1597 score 500.000\n",
      "episode:  1598 score 500.000\n",
      "episode:  1599 score 500.000\n",
      "episode:  1600 score 500.000\n",
      "episode:  1601 score 500.000\n",
      "episode:  1602 score 500.000\n",
      "episode:  1603 score 500.000\n",
      "episode:  1604 score 500.000\n",
      "episode:  1605 score 500.000\n",
      "episode:  1606 score 500.000\n",
      "episode:  1607 score 500.000\n",
      "episode:  1608 score 500.000\n",
      "episode:  1609 score 500.000\n",
      "episode:  1610 score 500.000\n",
      "episode:  1611 score 500.000\n",
      "episode:  1612 score 500.000\n",
      "episode:  1613 score 500.000\n",
      "episode:  1614 score 500.000\n",
      "episode:  1615 score 500.000\n",
      "episode:  1616 score 500.000\n",
      "episode:  1617 score 500.000\n",
      "episode:  1618 score 500.000\n",
      "episode:  1619 score 500.000\n",
      "episode:  1620 score 500.000\n",
      "episode:  1621 score 500.000\n",
      "episode:  1622 score 500.000\n",
      "episode:  1623 score 500.000\n",
      "episode:  1624 score 500.000\n",
      "episode:  1625 score 500.000\n",
      "episode:  1626 score 500.000\n",
      "episode:  1627 score 500.000\n",
      "episode:  1628 score 500.000\n",
      "episode:  1629 score 500.000\n",
      "episode:  1630 score 500.000\n",
      "episode:  1631 score 500.000\n",
      "episode:  1632 score 500.000\n",
      "episode:  1633 score 500.000\n",
      "episode:  1634 score 500.000\n",
      "episode:  1635 score 500.000\n",
      "episode:  1636 score 500.000\n",
      "episode:  1637 score 500.000\n",
      "episode:  1638 score 500.000\n",
      "episode:  1639 score 500.000\n",
      "episode:  1640 score 500.000\n",
      "episode:  1641 score 500.000\n",
      "episode:  1642 score 500.000\n",
      "episode:  1643 score 500.000\n",
      "episode:  1644 score 500.000\n",
      "episode:  1645 score 500.000\n",
      "episode:  1646 score 500.000\n",
      "episode:  1647 score 500.000\n",
      "episode:  1648 score 500.000\n",
      "episode:  1649 score 500.000\n",
      "episode:  1650 score 500.000\n",
      "episode:  1651 score 500.000\n",
      "episode:  1652 score 500.000\n",
      "episode:  1653 score 500.000\n",
      "episode:  1654 score 500.000\n",
      "episode:  1655 score 500.000\n",
      "episode:  1656 score 500.000\n",
      "episode:  1657 score 500.000\n",
      "episode:  1658 score 500.000\n",
      "episode:  1659 score 500.000\n",
      "episode:  1660 score 500.000\n",
      "episode:  1661 score 500.000\n",
      "episode:  1662 score 500.000\n",
      "episode:  1663 score 500.000\n",
      "episode:  1664 score 500.000\n",
      "episode:  1665 score 500.000\n",
      "episode:  1666 score 500.000\n",
      "episode:  1667 score 447.000\n",
      "episode:  1668 score 500.000\n",
      "episode:  1669 score 500.000\n",
      "episode:  1670 score 500.000\n",
      "episode:  1671 score 500.000\n",
      "episode:  1672 score 500.000\n",
      "episode:  1673 score 500.000\n",
      "episode:  1674 score 500.000\n",
      "episode:  1675 score 500.000\n",
      "episode:  1676 score 500.000\n",
      "episode:  1677 score 500.000\n",
      "episode:  1678 score 500.000\n",
      "episode:  1679 score 500.000\n",
      "episode:  1680 score 500.000\n",
      "episode:  1681 score 500.000\n",
      "episode:  1682 score 500.000\n",
      "episode:  1683 score 500.000\n",
      "episode:  1684 score 500.000\n",
      "episode:  1685 score 500.000\n",
      "episode:  1686 score 500.000\n",
      "episode:  1687 score 500.000\n",
      "episode:  1688 score 500.000\n",
      "episode:  1689 score 500.000\n",
      "episode:  1690 score 500.000\n",
      "episode:  1691 score 500.000\n",
      "episode:  1692 score 500.000\n",
      "episode:  1693 score 500.000\n",
      "episode:  1694 score 500.000\n",
      "episode:  1695 score 500.000\n",
      "episode:  1696 score 500.000\n",
      "episode:  1697 score 500.000\n",
      "episode:  1698 score 500.000\n",
      "episode:  1699 score 500.000\n",
      "episode:  1700 score 500.000\n",
      "episode:  1701 score 500.000\n",
      "episode:  1702 score 500.000\n",
      "episode:  1703 score 500.000\n",
      "episode:  1704 score 500.000\n",
      "episode:  1705 score 500.000\n",
      "episode:  1706 score 500.000\n",
      "episode:  1707 score 500.000\n",
      "episode:  1708 score 500.000\n",
      "episode:  1709 score 500.000\n",
      "episode:  1710 score 500.000\n",
      "episode:  1711 score 500.000\n",
      "episode:  1712 score 500.000\n",
      "episode:  1713 score 500.000\n",
      "episode:  1714 score 500.000\n",
      "episode:  1715 score 500.000\n",
      "episode:  1716 score 500.000\n",
      "episode:  1717 score 500.000\n",
      "episode:  1718 score 500.000\n",
      "episode:  1719 score 500.000\n",
      "episode:  1720 score 500.000\n",
      "episode:  1721 score 500.000\n",
      "episode:  1722 score 500.000\n",
      "episode:  1723 score 500.000\n",
      "episode:  1724 score 500.000\n",
      "episode:  1725 score 500.000\n",
      "episode:  1726 score 500.000\n",
      "episode:  1727 score 500.000\n",
      "episode:  1728 score 500.000\n",
      "episode:  1729 score 500.000\n",
      "episode:  1730 score 500.000\n",
      "episode:  1731 score 500.000\n",
      "episode:  1732 score 500.000\n",
      "episode:  1733 score 500.000\n",
      "episode:  1734 score 500.000\n",
      "episode:  1735 score 500.000\n",
      "episode:  1736 score 500.000\n",
      "episode:  1737 score 500.000\n",
      "episode:  1738 score 500.000\n",
      "episode:  1739 score 500.000\n",
      "episode:  1740 score 500.000\n",
      "episode:  1741 score 500.000\n",
      "episode:  1742 score 500.000\n",
      "episode:  1743 score 500.000\n",
      "episode:  1744 score 500.000\n",
      "episode:  1745 score 500.000\n",
      "episode:  1746 score 500.000\n",
      "episode:  1747 score 500.000\n",
      "episode:  1748 score 500.000\n",
      "episode:  1749 score 500.000\n",
      "episode:  1750 score 500.000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:  1751 score 500.000\n",
      "episode:  1752 score 500.000\n",
      "episode:  1753 score 500.000\n",
      "episode:  1754 score 500.000\n",
      "episode:  1755 score 500.000\n",
      "episode:  1756 score 500.000\n",
      "episode:  1757 score 500.000\n",
      "episode:  1758 score 500.000\n",
      "episode:  1759 score 500.000\n",
      "episode:  1760 score 500.000\n",
      "episode:  1761 score 500.000\n",
      "episode:  1762 score 500.000\n",
      "episode:  1763 score 500.000\n",
      "episode:  1764 score 500.000\n",
      "episode:  1765 score 500.000\n",
      "episode:  1766 score 500.000\n",
      "episode:  1767 score 500.000\n",
      "episode:  1768 score 500.000\n",
      "episode:  1769 score 500.000\n",
      "episode:  1770 score 500.000\n",
      "episode:  1771 score 500.000\n",
      "episode:  1772 score 500.000\n",
      "episode:  1773 score 500.000\n",
      "episode:  1774 score 500.000\n",
      "episode:  1775 score 500.000\n",
      "episode:  1776 score 500.000\n",
      "episode:  1777 score 500.000\n",
      "episode:  1778 score 500.000\n",
      "episode:  1779 score 500.000\n",
      "episode:  1780 score 500.000\n",
      "episode:  1781 score 500.000\n",
      "episode:  1782 score 500.000\n",
      "episode:  1783 score 500.000\n",
      "episode:  1784 score 500.000\n",
      "episode:  1785 score 500.000\n",
      "episode:  1786 score 500.000\n",
      "episode:  1787 score 500.000\n",
      "episode:  1788 score 500.000\n",
      "episode:  1789 score 500.000\n",
      "episode:  1790 score 500.000\n",
      "episode:  1791 score 500.000\n",
      "episode:  1792 score 500.000\n",
      "episode:  1793 score 500.000\n",
      "episode:  1794 score 500.000\n",
      "episode:  1795 score 500.000\n",
      "episode:  1796 score 500.000\n",
      "episode:  1797 score 500.000\n",
      "episode:  1798 score 500.000\n",
      "episode:  1799 score 500.000\n",
      "episode:  1800 score 500.000\n",
      "episode:  1801 score 500.000\n",
      "episode:  1802 score 500.000\n",
      "episode:  1803 score 500.000\n",
      "episode:  1804 score 500.000\n",
      "episode:  1805 score 500.000\n",
      "episode:  1806 score 500.000\n",
      "episode:  1807 score 500.000\n",
      "episode:  1808 score 500.000\n",
      "episode:  1809 score 500.000\n",
      "episode:  1810 score 500.000\n",
      "episode:  1811 score 500.000\n",
      "episode:  1812 score 500.000\n",
      "episode:  1813 score 500.000\n",
      "episode:  1814 score 500.000\n",
      "episode:  1815 score 500.000\n",
      "episode:  1816 score 500.000\n",
      "episode:  1817 score 500.000\n",
      "episode:  1818 score 500.000\n",
      "episode:  1819 score 500.000\n",
      "episode:  1820 score 500.000\n",
      "episode:  1821 score 500.000\n",
      "episode:  1822 score 500.000\n",
      "episode:  1823 score 500.000\n",
      "episode:  1824 score 500.000\n",
      "episode:  1825 score 500.000\n",
      "episode:  1826 score 500.000\n",
      "episode:  1827 score 500.000\n",
      "episode:  1828 score 500.000\n",
      "episode:  1829 score 500.000\n",
      "episode:  1830 score 500.000\n",
      "episode:  1831 score 500.000\n",
      "episode:  1832 score 500.000\n",
      "episode:  1833 score 500.000\n",
      "episode:  1834 score 500.000\n",
      "episode:  1835 score 500.000\n",
      "episode:  1836 score 500.000\n",
      "episode:  1837 score 500.000\n",
      "episode:  1838 score 500.000\n",
      "episode:  1839 score 500.000\n",
      "episode:  1840 score 500.000\n",
      "episode:  1841 score 500.000\n",
      "episode:  1842 score 500.000\n",
      "episode:  1843 score 500.000\n",
      "episode:  1844 score 500.000\n",
      "episode:  1845 score 500.000\n",
      "episode:  1846 score 500.000\n",
      "episode:  1847 score 500.000\n",
      "episode:  1848 score 500.000\n",
      "episode:  1849 score 500.000\n",
      "episode:  1850 score 500.000\n",
      "episode:  1851 score 500.000\n",
      "episode:  1852 score 500.000\n",
      "episode:  1853 score 500.000\n",
      "episode:  1854 score 500.000\n",
      "episode:  1855 score 500.000\n",
      "episode:  1856 score 500.000\n",
      "episode:  1857 score 500.000\n",
      "episode:  1858 score 500.000\n",
      "episode:  1859 score 500.000\n",
      "episode:  1860 score 500.000\n",
      "episode:  1861 score 500.000\n",
      "episode:  1862 score 500.000\n",
      "episode:  1863 score 500.000\n",
      "episode:  1864 score 500.000\n",
      "episode:  1865 score 500.000\n",
      "episode:  1866 score 500.000\n",
      "episode:  1867 score 500.000\n",
      "episode:  1868 score 500.000\n",
      "episode:  1869 score 500.000\n",
      "episode:  1870 score 500.000\n",
      "episode:  1871 score 500.000\n",
      "episode:  1872 score 500.000\n",
      "episode:  1873 score 500.000\n",
      "episode:  1874 score 500.000\n",
      "episode:  1875 score 500.000\n",
      "episode:  1876 score 500.000\n",
      "episode:  1877 score 500.000\n",
      "episode:  1878 score 500.000\n",
      "episode:  1879 score 500.000\n",
      "episode:  1880 score 500.000\n",
      "episode:  1881 score 500.000\n",
      "episode:  1882 score 500.000\n",
      "episode:  1883 score 500.000\n",
      "episode:  1884 score 500.000\n",
      "episode:  1885 score 500.000\n",
      "episode:  1886 score 500.000\n",
      "episode:  1887 score 500.000\n",
      "episode:  1888 score 500.000\n",
      "episode:  1889 score 500.000\n",
      "episode:  1890 score 500.000\n",
      "episode:  1891 score 500.000\n",
      "episode:  1892 score 500.000\n",
      "episode:  1893 score 500.000\n",
      "episode:  1894 score 500.000\n",
      "episode:  1895 score 500.000\n",
      "episode:  1896 score 500.000\n",
      "episode:  1897 score 500.000\n",
      "episode:  1898 score 500.000\n",
      "episode:  1899 score 500.000\n",
      "episode:  1900 score 500.000\n",
      "episode:  1901 score 500.000\n",
      "episode:  1902 score 500.000\n",
      "episode:  1903 score 500.000\n",
      "episode:  1904 score 500.000\n",
      "episode:  1905 score 500.000\n",
      "episode:  1906 score 500.000\n",
      "episode:  1907 score 500.000\n",
      "episode:  1908 score 500.000\n",
      "episode:  1909 score 500.000\n",
      "episode:  1910 score 500.000\n",
      "episode:  1911 score 500.000\n",
      "episode:  1912 score 500.000\n",
      "episode:  1913 score 500.000\n",
      "episode:  1914 score 500.000\n",
      "episode:  1915 score 500.000\n",
      "episode:  1916 score 500.000\n",
      "episode:  1917 score 500.000\n",
      "episode:  1918 score 500.000\n",
      "episode:  1919 score 500.000\n",
      "episode:  1920 score 500.000\n",
      "episode:  1921 score 500.000\n",
      "episode:  1922 score 500.000\n",
      "episode:  1923 score 500.000\n",
      "episode:  1924 score 500.000\n",
      "episode:  1925 score 500.000\n",
      "episode:  1926 score 500.000\n",
      "episode:  1927 score 500.000\n",
      "episode:  1928 score 500.000\n",
      "episode:  1929 score 500.000\n",
      "episode:  1930 score 500.000\n",
      "episode:  1931 score 500.000\n",
      "episode:  1932 score 500.000\n",
      "episode:  1933 score 500.000\n",
      "episode:  1934 score 500.000\n",
      "episode:  1935 score 500.000\n",
      "episode:  1936 score 500.000\n",
      "episode:  1937 score 500.000\n",
      "episode:  1938 score 500.000\n",
      "episode:  1939 score 500.000\n",
      "episode:  1940 score 500.000\n",
      "episode:  1941 score 500.000\n",
      "episode:  1942 score 500.000\n",
      "episode:  1943 score 500.000\n",
      "episode:  1944 score 500.000\n",
      "episode:  1945 score 500.000\n",
      "episode:  1946 score 500.000\n",
      "episode:  1947 score 500.000\n",
      "episode:  1948 score 500.000\n",
      "episode:  1949 score 500.000\n",
      "episode:  1950 score 500.000\n",
      "episode:  1951 score 500.000\n",
      "episode:  1952 score 500.000\n",
      "episode:  1953 score 500.000\n",
      "episode:  1954 score 500.000\n",
      "episode:  1955 score 500.000\n",
      "episode:  1956 score 500.000\n",
      "episode:  1957 score 500.000\n",
      "episode:  1958 score 500.000\n",
      "episode:  1959 score 500.000\n",
      "episode:  1960 score 500.000\n",
      "episode:  1961 score 500.000\n",
      "episode:  1962 score 500.000\n",
      "episode:  1963 score 500.000\n",
      "episode:  1964 score 500.000\n",
      "episode:  1965 score 500.000\n",
      "episode:  1966 score 500.000\n",
      "episode:  1967 score 500.000\n",
      "episode:  1968 score 500.000\n",
      "episode:  1969 score 500.000\n",
      "episode:  1970 score 500.000\n",
      "episode:  1971 score 500.000\n",
      "episode:  1972 score 500.000\n",
      "episode:  1973 score 500.000\n",
      "episode:  1974 score 500.000\n",
      "episode:  1975 score 500.000\n",
      "episode:  1976 score 500.000\n",
      "episode:  1977 score 500.000\n",
      "episode:  1978 score 500.000\n",
      "episode:  1979 score 500.000\n",
      "episode:  1980 score 500.000\n",
      "episode:  1981 score 500.000\n",
      "episode:  1982 score 500.000\n",
      "episode:  1983 score 500.000\n",
      "episode:  1984 score 500.000\n",
      "episode:  1985 score 500.000\n",
      "episode:  1986 score 500.000\n",
      "episode:  1987 score 500.000\n",
      "episode:  1988 score 500.000\n",
      "episode:  1989 score 500.000\n",
      "episode:  1990 score 500.000\n",
      "episode:  1991 score 500.000\n",
      "episode:  1992 score 500.000\n",
      "episode:  1993 score 500.000\n",
      "episode:  1994 score 500.000\n",
      "episode:  1995 score 500.000\n",
      "episode:  1996 score 500.000\n",
      "episode:  1997 score 500.000\n",
      "episode:  1998 score 500.000\n",
      "episode:  1999 score 500.000\n",
      "episode:  2000 score 500.000\n",
      "episode:  2001 score 500.000\n",
      "episode:  2002 score 500.000\n",
      "episode:  2003 score 500.000\n",
      "episode:  2004 score 500.000\n",
      "episode:  2005 score 500.000\n",
      "episode:  2006 score 500.000\n",
      "episode:  2007 score 500.000\n",
      "episode:  2008 score 500.000\n",
      "episode:  2009 score 500.000\n",
      "episode:  2010 score 500.000\n",
      "episode:  2011 score 500.000\n",
      "episode:  2012 score 500.000\n",
      "episode:  2013 score 500.000\n",
      "episode:  2014 score 500.000\n",
      "episode:  2015 score 500.000\n",
      "episode:  2016 score 500.000\n",
      "episode:  2017 score 500.000\n",
      "episode:  2018 score 500.000\n",
      "episode:  2019 score 500.000\n",
      "episode:  2020 score 500.000\n",
      "episode:  2021 score 500.000\n",
      "episode:  2022 score 500.000\n",
      "episode:  2023 score 500.000\n",
      "episode:  2024 score 500.000\n",
      "episode:  2025 score 500.000\n",
      "episode:  2026 score 500.000\n",
      "episode:  2027 score 500.000\n",
      "episode:  2028 score 500.000\n",
      "episode:  2029 score 500.000\n",
      "episode:  2030 score 500.000\n",
      "episode:  2031 score 500.000\n",
      "episode:  2032 score 500.000\n",
      "episode:  2033 score 500.000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:  2034 score 500.000\n",
      "episode:  2035 score 500.000\n",
      "episode:  2036 score 500.000\n",
      "episode:  2037 score 500.000\n",
      "episode:  2038 score 500.000\n",
      "episode:  2039 score 500.000\n",
      "episode:  2040 score 500.000\n",
      "episode:  2041 score 500.000\n",
      "episode:  2042 score 500.000\n",
      "episode:  2043 score 500.000\n",
      "episode:  2044 score 500.000\n",
      "episode:  2045 score 500.000\n",
      "episode:  2046 score 500.000\n",
      "episode:  2047 score 500.000\n",
      "episode:  2048 score 500.000\n",
      "episode:  2049 score 500.000\n",
      "episode:  2050 score 500.000\n",
      "episode:  2051 score 500.000\n",
      "episode:  2052 score 500.000\n",
      "episode:  2053 score 500.000\n",
      "episode:  2054 score 500.000\n",
      "episode:  2055 score 500.000\n",
      "episode:  2056 score 500.000\n",
      "episode:  2057 score 500.000\n",
      "episode:  2058 score 500.000\n",
      "episode:  2059 score 500.000\n",
      "episode:  2060 score 500.000\n",
      "episode:  2061 score 500.000\n",
      "episode:  2062 score 500.000\n",
      "episode:  2063 score 500.000\n",
      "episode:  2064 score 500.000\n",
      "episode:  2065 score 500.000\n",
      "episode:  2066 score 500.000\n",
      "episode:  2067 score 500.000\n",
      "episode:  2068 score 500.000\n",
      "episode:  2069 score 500.000\n",
      "episode:  2070 score 500.000\n",
      "episode:  2071 score 500.000\n",
      "episode:  2072 score 500.000\n",
      "episode:  2073 score 500.000\n",
      "episode:  2074 score 500.000\n",
      "episode:  2075 score 500.000\n",
      "episode:  2076 score 500.000\n",
      "episode:  2077 score 500.000\n",
      "episode:  2078 score 500.000\n",
      "episode:  2079 score 500.000\n",
      "episode:  2080 score 500.000\n",
      "episode:  2081 score 500.000\n",
      "episode:  2082 score 500.000\n",
      "episode:  2083 score 500.000\n",
      "episode:  2084 score 500.000\n",
      "episode:  2085 score 500.000\n",
      "episode:  2086 score 500.000\n",
      "episode:  2087 score 500.000\n",
      "episode:  2088 score 500.000\n",
      "episode:  2089 score 500.000\n",
      "episode:  2090 score 500.000\n",
      "episode:  2091 score 500.000\n",
      "episode:  2092 score 500.000\n",
      "episode:  2093 score 500.000\n",
      "episode:  2094 score 500.000\n",
      "episode:  2095 score 500.000\n",
      "episode:  2096 score 500.000\n",
      "episode:  2097 score 500.000\n",
      "episode:  2098 score 500.000\n",
      "episode:  2099 score 500.000\n",
      "episode:  2100 score 500.000\n",
      "episode:  2101 score 500.000\n",
      "episode:  2102 score 500.000\n",
      "episode:  2103 score 500.000\n",
      "episode:  2104 score 500.000\n",
      "episode:  2105 score 500.000\n",
      "episode:  2106 score 500.000\n",
      "episode:  2107 score 500.000\n",
      "episode:  2108 score 500.000\n",
      "episode:  2109 score 500.000\n",
      "episode:  2110 score 500.000\n",
      "episode:  2111 score 500.000\n",
      "episode:  2112 score 500.000\n",
      "episode:  2113 score 500.000\n",
      "episode:  2114 score 500.000\n",
      "episode:  2115 score 500.000\n",
      "episode:  2116 score 500.000\n",
      "episode:  2117 score 500.000\n",
      "episode:  2118 score 500.000\n",
      "episode:  2119 score 500.000\n",
      "episode:  2120 score 500.000\n",
      "episode:  2121 score 500.000\n",
      "episode:  2122 score 500.000\n",
      "episode:  2123 score 500.000\n",
      "episode:  2124 score 500.000\n",
      "episode:  2125 score 500.000\n",
      "episode:  2126 score 500.000\n",
      "episode:  2127 score 500.000\n",
      "episode:  2128 score 500.000\n",
      "episode:  2129 score 500.000\n",
      "episode:  2130 score 500.000\n",
      "episode:  2131 score 500.000\n",
      "episode:  2132 score 500.000\n",
      "episode:  2133 score 500.000\n",
      "episode:  2134 score 500.000\n",
      "episode:  2135 score 500.000\n",
      "episode:  2136 score 500.000\n",
      "episode:  2137 score 500.000\n",
      "episode:  2138 score 500.000\n",
      "episode:  2139 score 500.000\n",
      "episode:  2140 score 500.000\n",
      "episode:  2141 score 500.000\n",
      "episode:  2142 score 500.000\n",
      "episode:  2143 score 500.000\n",
      "episode:  2144 score 500.000\n",
      "episode:  2145 score 500.000\n",
      "episode:  2146 score 500.000\n",
      "episode:  2147 score 500.000\n",
      "episode:  2148 score 500.000\n",
      "episode:  2149 score 500.000\n",
      "episode:  2150 score 500.000\n",
      "episode:  2151 score 500.000\n",
      "episode:  2152 score 500.000\n",
      "episode:  2153 score 500.000\n",
      "episode:  2154 score 500.000\n",
      "episode:  2155 score 500.000\n",
      "episode:  2156 score 500.000\n",
      "episode:  2157 score 500.000\n",
      "episode:  2158 score 500.000\n",
      "episode:  2159 score 500.000\n",
      "episode:  2160 score 500.000\n",
      "episode:  2161 score 500.000\n",
      "episode:  2162 score 500.000\n",
      "episode:  2163 score 500.000\n",
      "episode:  2164 score 500.000\n",
      "episode:  2165 score 500.000\n",
      "episode:  2166 score 500.000\n",
      "episode:  2167 score 500.000\n",
      "episode:  2168 score 500.000\n",
      "episode:  2169 score 500.000\n",
      "episode:  2170 score 500.000\n",
      "episode:  2171 score 500.000\n",
      "episode:  2172 score 500.000\n",
      "episode:  2173 score 500.000\n",
      "episode:  2174 score 500.000\n",
      "episode:  2175 score 500.000\n",
      "episode:  2176 score 500.000\n",
      "episode:  2177 score 500.000\n",
      "episode:  2178 score 500.000\n",
      "episode:  2179 score 500.000\n",
      "episode:  2180 score 500.000\n",
      "episode:  2181 score 500.000\n",
      "episode:  2182 score 500.000\n",
      "episode:  2183 score 500.000\n",
      "episode:  2184 score 500.000\n",
      "episode:  2185 score 500.000\n",
      "episode:  2186 score 500.000\n",
      "episode:  2187 score 500.000\n",
      "episode:  2188 score 500.000\n",
      "episode:  2189 score 500.000\n",
      "episode:  2190 score 500.000\n",
      "episode:  2191 score 500.000\n",
      "episode:  2192 score 500.000\n",
      "episode:  2193 score 500.000\n",
      "episode:  2194 score 500.000\n",
      "episode:  2195 score 500.000\n",
      "episode:  2196 score 500.000\n",
      "episode:  2197 score 500.000\n",
      "episode:  2198 score 500.000\n",
      "episode:  2199 score 500.000\n",
      "episode:  2200 score 500.000\n",
      "episode:  2201 score 500.000\n",
      "episode:  2202 score 500.000\n",
      "episode:  2203 score 500.000\n",
      "episode:  2204 score 500.000\n",
      "episode:  2205 score 500.000\n",
      "episode:  2206 score 500.000\n",
      "episode:  2207 score 500.000\n",
      "episode:  2208 score 500.000\n",
      "episode:  2209 score 500.000\n",
      "episode:  2210 score 500.000\n",
      "episode:  2211 score 500.000\n",
      "episode:  2212 score 500.000\n",
      "episode:  2213 score 500.000\n",
      "episode:  2214 score 500.000\n",
      "episode:  2215 score 500.000\n",
      "episode:  2216 score 500.000\n",
      "episode:  2217 score 500.000\n",
      "episode:  2218 score 500.000\n",
      "episode:  2219 score 500.000\n",
      "episode:  2220 score 500.000\n",
      "episode:  2221 score 500.000\n",
      "episode:  2222 score 500.000\n",
      "episode:  2223 score 500.000\n",
      "episode:  2224 score 500.000\n",
      "episode:  2225 score 500.000\n",
      "episode:  2226 score 500.000\n",
      "episode:  2227 score 500.000\n",
      "episode:  2228 score 500.000\n",
      "episode:  2229 score 500.000\n",
      "episode:  2230 score 500.000\n",
      "episode:  2231 score 500.000\n",
      "episode:  2232 score 500.000\n",
      "episode:  2233 score 500.000\n",
      "episode:  2234 score 500.000\n",
      "episode:  2235 score 500.000\n",
      "episode:  2236 score 500.000\n",
      "episode:  2237 score 500.000\n",
      "episode:  2238 score 500.000\n",
      "episode:  2239 score 82.000\n",
      "episode:  2240 score 500.000\n",
      "episode:  2241 score 500.000\n",
      "episode:  2242 score 500.000\n",
      "episode:  2243 score 88.000\n",
      "episode:  2244 score 500.000\n",
      "episode:  2245 score 500.000\n",
      "episode:  2246 score 500.000\n",
      "episode:  2247 score 500.000\n",
      "episode:  2248 score 500.000\n",
      "episode:  2249 score 500.000\n",
      "episode:  2250 score 500.000\n",
      "episode:  2251 score 500.000\n",
      "episode:  2252 score 500.000\n",
      "episode:  2253 score 500.000\n",
      "episode:  2254 score 500.000\n",
      "episode:  2255 score 500.000\n",
      "episode:  2256 score 500.000\n",
      "episode:  2257 score 500.000\n",
      "episode:  2258 score 500.000\n",
      "episode:  2259 score 500.000\n",
      "episode:  2260 score 500.000\n",
      "episode:  2261 score 500.000\n",
      "episode:  2262 score 500.000\n",
      "episode:  2263 score 500.000\n",
      "episode:  2264 score 500.000\n",
      "episode:  2265 score 500.000\n",
      "episode:  2266 score 500.000\n",
      "episode:  2267 score 500.000\n",
      "episode:  2268 score 500.000\n",
      "episode:  2269 score 500.000\n",
      "episode:  2270 score 500.000\n",
      "episode:  2271 score 500.000\n",
      "episode:  2272 score 500.000\n",
      "episode:  2273 score 500.000\n",
      "episode:  2274 score 500.000\n",
      "episode:  2275 score 500.000\n",
      "episode:  2276 score 500.000\n",
      "episode:  2277 score 500.000\n",
      "episode:  2278 score 500.000\n",
      "episode:  2279 score 500.000\n",
      "episode:  2280 score 500.000\n",
      "episode:  2281 score 500.000\n",
      "episode:  2282 score 500.000\n",
      "episode:  2283 score 500.000\n",
      "episode:  2284 score 500.000\n",
      "episode:  2285 score 500.000\n",
      "episode:  2286 score 500.000\n",
      "episode:  2287 score 500.000\n",
      "episode:  2288 score 500.000\n",
      "episode:  2289 score 500.000\n",
      "episode:  2290 score 500.000\n",
      "episode:  2291 score 500.000\n",
      "episode:  2292 score 500.000\n",
      "episode:  2293 score 500.000\n",
      "episode:  2294 score 500.000\n",
      "episode:  2295 score 500.000\n",
      "episode:  2296 score 500.000\n",
      "episode:  2297 score 500.000\n",
      "episode:  2298 score 500.000\n",
      "episode:  2299 score 500.000\n",
      "episode:  2300 score 500.000\n",
      "episode:  2301 score 500.000\n",
      "episode:  2302 score 500.000\n",
      "episode:  2303 score 500.000\n",
      "episode:  2304 score 495.000\n",
      "episode:  2305 score 500.000\n",
      "episode:  2306 score 500.000\n",
      "episode:  2307 score 500.000\n",
      "episode:  2308 score 500.000\n",
      "episode:  2309 score 500.000\n",
      "episode:  2310 score 500.000\n",
      "episode:  2311 score 500.000\n",
      "episode:  2312 score 500.000\n",
      "episode:  2313 score 500.000\n",
      "episode:  2314 score 500.000\n",
      "episode:  2315 score 500.000\n",
      "episode:  2316 score 500.000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:  2317 score 500.000\n",
      "episode:  2318 score 396.000\n",
      "episode:  2319 score 494.000\n",
      "episode:  2320 score 500.000\n",
      "episode:  2321 score 500.000\n",
      "episode:  2322 score 500.000\n",
      "episode:  2323 score 500.000\n",
      "episode:  2324 score 500.000\n",
      "episode:  2325 score 500.000\n",
      "episode:  2326 score 500.000\n",
      "episode:  2327 score 500.000\n",
      "episode:  2328 score 500.000\n",
      "episode:  2329 score 324.000\n",
      "episode:  2330 score 500.000\n",
      "episode:  2331 score 457.000\n",
      "episode:  2332 score 500.000\n",
      "episode:  2333 score 427.000\n",
      "episode:  2334 score 446.000\n",
      "episode:  2335 score 500.000\n",
      "episode:  2336 score 500.000\n",
      "episode:  2337 score 321.000\n",
      "episode:  2338 score 380.000\n",
      "episode:  2339 score 500.000\n",
      "episode:  2340 score 442.000\n",
      "episode:  2341 score 500.000\n",
      "episode:  2342 score 500.000\n",
      "episode:  2343 score 500.000\n",
      "episode:  2344 score 500.000\n",
      "episode:  2345 score 420.000\n",
      "episode:  2346 score 500.000\n",
      "episode:  2347 score 500.000\n",
      "episode:  2348 score 362.000\n",
      "episode:  2349 score 500.000\n",
      "episode:  2350 score 500.000\n",
      "episode:  2351 score 330.000\n",
      "episode:  2352 score 315.000\n",
      "episode:  2353 score 500.000\n",
      "episode:  2354 score 500.000\n",
      "episode:  2355 score 500.000\n",
      "episode:  2356 score 500.000\n",
      "episode:  2357 score 500.000\n",
      "episode:  2358 score 500.000\n",
      "episode:  2359 score 500.000\n",
      "episode:  2360 score 500.000\n",
      "episode:  2361 score 500.000\n",
      "episode:  2362 score 247.000\n",
      "episode:  2363 score 500.000\n",
      "episode:  2364 score 431.000\n",
      "episode:  2365 score 469.000\n",
      "episode:  2366 score 393.000\n",
      "episode:  2367 score 468.000\n",
      "episode:  2368 score 401.000\n",
      "episode:  2369 score 331.000\n",
      "episode:  2370 score 500.000\n",
      "episode:  2371 score 489.000\n",
      "episode:  2372 score 417.000\n",
      "episode:  2373 score 325.000\n",
      "episode:  2374 score 370.000\n",
      "episode:  2375 score 377.000\n",
      "episode:  2376 score 358.000\n",
      "episode:  2377 score 336.000\n",
      "episode:  2378 score 352.000\n",
      "episode:  2379 score 500.000\n",
      "episode:  2380 score 370.000\n",
      "episode:  2381 score 261.000\n",
      "episode:  2382 score 403.000\n",
      "episode:  2383 score 350.000\n",
      "episode:  2384 score 239.000\n",
      "episode:  2385 score 500.000\n",
      "episode:  2386 score 231.000\n",
      "episode:  2387 score 486.000\n",
      "episode:  2388 score 259.000\n",
      "episode:  2389 score 257.000\n",
      "episode:  2390 score 201.000\n",
      "episode:  2391 score 341.000\n",
      "episode:  2392 score 270.000\n",
      "episode:  2393 score 197.000\n",
      "episode:  2394 score 243.000\n",
      "episode:  2395 score 260.000\n",
      "episode:  2396 score 283.000\n",
      "episode:  2397 score 304.000\n",
      "episode:  2398 score 250.000\n",
      "episode:  2399 score 494.000\n",
      "episode:  2400 score 269.000\n",
      "episode:  2401 score 388.000\n",
      "episode:  2402 score 407.000\n",
      "episode:  2403 score 412.000\n",
      "episode:  2404 score 400.000\n",
      "episode:  2405 score 262.000\n",
      "episode:  2406 score 246.000\n",
      "episode:  2407 score 308.000\n",
      "episode:  2408 score 432.000\n",
      "episode:  2409 score 281.000\n",
      "episode:  2410 score 386.000\n",
      "episode:  2411 score 301.000\n",
      "episode:  2412 score 307.000\n",
      "episode:  2413 score 297.000\n",
      "episode:  2414 score 268.000\n",
      "episode:  2415 score 244.000\n",
      "episode:  2416 score 264.000\n",
      "episode:  2417 score 258.000\n",
      "episode:  2418 score 277.000\n",
      "episode:  2419 score 412.000\n",
      "episode:  2420 score 392.000\n",
      "episode:  2421 score 256.000\n",
      "episode:  2422 score 259.000\n",
      "episode:  2423 score 232.000\n",
      "episode:  2424 score 262.000\n",
      "episode:  2425 score 299.000\n",
      "episode:  2426 score 256.000\n",
      "episode:  2427 score 250.000\n",
      "episode:  2428 score 278.000\n",
      "episode:  2429 score 202.000\n",
      "episode:  2430 score 297.000\n",
      "episode:  2431 score 225.000\n",
      "episode:  2432 score 190.000\n",
      "episode:  2433 score 247.000\n",
      "episode:  2434 score 228.000\n",
      "episode:  2435 score 286.000\n",
      "episode:  2436 score 246.000\n",
      "episode:  2437 score 233.000\n",
      "episode:  2438 score 253.000\n",
      "episode:  2439 score 235.000\n",
      "episode:  2440 score 141.000\n",
      "episode:  2441 score 263.000\n",
      "episode:  2442 score 281.000\n",
      "episode:  2443 score 259.000\n",
      "episode:  2444 score 202.000\n",
      "episode:  2445 score 491.000\n",
      "episode:  2446 score 184.000\n",
      "episode:  2447 score 172.000\n",
      "episode:  2448 score 184.000\n",
      "episode:  2449 score 153.000\n",
      "episode:  2450 score 199.000\n",
      "episode:  2451 score 182.000\n",
      "episode:  2452 score 289.000\n",
      "episode:  2453 score 230.000\n",
      "episode:  2454 score 271.000\n",
      "episode:  2455 score 276.000\n",
      "episode:  2456 score 189.000\n",
      "episode:  2457 score 146.000\n",
      "episode:  2458 score 220.000\n",
      "episode:  2459 score 172.000\n",
      "episode:  2460 score 132.000\n",
      "episode:  2461 score 176.000\n",
      "episode:  2462 score 440.000\n",
      "episode:  2463 score 213.000\n",
      "episode:  2464 score 202.000\n",
      "episode:  2465 score 143.000\n",
      "episode:  2466 score 113.000\n",
      "episode:  2467 score 292.000\n",
      "episode:  2468 score 149.000\n",
      "episode:  2469 score 170.000\n",
      "episode:  2470 score 122.000\n",
      "episode:  2471 score 181.000\n",
      "episode:  2472 score 188.000\n",
      "episode:  2473 score 262.000\n",
      "episode:  2474 score 137.000\n",
      "episode:  2475 score 201.000\n",
      "episode:  2476 score 136.000\n",
      "episode:  2477 score 126.000\n",
      "episode:  2478 score 167.000\n",
      "episode:  2479 score 171.000\n",
      "episode:  2480 score 203.000\n",
      "episode:  2481 score 134.000\n",
      "episode:  2482 score 157.000\n",
      "episode:  2483 score 140.000\n",
      "episode:  2484 score 188.000\n",
      "episode:  2485 score 118.000\n",
      "episode:  2486 score 163.000\n",
      "episode:  2487 score 134.000\n",
      "episode:  2488 score 173.000\n",
      "episode:  2489 score 118.000\n",
      "episode:  2490 score 111.000\n",
      "episode:  2491 score 131.000\n",
      "episode:  2492 score 194.000\n",
      "episode:  2493 score 225.000\n",
      "episode:  2494 score 141.000\n",
      "episode:  2495 score 194.000\n",
      "episode:  2496 score 142.000\n",
      "episode:  2497 score 123.000\n",
      "episode:  2498 score 182.000\n",
      "episode:  2499 score 285.000\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import gym\n",
    "from gym import wrappers#save playing video footage\n",
    "import torch\n",
    "from torch.distributions import constraints\n",
    "from torch.distributions.exp_family import ExponentialFamily\n",
    "from torch.distributions.utils import _standard_normal, broadcast_all\n",
    "\n",
    "if __name__=='__main__':\n",
    "    #env=gym.make('BipedalWalker-v2')\n",
    "    env=gym.make('CartPole-v1')\n",
    "    agent=Agent(lr=0.001, state_dim=[4], gamma=0.99, num_layers=3, hidden_dim=256,num_actions=2)\n",
    "    score_history=[]\n",
    "    score=0\n",
    "    n_episodes=2500\n",
    "        \n",
    "    for i in range(n_episodes):\n",
    "        print('episode: ',i, 'score %.3f' % score)\n",
    "        done=False\n",
    "        score = 0\n",
    "        observation = env.reset()\n",
    "        while not done:\n",
    "            action=agent.choose_action(observation)\n",
    "            observation_new,reward,done,info=env.step(action)\n",
    "#           agent.store_rewards(reward)\n",
    "            agent.reward_memory.append(reward)\n",
    "            observation=observation_new\n",
    "            score += reward\n",
    "        score_history.append(score)\n",
    "        agent.learn()\n",
    "    filename='CartPole-v1.png'\n",
    "    #plotLearningNoEpsilons(score_history, filename=filename,windoe=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Rolling average\n",
    "\n",
    "def rolling_average(data, *, window_size):\n",
    "    \"\"\"Smoothen the 1-d data array using a rollin average.\n",
    "\n",
    "    Args:\n",
    "        data: 1-d numpy.array\n",
    "        window_size: size of the smoothing window\n",
    "\n",
    "    Returns:\n",
    "        smooth_data: a 1-d numpy.array with the same size as data\n",
    "    \"\"\"\n",
    "    #assert data.ndim == 1\n",
    "    kernel = np.ones(window_size)\n",
    "    smooth_data = np.convolve(data, kernel) / np.convolve(\n",
    "        np.ones_like(data), kernel\n",
    "    )\n",
    "    return smooth_data[: -window_size + 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'returns')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxdZZ3H8c8va9s0aZIm3dKm6ZKuUKC0pVBkVWTRKSoI6MgiUh3F0QFHcBsRZ0Z0VNBRUBAcQBYRUJBFKZusXWnpvqRt2qRb0mZvkmZ75o9zEtI2aW/b3Hvu8n2/Xvd1z33O9ntyb+7vnvOc8zzmnENERAQgKegAREQkeigpiIhIFyUFERHpoqQgIiJdlBRERKSLkoKIiHRRUhCJEDMrMjNnZil9tL0vmtldfb1tM3vazC48/gglFvXJh1NEIsvM0oDvArPDsPk7gHuAv4Vh2xLldKQgCcc8sf7Znwusc85t7+sNO+cWAVlmNqOvty3RL9b/MSROmdktZrbdzOrNbL2Zne+XJ5vZt81skz9vqZmN8uedYWaLzazWfz6j2/ZeN7P/MrO3gUZgrJkNMrP7zWynv6//NLPkHmIZYWZNZpbbrewUM9tjZqlmNt7M/uHvd4+Z/THEOva6fzO71szeMrOfmlm1mW0xs4u6rX4R8I8wbRvgdeCSUOoh8UVJQaKOmU0EbgRmOucygY8Cpf7sm4CrgIuBLODzQKP/hf088EtgMPBz4HkzG9xt058D5gGZwFbgQaANGA+cAlwAfOHgeJxzO4B3gU91K/4M8KRzrhX4IfASkAOMBP43xKoeaf+nAeuBPOAnwP1mZv68E/154dg2wFrgpBDrIfHEOaeHHlH1wPsiqwA+DKQeNG89MLeHdT4HLDqo7F3gWn/6deD2bvOGAvuB/t3KrgJe6yWmLwCv+tMGlAFn+a8fAu4FRh6hXkWAw2vLO+z+gWuBkm7zBvjrDvNfbwQuDMe2/bIbOuurR2I91NAsUcc5V2JmXwduA6aa2d+Bm5z3i30UsKmH1Ubg/frvbitQ0O11Wbfp0UAqsLPbD+Skg5bp7kngf81sBFCM9yX6pj/vm3hHC4vMrBr4mXPugSNUM5T97+qccM41+ssN9Iuq8Y54wrFt/G3XHKEOEoeUFCQqOeceBR41syzgt8CP8Y4GyoBxwKqDVtmB92XYXSEHXkHTvUvgMrxf03nOubYQ4qkxs5eATwOTgcec835SO+d24f2yxszOBF42szeccyWH2eRR7b8HK4AJYdo2eHV8/xjXlRimNgWJOmY20czOM7N0oBloAtr92b8Dfmhmxf5VRNP8doMXgAlm9hkzSzGzK4ApwHM97cM5txOvHeBnZpZlZklmNs7Mzj5MaI8CV+O1LTzaLd7LzWyk/7IaL/m0H7r6ce+/uxeAHpftg23jb/vFo1he4oSSgkSjdLxr5ffgneYYAnzbn/dz4Am8L7064H68c+d7gY8BNwN78U7pfMw5t+cw+7kaSAPW4H2ZPwkMP8zyz+KdOtrtnOv+K3omsNDMGvxlvuac2xJCPY92/939FZjkn87q022b2Uxgn/MuTZUEY/4RsIjEGDObB0xxzn29j7f7FHC/c+6FvtyuxAYlBRER6aLTRyIi0kVJQUREuigpiIhIFyUFERHpEtM3r+Xl5bmioqKgwxARiSlLly7d45zL72leTCeFoqIilixZEnQYIiIxxcwO7hKmi04fiYhIFyUFERHpoqQgIiJdlBRERKRLWJOCmZWa2UozW25mS/yyXDObb2Yb/eccv9zM7JdmVmJmK8xsejhjExGRQ0XiSOFc59zJzrnOQcBvBV5xzhUDr/ivwRtztth/zAPuiUBsIiLSTRCnj+bijR+L/3xpt/KHnGcBkG1moXYjLCIifSDc9yk44CUzc8BvnXP3AkP9QUBwzu00syH+sgUcOFxguV+2M8wxSoxrbm3n+8+s5uW1uw+Zd8BQ9F7JYecfvHjnfOfgxIJB/O6aGdihG+1za3fW8bOXNjBlRBa5A1JJSU4iNdlITU5izvg8BmekkZJ8/L/papta2d/aTkt7B23tjrLqRhZtqaKlvYMNu+qpaWqlcX87yUl2wLB1PfWu3Fnkui35Qdmh6x6whRCXO3gf3cPoqcPnULbRe5wHBdetzAxu+6epfGxab8NZxK5wJ4U5zrkd/hf/fDNbd5hle/pPO+Rt9vuQnwdQWFjYN1FKTGpt7+DO+Rt4bsVOtlU1ctEJwxg8MK1r/sFfEgd/mA79EnG9zn98cRmvrKvg5Nvnc+tFk7hqVng/e5+4+22aWzt6THSdCnMHMDQrncLcDJKOMk85YHlZDSUVDT3OTzLIzUijMHcAgwakktUvFTgwiXZOHlhmh5bZgfO6r9w97M5ke2BZCMsdaf9HGWf3Dfa8f/jDgm3c+OgyhmX1+yCeHvaZm5HG6MEZxJKwJgV/oHWccxVm9mdgFrDbzIb7RwnDgQp/8XK8Qdk7jcQbd/fgbd4L3AswY8YMDQaRwJ5ZvoO7X9/E1BFZPPj5WZw9oce79vvEty6azM1/ep+X1+7mW0+vZPLwLE4elR2WfZVUNNDc2sEZ4wZz+9wTGJyRRmt7B60djiWlVWzb20htUyubKhvYWdvMu5sON7hc7/qnJfPFs8dSmDuA1KQkUlOMlKQkxuZnMHXEoD6uVXxp74DHFm3jst+8e8RlU5KMy04dyQ/mTiU9JTkC0R2fsA2yY2YZQJJzrt6fng/cDpwP7HXO3WFmtwK5zrlvmtklwI3AxcBpwC+dc7MOt48ZM2Y4dXORuC7/zTvsbWjhlZvPjsgpHYAFm/dyw4NLOLUoh/+77rAfz2Oyb38b/3z/QlaW17Lg2+eTNzC9z/chx6+1vYPFpVW0tneexup2iqnz2TnW72rg8cXb2Lq3kSnDs7h97lRmFOUGEPGBzGxpt4t/DhDOI4WhwJ/9f9YU4FHn3N/MbDHwhJldD2wDLveXfwEvIZQAjcB1YYxNYtzqHbUsLq3m1osmRSwhAMweO5hr5xTxq9dKKN2zj6K8vjs1cOf8DfzilY0A3PChMUoIUSw1OYkzxuUdcbnzJg3lS2eP5aYn3ufPy7Zz2W/e5esfLua6M8YwaEBqBCI9ejE9HKeOFBLX1Q8sYkV5Da9/4xyyB6QdeYU+tKu2mdk/egWA8yYNYWhWvx6W6vn/qrd/t02VDSwurQbggilDuffqHn/ESQxbtb2WT979Di3tHZw6Oocnv3R6RH/QdBfUkYJIWOyqbeaNDZXc/JEJEU8IAMMG9eNHnzyRX71awsrttazcXtvjcr39u/f0PeAcDMlM56l/OYNRuQP6LliJGicUDGLp9z7MD/66hieXlvM/f1/PNy+cFHRYh1BSkJizuLQKgLMnhq9h+UiumlUY9iuQJP5k9kvlJ5+axobd9Tzw9hZu+NBYcjIi/8PmcNT3kcScpVur6Z+azOThWUGHInLUkpKM/7nsJJpbO3hkYa/DGgRGSUFizqrttUwdkUVqH9y8JRKEicMyOXtCPr9/u5T9be1Bh3MA/VdJTHHOsX53PROHZQYdishx+exphezd18J9b2wOOpQDKClITNlZ20x9cxuTlBQkxp07yevhZ+GWqoAjOZCSgsSU9bvqAZg4TO0JEttSk5O49owilpRW09reEXQ4XZQUJKas3+0nhaE6UpDYd9qYXJpa21lR3vNlzUFQUpCYsn5XPcOy+kXt3aAiR2PWGK/LiwWb9wYcyQeUFCSmbKpsoHjowKDDEOkTgwemM3FoJm+XHFunhuGgpCAxpayqUXf8SlyZPTaXdzbtZXddc9ChAEoKEkPqm1upbmxlVI6SgsSPT506EoDf/iM6Lk1VUpCYUVbVBMCo3P4BRyLSd6aNzOZj04bzzPLttHcE30GpkoLEjLLqRsAbcUwknnx06jD27mth6dbqoENRUpDYUVblJQWdPpJ4c+6kIfRPTea5FYcMNhlxSgoSM8qrmxiYnkK2LkeVODMwPYXZY3N5a2PwVyEpKUjMKKtqZGRO/8AGJhEJpzOL89m8Zx/l/mnSoCgpSMwoq9blqBK/zvP7Qvrbql2BxqGkIDHBOUdZVZPaEyRujcnLYOqILCUFkVDsaWihqbWdQl2OKnFszvg8VpTX0twa3BgLSgoSEzovR9XpI4lnM4tyaWnv4P2ymsBiUFKQmNB1OaqSgsSxGaNzgA/GIQ+CkoLEhPJq727mkTk6fSTxKycjjQlDB7KoNLib2JQUJCaUVTWSNzCNAWkpQYciElYzi3J5b2s1bQENvKOkIDGhrLqRkbrySBLArDG5NOxvY+3O+kD2r6QgMWFbVaP6PJKEMLPIG3hnUUDtCkoKEvXa2jvYUdOs3lElIYzI9j7nP3xuTSD7V1KQqFde3UR7h2N0bkbQoYhExIC0ZABKKiJ/CklJQaJeSUUDAOOGaBhOSQz/d90sAC77zbsR37eSgkS9kkovKYxXUpAEMbPIu19hUP/I9wispCBRr6SigfzM9ED+QUSCYGZ87fxiyqoa2be/LaL7VlKQqLepsoHx+TpKkMRy0qhBdDhYvaMuovsNe1Iws2QzW2Zmz/mvx5jZQjPbaGZ/NLM0vzzdf13izy8Kd2wS/do7HBt3N1A8VElBEssJBYMAWFEe2X6QInGk8DVgbbfXPwbudM4VA9XA9X759UC1c248cKe/nCS45WU1NOxv45TC7KBDEYmoIZn9GD6oHyvKayO637AmBTMbCVwC/M5/bcB5wJP+Ig8Cl/rTc/3X+PPPNw2xldDe21bNp+55B4AZo3MDjkYk8qaNHMT7cXakcBfwTaCzE4/BQI1zrrPlpBwo8KcLgDIAf36tv7wkoI4Oxw+fW4MZPPj5WeodVRLSyaNy2Lq3kZrGlojtM2y9i5nZx4AK59xSMzuns7iHRV0I87pvdx4wD6CwsLAPIpWgOeeY9/BSqvd98MHfVddMeXUTt1w4ibMn5AcYnUhwpo7IAmDtznpOHxeZ38jhPFKYA/yTmZUCj+OdNroLyDazzmQ0EtjhT5cDowD8+YOAQzr/cM7d65yb4ZybkZ+vL4t48MzyHcxfs5uV22tJT00iPTWJosEZ3HrRJL509tigwxMJzKThmQCs2xW5K5DCdqTgnPsW8C0A/0jhG865z5rZn4DL8BLFNcAz/irP+q/f9ee/6pw75EhB4s/v395CYe4AXrn5bFKTdZW0SKf8gekMzkhjSWk1180ZE5F9BvEfeAtwk5mV4LUZ3O+X3w8M9stvAm4NIDaJsIb9bazaUcfck0coIYgcxMw4Y3xeREdii8iIJc6514HX/enNwKwelmkGLo9EPBI9lpRW0d7hOG2MrikQ6clJIwfx1/d3UFm/n/zM9LDvTz/NJFALt1SRkmRMH637EER6MnWEdxPb2p2RaVdQUpBALd5SxYkjB2mYTZFeTBnuXYG0RklB4p1zjvW76jnB/yUkIocaNCCVguz+EesDSUlBAlNZv5/6/W2My9fgOSKHM2VElk4fSfzrHDxn/JDMgCMRiW6Th2exubKB5tb2sO9LSUECo8FzREIzZXgWHQ7W7wr/8JxKChKYkooGBqanMDQr/JfZicSyyRG8s1lJQQJTUtHAuPwM1BmuyOGNyhnAgLRk1ulIQeLZpsoGxunUkcgRJSUZxUMzWbdTSUHiVF1zK7vr9qs9QSREk4dlsm5XHeHuEk5JQQKxqfPKI429LBKSicMyqW5spbJ+f1j3o6QggfjgclQlBZFQTBrm3dkc7nYFJQUJREllA6nJRqFGVBMJyaRh3hVI4b4sVUlBjtvqHbXsbTi6Q9oNu+oZmzeQFHWXLRKSnIw0hmSmszbMl6XqP1KOyyMLt3LJL9/i9B+9Snl1Y8jrrdpR1zXUoIiEZtLwLB0pSHR7+N2tJBm0tHfwoxfWhbRORV0zlfX7mVqgjvBEjsakYZlsrGigrb0jbPtQUpBjdvMT77NuVz03XzCRmz4ygedX7uTFlTuPuF5nb48n6EhB5KhMGpZJS1sHpXv3hW0fSgpyTNbtquOp98oBOGdiPvPOGsukYZl886kVlFUd/jTS6h21gNfzo4iEbqLf2Lw2jDexKSnIMXljQyUAN39kAlNHDKJfajK//dypGHDDQ0toaum9N8dV2+soGjyAzH6pEYpWJD6MHzKQ5CQLa7uCkoIc4qF3Szn59pd4Zvn2Huc75/jvF9ZRPGQgXz2/uKt89OAMfnr5SazbVc+r6yp6XXdZWbXaE0SOQXpKMmPzMsLaMZ6SghygtqmV/3hmNTWNrdw5f0OPyzyxpAyA8ycPPWTeuZOG0D81mcWlVT2uu6K8lt11+zl34pC+C1okgUwanhXWG9iUFOQAr6zdDcCpo3Mo3dvImh6GAHzo3a0AfOOCCYfMS01OYvrobBZu6Tkp/O6tLSQnGedPUlIQORaThmVSXt1EfXNrWLavpCAHWLi5iqx+KfzksmkAvL7hwNNAr67bzeoddXzhzDG93ng2q2gw63bVUdt04Ie2o8Px+voKJg/PJCcjLTwVEIlznXc2b9gdnqMFJQU5wKLSKmaNyWVc/kAmDs3k7ZI9XfM6OhxfeWQZAJ87fXSv2/jQhDycg5N+8BLtHY49Dft5c2Ml9765mfrmNq45vSjc1RCJW+G+AiklLFuVmFRR18yWPfv4zKxCAM4szuPhBVtpbm2nX2oy//3CWppa2/n2xZMYPTij1+1ML8whMz2F+v1tvF2yhzteXMcaf9DxlCTjtDGDI1IfkXhUkN2fO684iZlFuWHZvo4UpMvr673LTGeP9b60TxuTS0tbB6u219Lc2s4jC7eRmmx85rTejxI6Lf7uhwG4+oFFrNlZx8Shmfxx3mze+dZ5FA5WJ3gix8rM+MQpIxmZE57/Ix0pSJc1O+vISEvmhALvprKTR2UDsHRrNRsrGmhqbefuz05nYPqRPzb9UpP5/sen8KtXS7hqViH/9pEJJCdp2E2RaKekIF02VTYwNn9g15jJQ7L6MX7IQH70otenUd7ANM6fHPpVQ9fNGcN1c8aEJVYRCQ+dPpIumyv3MTb/wLaCC6Z8cC/CXVecQnpKcqTDEpEI0pGCANDU0s72miY+nTfqgPIvnTOO0r37+ME/nUB+ZnpA0YlIpCgpCABb9ni9Lo4bcuCRQla/VO7+7KlBhCQiAdDpIwFg8x5vzOSxeRozWSSRhS0pmFk/M1tkZu+b2Woz+4FfPsbMFprZRjP7o5ml+eXp/usSf35RuGKTQ22q8I4UxuT1fv+BiMS/cB4p7AfOc86dBJwMXGhms4EfA3c654qBauB6f/nrgWrn3HjgTn85iZDNexooyO5P/zQ1JIsksrAlBedp8F+m+g8HnAc86Zc/CFzqT8/1X+PPP986r42UsOvpyiMRSTxhbVMws2QzWw5UAPOBTUCNc67NX6QcKPCnC4AyAH9+LXBIfwhmNs/MlpjZksrKynCGnzCcc2yubGBcvtoTRBJdWJOCc67dOXcyMBKYBUzuaTH/uaejAndIgXP3OudmOOdm5Ofn912wCWx33X72tbQzTkcKIgkvpKRgZpebWaY//V0ze9rMpoe6E+dcDfA6MBvINrPOS2FHAjv86XJglL+PFGAQ0HOn/NKnNlf6Vx7pSEEk4YV6pPA951y9mZ0JfBTv3P89h1vBzPLNLNuf7g98GFgLvAZc5i92DfCMP/2s/xp//qvOuUOOFKTvlXQlBR0piCS6UJNC5yjslwD3OOeeAY40Sspw4DUzWwEsBuY7554DbgFuMrMSvDaD+/3l7wcG++U3AbeGXg05Hht215OZnsKwrH5BhyIiAQv1jubtZvZbvF/7PzazdI6QUJxzK4BTeijfjNe+cHB5M3B5iPFIH9qwu4HioR90hCciiSvUI4VPA38HLvTbB3KBfw9bVBIxzjk27q5nwtDMoEMRkSgQUlJwzjXinfvfZ2aFePccrAtnYBIZZVVNVDe2MrVgUNChiEgUCOn0kZl9Ffg+sBvo8IsdMC1McUmELC71LvCaFaah/UQktoTapvA1YKJzbm84g5HIW7qtmsx+KRQP0eWoIhJ6m0IZ3h3GEmdWba9l2shBJGmoTBEh9COFzcDrZvY8Xkd3ADjnfh6WqCQiWts7WLeznuvmFAUdiohEiVCTwjb/kcaR70+QGLF2Zx0t7R2cOFKNzCLiOWJSMLNkYKBzTpegxhHnHJ+8+x0ATh2dE3A0IhItjtim4JxrB0Lu50hiw9Kt1bR1eL2IDB/UP+BoRCRahHr6aLmZPQv8CdjXWeicezosUUnYvV3iXUi27HsfCTgSEYkmoSaFXGAv3gA5nRygpBCDOjocd768gf6pyeRkqIlIRD4QUlJwzl0X7kAkcrZWNQIwfXR2wJGISLQJ9Y7m39PzgDef7/OIJOxWbvduOfnOxVMCjkREok2op4+e6zbdD/gEHwyOIzFmZXkNaSlJFA/VXcwicqBQTx891f21mT0GvByWiCTsVm6vZfLwLFKTwzoaq4jEoGP9VigGCvsyEImMjg7Hqu11TFOvqCLSg1DbFOo5sE1hF94IahJjSvfuo2F/GycqKYhID0I9faQRWOJEZyOzurYQkZ6EdPrIzF4JpUyi3/KyGtJTktRVtoj06LBHCmbWDxgA5JlZDtDZv3IWMCLMsUkfa2xp44WVO5kzPo8UNTKLSA+OdProi8DX8RLAe93K64Bfhyso6XtV+1r40sNLqazfz/Vnjgk6HBGJUodNCs65XwC/MLOvOuf+N0IxSR8rq2rkmt8voryqiV9ceQpzxucFHZKIRKlQb157wMy+CxQ65+aZWTHe8JzPHWlFCU5FXTNffuQ9b8jN9BT+8IXTmDVGYzGLSO9CPbH8ANACnOG/Lgf+MywRSZ/5w8JtLNlazUUnDOPpL89RQhCRIwr1SGGcc+4KM7sKwDnXZGYa1DfKvbauginDs7j7s6cGHYqIxIhQjxRazKw//g1sZjaObmM1S3Tp6HCc+eNXWbm9loHpoeZ9EZHQhuM04DfA34BRZvYIMAe4NryhybF6v7yG8uomAL587riAoxGRWHLEpOCcc2b2NeACYDbevQpfc87tCXdwcmx+8Nc1ANz7uVM5Z+KQgKMRkVgS6rmFBcBY59zz4QxGjl9LWwfLy2oAuGDqsICjEZFYE2pSOBf4opltxRuj2fAOIqaFLTI5Jv/YUAnAHZ88MeBIRCQWhZoULgprFNJn/rysHDO49JSCoEMRkRgU0tVHzrmtPT0Ot46ZjTKz18xsrZmt9tslMLNcM5tvZhv95xy/3Mzsl2ZWYmYrzGz68Vcv8ZRUNHBqYQ79UpODDkVEYlA4e0VrA252zk3Ga6D+iplNAW4FXnHOFQOv+K/BOxop9h/zgHvCGFtcam5tZ1PlPk4bq5vUROTYhC0pOOd2Oufe86frgbVAATAXeNBf7EHgUn96LvCQ8ywAss1seLjii0clFQ20dzgmD88KOhQRiVER6T/ZzIqAU4CFwFDn3E7wEgfQec1kAVDWbbVyv0xCtHZnHYCSgogcs7AnBTMbCDwFfN05V3e4RXsoc4csZDbPzJaY2ZLKysq+CjMurN5RR//UZIoGZwQdiojEqLAmBTNLxUsIjzjnnvaLd3eeFvKfK/zycmBUt9VHAjsO3qZz7l7n3Azn3Iz8/PzwBR+DVm2v5YSCLJKT1C2ViBybsCUFv3uM+4G1zrmfd5v1LHCNP30N8Ey38qv9q5BmA7Wdp5nkyBpb2li1o5YTC7KDDkVEYlg4e0ubA3wOWGlmy/2ybwN3AE+Y2fXANuByf94LwMVACdAIXBfG2OLOe1traG7t4KwJGkBHRI5d2JKCc+4tem4nADi/h+Ud8JVwxRPv3tm0BzOYPjon6FBEJIZp9PY48fLa3UwvzCGrX2rQoYhIDFNSiAMVdc1s2N3AeZPUI6qIHB8lhTjw8lrvAq6zJ+hqLBE5PkoKceD9shpyBqQydYRuWhOR46OkEAfeL69h2shsNGy2iBwvJYUY97dVO1m3q16d4IlIn1BSiHFPLi0H4MqZhQFHIiLxQEkhhjnnWLatho+fNILcjLSgwxGROKCkEMO21zSxd18LM3TDmoj0ESWFGLayvBaAk0epvyMR6RtKCjGspKIBgOKhAwOORETihZJCDCupbKAguz8D0sLZr6GIJBIlhRi2anstE4dlBh2GiMQRJYUYtbuumU2V+5it+xNEpA8pKcSo3725GYDTx2r8BBHpO0oKMcg5x31vbgFQf0ci0qeUFGLQHS+uA+Ca00eTpPGYRaQP6bKVGOKc486XN/LbNzYzNCud731sStAhiUicUVKIEU0t7dz+3GoeW1TGJ08p4I5PTSMlWQd6ItK3lBRiQFlVI198eClrd9XxxbPGcsuFk3TaSETCQkkhym3YXc+V9y6gtb2DB66dybkTNeSmiISPkkIUc87xnT+vBOCZr8xhbL66sxCR8NJJ6Sj24qpdLC6t5pYLJyohiEhEKClEsedX7mRIZjqXnzoq6FBEJEEoKUQp5xzvlOzh7An5alQWkYhRUohS1Y2tVDe2Mmm47lgWkchRUohSW/Z4YyWMyRsQcCQikkiUFKLU5sp9ABQNzgg4EhFJJEoKUWpJaTVZ/VIYraQgIhGkpBCFnHO8VbKH08cNJlmNzCISQUoKUWjr3ka21zRx5niNlSAikaWkEIXe3rQHgDlKCiISYWFLCmb2gJlVmNmqbmW5ZjbfzDb6zzl+uZnZL82sxMxWmNn0cMUVTeqaW/nRC2s56yev8eVHltLc2g7AWxv3MGJQP8bkqT1BRCIrnH0f/R/wK+ChbmW3Aq845+4ws1v917cAFwHF/uM04B7/OW41tbQz7baXABiYnsILK3fxwsq/Mb0wm/e21XDlzFGYqT1BRCIrbEcKzrk3gKqDiucCD/rTDwKXdit/yHkWANlmNjxcsUWDl9bsAuDcifmsvO0CfvypE5k9Npf3ttUA8JnTCoMMT0QSVKTbFIY653YC+M+d/UAXAGXdliv3y+LWHxZspSC7P/dfMxMz44qZhTx2w2wuOXE4pxRmc2LBoKBDFJEEFC1dZ/d0nsT1uKDZPGAeQGFhbP6aLqtqZHFpNd+8cOIB/RqZGb/+bEI0p4hIlIr0kcLuztNC/hCGuAgAAAr/SURBVHOFX14OdO8KdCSwo6cNOOfudc7NcM7NyM/PD2uw4fLEkjLM4OPTRgQdiojIASKdFJ4FrvGnrwGe6VZ+tX8V0mygtvM0U7zZ39bOg++U8qHifEblql8jEYkuYTt9ZGaPAecAeWZWDnwfuAN4wsyuB7YBl/uLvwBcDJQAjcB14YoraH9fvZu65jauPWN00KGIiBwibEnBOXdVL7PO72FZB3wlXLFEi+bWdu6cv4EJQwdy9gSNtSwi0Ud3NEdIa3sHNzy0hC179vHdS6aoTyMRiUpKChHywFtbeHPjHr5z8WTOmhCbDeQiEv+UFCKgpa2D+97czFkT8rnhrLFBhyMi0islhQh4ee1u9jS0cN0ZRUGHIiJyWEoKEfDowm0UZPfXaSMRiXpKCmFWumcfb5Xs4YqZo9S4LCJRT0khzJ5etp0kg0/PGHXkhUVEAqakcJSWbatmcenBnb/27s2NlZxYMIhhg/qFMSoRkb6hpHAU6ptb+cTd73D5b97lP59bQ0dHj332dVmzo45l22q4ZFpc9wIuInFESeEovLNpLwDDB/Xjd29t4auPL2N/W3uPy3Z0OG5/bjUD0pJ16khEYoaSwlEoqWgAYP5NZ/Ptiyfx/IqdXPPAImoaWw5Z9vHFZSzYXMV/fGwK2QPSIh2qiMgxUVI4Cgu3VDFiUD8Gpqcw76xx3HXFySzdWs1H7nyDl1bv6lqurb2DX726kVNH53DFTB0liEjsiJZBdqKac46b//Q+b2yo5N8+PKGr/NJTChg/ZCD//uQK5j28lLMm5HP5qSNZXlbDjtpmbp97gsZZFpGYoqQQgtU76nj6ve1kpCXzudMP7PL6hIJBPHvjHO57czMPvbOVrz62DIDzJw3h/MnqCVVEYouSQgj++v4OUpONt289r8f2gdTkJL58zni+eNY4FpdW0dDcxpnFeTpKEJGYo6RwBGVVjTy6cBuzxuQescE4OcmYPXZwhCITEel7amg+DOccV967gIaWNr5xwcSgwxERCTslhcNYvaOO7TVNXD17NKcU5gQdjohI2CkpHMZdL28EYN7Z4wKOREQkMpQUetHR4Vi4ZS/TC7MpyO4fdDgiIhGhpNCLDRX11De3ceXMwqBDERGJGCWFXlx9/yIAZo7JDTgSEZHIUVLoQUlFAxX1+5k4NJMxeRlBhyMiEjFKCj34zp9XkmTwwHUzgw5FRCSilBQO8vC7pSzcUsWN5xWrgVlEEo6SQjd/XLyN7z2zGoB/nq0GZhFJPEoKvtqmVm55aiUAb91yLkMyNXymiCSehOz7aPWOWjZV7uPj04bT0t7BHS+u4/dvlwLw80+fxMicAcEGKCISkIRMCn9Ztp373tzCv/rdXHe69owi5p5cEFBUIiLBS8ikcO2cMdz35hYAZozOoXhoJrfPnUpqss6miUhiS8ikUJDdn2Xf+wjbqho5aVR20OGIiESNhEwKADkZaeRkHH58BBGRRBNV50vM7EIzW29mJWZ2a9DxiIgkmqhJCmaWDPwauAiYAlxlZlOCjUpEJLFETVIAZgElzrnNzrkW4HFgbsAxiYgklGhKCgVAWbfX5X6ZiIhESDQlBeuhzB2ykNk8M1tiZksqKysjEJaISOKIpqRQDozq9noksOPghZxz9zrnZjjnZuTn50csOBGRRBBNSWExUGxmY8wsDbgSeDbgmEREEoo5d8gZmsCY2cXAXUAy8IBz7r+OsHwlsPUYd5cH7DnGdWOV6pwYVOfEcDx1Hu2c6/FUS1QlhUgysyXOuRlBxxFJqnNiUJ0TQ7jqHE2nj0REJGBKCiIi0iWRk8K9QQcQANU5MajOiSEsdU7YNgURETlUIh8piIjIQZQURESkS0ImhXjuotvMSs1spZktN7Mlflmumc03s43+c45fbmb2S//vsMLMpgcbfWjM7AEzqzCzVd3KjrqOZnaNv/xGM7smiLqEopf63mZm2/33ebl/j0/nvG/59V1vZh/tVh4zn3szG2Vmr5nZWjNbbWZf88vj+X3urc6Rfa+dcwn1wLsxbhMwFkgD3gemBB1XH9avFMg7qOwnwK3+9K3Aj/3pi4EX8fqdmg0sDDr+EOt4FjAdWHWsdQRygc3+c44/nRN03Y6ivrcB3+hh2Sn+ZzodGON/1pNj7XMPDAem+9OZwAa/bvH8PvdW54i+14l4pJCIXXTPBR70px8ELu1W/pDzLACyzWx4EAEeDefcG0DVQcVHW8ePAvOdc1XOuWpgPnBh+KM/er3Utzdzgcedc/udc1uAErzPfEx97p1zO51z7/nT9cBavF6T4/l97q3OvQnLe52ISSHeu+h2wEtmttTM5vllQ51zO8H74AFD/PJ4+lscbR3joe43+qdKHug8jUIc1tfMioBTgIUkyPt8UJ0hgu91IiaFkLrojmFznHPT8Uaw+4qZnXWYZeP9bwG91zHW634PMA44GdgJ/Mwvj6v6mtlA4Cng6865usMt2kNZTNa7hzpH9L1OxKQQUhfdsco5t8N/rgD+jHcoubvztJD/XOEvHk9/i6OtY0zX3Tm32znX7pzrAO7De58hjuprZql4X46POOee9ovj+n3uqc6Rfq8TMSnEbRfdZpZhZpmd08AFwCq8+nVedXEN8Iw//SxwtX/lxmygtvPQPAYdbR3/DlxgZjn+4fgFfllMOKjt5xN47zN49b3SzNLNbAxQDCwixj73ZmbA/cBa59zPu82K2/e5tzpH/L0OusU9iAfelQob8FrovxN0PH1Yr7F4Vxq8D6zurBswGHgF2Og/5/rlBvza/zusBGYEXYcQ6/kY3mF0K96vouuPpY7A5/Ea50qA64Ku11HW92G/Piv8f/jh3Zb/jl/f9cBF3cpj5nMPnIl3ymMFsNx/XBzn73NvdY7oe61uLkREpEsinj4SEZFeKCmIiEgXJQUREemipCAiIl2UFEQiyMzOMbMzgo5DpDdKCiKRdQ6gpCBRS0lBEoqZFfldE9/nd0/8kpn1N7PXzWyGv0yemZX609ea2V/M7K9mtsXMbjSzm8xsmZktMLPcw+zrX81sjd9nzeN+fzZfAv7N7wL5Q2aWb2ZPmdli/zHHX/c2M3vYzF71u3y+wS8fbmZv+OuvMrMPhflPJgkmJegARAJQDFzlnLvBzJ4APnWE5U/A65ysH94NULc4504xszuBq4G7elnvVmCMc26/mWU752rM7DdAg3PupwBm9ihwp3PuLTMrxLvbdrK//jS8bqAzgGVm9jxwFfB359x/mVkyMODY/gQiPVNSkES0xTm33J9eChQdYfnXnNeVcb2Z1QJ/9ctX4n1x92YF8IiZ/QX4Sy/LfBiY4vVwAEBWZ1clwDPOuSagycxew+vzZjHwgN9Hzl+61UOkT+j0kSSi/d2m2/F+HLXxwf9Dv8Ms39HtdQeH/2F1CV7XC6cCS82sp2WTgNOdcyf7jwI/AcGhPVs6542tcBawHXjYzK4+zP5FjpqSgoinFO/LG+Cy492YmSUBo5xzrwHfBLKBgUA93qhanV4Cbuy23snd5s01s35mNhivgXqxmY0GKpxz9+F1nhYTQ6hK7FBSEPH8FPgXM3sHyOuD7SUDfzCzlcAyvHaDGrxTT5/obGgG/hWY4TdGr8FriO60CHgeWAD80Hndop8DLDezZXhtIb/og1hFuqhDPJEoZGa30a1BWiRSdKQgIiJddKQgcpzM7NfAnIOKf+Gc+30Q8YgcDyUFERHpotNHIiLSRUlBRES6KCmIiEgXJQUREemipCAiIl2UFEREpMv/A49p9vf4MVASAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "x2=range(0,len(score_history))\n",
    "#plt.plot(x2,score_history)\n",
    "plt.plot(x2,rolling_average(score_history, window_size=200))\n",
    "plt.title('score vs len(len)')\n",
    "plt.xlabel('num_steps')\n",
    "plt.ylabel('returns')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}