4-Optimization/FinRL_HyperparameterTuning_using_Optuna_basic.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JwrAr8vEmP2M"
   },
   "source": [
    "# Introduction\n",
    "1. In this tutorial, we will be tuning hyperparameters for Stable baselines3 models using Optuna.\n",
    "2. The default model hyperparamters may not be adequate for your custom portfolio or custom state-space. Reinforcement learning algorithms are sensitive to hyperparamters, hence tuning is an important step.\n",
    "3. Hyperparamters are tuned based on an objective, which needs to be maximized or minimized. Here we tuned our hyperparamters to maximize the Sharpe Ratio "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qRF-7Vp5NCjU"
   },
   "outputs": [],
   "source": [
    "#Installing FinRL\n",
    "%%capture\n",
    "!pip install wrds\n",
    "!pip install swig\n",
    "!pip install git+https://github.com/AI4Finance-Foundation/FinRL.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LRJk36AZQGuh"
   },
   "outputs": [],
   "source": [
    "#Installing Optuna\n",
    "%%capture\n",
    "!pip3 install optuna"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "PqOKn-VWNGt4",
    "outputId": "32e0f056-37ad-4af4-f0de-71bcce6e5878"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/pyfolio/pos.py:27: UserWarning:\n",
      "\n",
      "Module \"zipline.assets\" not found; multipliers will not be applied to position notionals.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Importing the libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "# matplotlib.use('Agg')\n",
    "import datetime\n",
    "import optuna\n",
    "%matplotlib inline\n",
    "from finrl import config\n",
    "from finrl import config_tickers\n",
    "from optuna.integration import PyTorchLightningPruningCallback\n",
    "from finrl.meta.preprocessor.yahoodownloader import YahooDownloader\n",
    "from finrl.meta.preprocessor.preprocessors import FeatureEngineer, data_split\n",
    "from finrl.meta.env_stock_trading.env_stocktrading import StockTradingEnv\n",
    "from finrl.meta.env_stock_trading.env_stocktrading_np import StockTradingEnv as StockTradingEnv_numpy\n",
    "from finrl.agents.stablebaselines3.models import DRLAgent\n",
    "from finrl.agents.rllib.models import DRLAgent as DRLAgent_rllib\n",
    "from finrl.meta.data_processor import DataProcessor\n",
    "import joblib\n",
    "from finrl.plot import backtest_stats, backtest_plot, get_daily_return, get_baseline\n",
    "import ray\n",
    "from pprint import pprint\n",
    "\n",
    "import sys\n",
    "sys.path.append(\"../FinRL-Library\")\n",
    "\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "z7_fCHS6NMx9"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "if not os.path.exists(\"./\" + config.DATA_SAVE_DIR):\n",
    "    os.makedirs(\"./\" + config.DATA_SAVE_DIR)\n",
    "if not os.path.exists(\"./\" + config.TRAINED_MODEL_DIR):\n",
    "    os.makedirs(\"./\" + config.TRAINED_MODEL_DIR)\n",
    "if not os.path.exists(\"./\" + config.TENSORBOARD_LOG_DIR):\n",
    "    os.makedirs(\"./\" + config.TENSORBOARD_LOG_DIR)\n",
    "if not os.path.exists(\"./\" + config.RESULTS_DIR):\n",
    "    os.makedirs(\"./\" + config.RESULTS_DIR)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "71P6jMlEpikl"
   },
   "source": [
    "## Collecting data and preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "frT5V9lLOv9X",
    "outputId": "9d05910d-aa9c-4273-ecd3-33e4098eabbb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "Shape of DataFrame:  (96270, 8)\n"
     ]
    }
   ],
   "source": [
    "#Custom ticker list dataframe download\n",
    "ticker_list = config_tickers.DOW_30_TICKER\n",
    "df = YahooDownloader(start_date = '2009-01-01',\n",
    "                     end_date = '2021-10-01',\n",
    "                     ticker_list = ticker_list).fetch_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cntKg5nWO5qn",
    "outputId": "93a5b7cb-fc19-446b-da92-a43f070f6049"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully added technical indicators\n",
      "[*********************100%***********************]  1 of 1 completed\n",
      "Shape of DataFrame:  (3208, 8)\n",
      "Successfully added vix\n",
      "Successfully added turbulence index\n"
     ]
    }
   ],
   "source": [
    "#You can add technical indicators and turbulence factor to dataframe\n",
    "#Just set the use_technical_indicator=True, use_vix=True and use_turbulence=True\n",
    "fe = FeatureEngineer(\n",
    "                    use_technical_indicator=True,\n",
    "                    tech_indicator_list = config.INDICATORS,\n",
    "                    use_vix=True,\n",
    "                    use_turbulence=True,\n",
    "                    user_defined_feature = False)\n",
    "\n",
    "processed = fe.preprocess_data(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5diXih4zPE6m"
   },
   "outputs": [],
   "source": [
    "list_ticker = processed[\"tic\"].unique().tolist()\n",
    "list_date = list(pd.date_range(processed['date'].min(),processed['date'].max()).astype(str))\n",
    "combination = list(itertools.product(list_date,list_ticker))\n",
    "\n",
    "processed_full = pd.DataFrame(combination,columns=[\"date\",\"tic\"]).merge(processed,on=[\"date\",\"tic\"],how=\"left\")\n",
    "processed_full = processed_full[processed_full['date'].isin(processed['date'])]\n",
    "processed_full = processed_full.sort_values(['date','tic'])\n",
    "\n",
    "processed_full = processed_full.fillna(0)\n",
    "processed_full.sort_values(['date','tic'],ignore_index=True).head(5)\n",
    "\n",
    "processed_full.to_csv('processed_full.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3RrJiFbSPKE2",
    "outputId": "5ca01b3e-8e7d-4e78-8d3f-289b8f3f8674"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "86790\n",
      "10710\n"
     ]
    }
   ],
   "source": [
    "train = data_split(processed_full, '2009-01-01','2020-07-01')\n",
    "trade = data_split(processed_full, '2020-05-01','2021-10-01')\n",
    "print(len(train))\n",
    "print(len(trade))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ub4JTTRcPOel",
    "outputId": "432716f9-82ed-463b-c4ac-a0901fe5a0ac"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Stock Dimension: 30, State Space: 181\n"
     ]
    }
   ],
   "source": [
    "stock_dimension = len(train.tic.unique())\n",
    "state_space = 1 + 2*stock_dimension + len(config.INDICATORS) * stock_dimension\n",
    "print(f\"Stock Dimension: {stock_dimension}, State Space: {state_space}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YiF95zXgPTsd"
   },
   "outputs": [],
   "source": [
    "#Defining the environment kwargs\n",
    "\n",
    "env_kwargs = {\n",
    "    \"hmax\": 100, \n",
    "    \"initial_amount\": 1000000, \n",
    "    \"buy_cost_pct\": 0.001,\n",
    "    \"sell_cost_pct\": 0.001,\n",
    "    \"state_space\": state_space, \n",
    "    \"stock_dim\": stock_dimension, \n",
    "    \"tech_indicator_list\": config.INDICATORS, \n",
    "    \"action_space\": stock_dimension, \n",
    "    \"reward_scaling\": 1e-4\n",
    "    \n",
    "}\n",
    "#Instantiate the training gym compatible environment\n",
    "e_train_gym = StockTradingEnv(df = train, **env_kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "892NcZALPWHF",
    "outputId": "88d8958a-a9aa-4fbf-9338-c990f7829a1b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'stable_baselines3.common.vec_env.dummy_vec_env.DummyVecEnv'>\n"
     ]
    }
   ],
   "source": [
    "#Instantiate the training environment\n",
    "# Also instantiate our training gent\n",
    "env_train, _ = e_train_gym.get_sb_env()\n",
    "print(type(env_train))\n",
    "agent = DRLAgent(env = env_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3EwB1T8opX2o"
   },
   "outputs": [],
   "source": [
    "#Instantiate the trading environment\n",
    "e_trade_gym = StockTradingEnv(df = trade, turbulence_threshold = None, **env_kwargs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zOynTQluppye"
   },
   "source": [
    "## Tuninf hyperparameters using Optuna\n",
    "1. Go to this [link](https://github.com/DLR-RM/rl-baselines3-zoo/blob/master/utils/hyperparams_opt.py), you will find all possible hyperparamters to tune for all the models.\n",
    "2. For your model, grab those hyperparamters which you want to optimize and then return a dictionary of hyperparamters.\n",
    "3. There is a feature in Optuna called as hyperparamters importance, you can point out those hyperparamters which are important for tuning.\n",
    "4. By default Optuna use [TPESampler](https://www.youtube.com/watch?v=tdwgR1AqQ8Y) for sampling hyperparamters from the search space. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_vojRvAsP2ja"
   },
   "outputs": [],
   "source": [
    "def sample_ddpg_params(trial:optuna.Trial):\n",
    "  # Size of the replay buffer\n",
    "  buffer_size = trial.suggest_categorical(\"buffer_size\", [int(1e4), int(1e5), int(1e6)])\n",
    "  learning_rate = trial.suggest_loguniform(\"learning_rate\", 1e-5, 1)\n",
    "  batch_size = trial.suggest_categorical(\"batch_size\", [32, 64, 128, 256, 512])\n",
    "  \n",
    "  return {\"buffer_size\": buffer_size,\n",
    "          \"learning_rate\":learning_rate,\n",
    "          \"batch_size\":batch_size}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xL7LeLeWrj6H"
   },
   "outputs": [],
   "source": [
    "#Calculate the Sharpe ratio\n",
    "#This is our objective for tuning\n",
    "def calculate_sharpe(df):\n",
    "  df['daily_return'] = df['account_value'].pct_change(1)\n",
    "  if df['daily_return'].std() !=0:\n",
    "    sharpe = (252**0.5)*df['daily_return'].mean()/ \\\n",
    "          df['daily_return'].std()\n",
    "    return sharpe\n",
    "  else:\n",
    "    return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gCRy_kL648DM"
   },
   "source": [
    "## Callbacks\n",
    "1. The callback will terminate if the improvement margin is below certain point\n",
    "2. It will terminate after certain number of trial_number are reached, not before that\n",
    "3. It will hold its patience to reach the threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MTPSF3bpHDUT"
   },
   "outputs": [],
   "source": [
    "class LoggingCallback:\n",
    "    def __init__(self,threshold,trial_number,patience):\n",
    "      '''\n",
    "      threshold:int tolerance for increase in sharpe ratio\n",
    "      trial_number: int Prune after minimum number of trials\n",
    "      patience: int patience for the threshold\n",
    "      '''\n",
    "      self.threshold = threshold\n",
    "      self.trial_number  = trial_number\n",
    "      self.patience = patience\n",
    "      self.cb_list = [] #Trials list for which threshold is reached\n",
    "    def __call__(self,study:optuna.study, frozen_trial:optuna.Trial):\n",
    "      #Setting the best value in the current trial\n",
    "      study.set_user_attr(\"previous_best_value\", study.best_value)\n",
    "      \n",
    "      #Checking if the minimum number of trials have pass\n",
    "      if frozen_trial.number >self.trial_number:\n",
    "          previous_best_value = study.user_attrs.get(\"previous_best_value\",None)\n",
    "          #Checking if the previous and current objective values have the same sign\n",
    "          if previous_best_value * study.best_value >=0:\n",
    "              #Checking for the threshold condition\n",
    "              if abs(previous_best_value-study.best_value) < self.threshold: \n",
    "                  self.cb_list.append(frozen_trial.number)\n",
    "                  #If threshold is achieved for the patience amount of time\n",
    "                  if len(self.cb_list)>self.patience:\n",
    "                      print('The study stops now...')\n",
    "                      print('With number',frozen_trial.number ,'and value ',frozen_trial.value)\n",
    "                      print('The previous and current best values are {} and {} respectively'\n",
    "                              .format(previous_best_value, study.best_value))\n",
    "                      study.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "import sys   \n",
    "\n",
    "os.makedirs(\"models\",exist_ok=True)\n",
    "\n",
    "def objective(trial:optuna.Trial):\n",
    "  #Trial will suggest a set of hyperparamters from the specified range\n",
    "  hyperparameters = sample_ddpg_params(trial)\n",
    "  model_ddpg = agent.get_model(\"ddpg\",model_kwargs = hyperparameters )\n",
    "  #You can increase it for better comparison\n",
    "  trained_ddpg = agent.train_model(model=model_ddpg,\n",
    "                                  tb_log_name=\"ddpg\" ,\n",
    "                             total_timesteps=50000)\n",
    "  trained_ddpg.save('models/ddpg_{}.pth'.format(trial.number))\n",
    "  clear_output(wait=True)\n",
    "  #For the given hyperparamters, determine the account value in the trading period\n",
    "  df_account_value, df_actions = DRLAgent.DRL_prediction(\n",
    "    model=trained_ddpg, \n",
    "    environment = e_trade_gym)\n",
    "  #Calculate sharpe from the account value\n",
    "  sharpe = calculate_sharpe(df_account_value)\n",
    "\n",
    "  return sharpe\n",
    "\n",
    "#Create a study object and specify the direction as 'maximize'\n",
    "#As you want to maximize sharpe\n",
    "#Pruner stops not promising iterations\n",
    "#Use a pruner, else you will get error related to divergence of model\n",
    "#You can also use Multivariate samplere\n",
    "#sampler = optuna.samplers.TPESampler(multivarite=True,seed=42)\n",
    "sampler = optuna.samplers.TPESampler(seed=42)\n",
    "study = optuna.create_study(study_name=\"ddpg_study\",direction='maximize',\n",
    "                            sampler = sampler, pruner=optuna.pruners.HyperbandPruner())\n",
    "\n",
    "logging_callback = LoggingCallback(threshold=1e-5,patience=30,trial_number=5)\n",
    "#You can increase the n_trials for a better search space scanning\n",
    "study.optimize(objective, n_trials=30,catch=(ValueError,),callbacks=[logging_callback])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "sg7LRlHmj9GB",
    "outputId": "b089f7d6-7e92-44f1-b535-5ed127105366"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['final_ddpg_study__.pkl']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.dump(study, \"final_ddpg_study__.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "SVMue1-xuGHC",
    "outputId": "115d9f90-081f-4ef0-e687-2d06eea505fd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hyperparameters after tuning {'buffer_size': 100000, 'learning_rate': 9.548041810464153e-05, 'batch_size': 512}\n",
      "Hyperparameters before tuning {'batch_size': 128, 'buffer_size': 50000, 'learning_rate': 0.001}\n"
     ]
    }
   ],
   "source": [
    "#Get the best hyperparamters\n",
    "print('Hyperparameters after tuning',study.best_params)\n",
    "print('Hyperparameters before tuning',config.DDPG_PARAMS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tsZmMw0ykmYo",
    "outputId": "0da397f4-b412-49f5-aa93-34cdc4bcbf8d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FrozenTrial(number=6, values=[1.9680115561891418], datetime_start=datetime.datetime(2021, 10, 16, 16, 1, 20, 183649), datetime_complete=datetime.datetime(2021, 10, 16, 16, 47, 48, 263917), params={'buffer_size': 100000, 'learning_rate': 9.548041810464153e-05, 'batch_size': 512}, distributions={'buffer_size': CategoricalDistribution(choices=(10000, 100000, 1000000)), 'learning_rate': LogUniformDistribution(high=1.0, low=1e-05), 'batch_size': CategoricalDistribution(choices=(32, 64, 128, 256, 512))}, user_attrs={}, system_attrs={}, intermediate_values={}, trial_id=6, state=TrialState.COMPLETE, value=None)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "study.best_trial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "fETqKJj4uSi5"
   },
   "outputs": [],
   "source": [
    "from stable_baselines3 import DDPG\n",
    "tuned_model_ddpg = DDPG.load('models/ddpg_{}.pth'.format(study.best_trial.number),env=env_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "FgchX1LLuua-",
    "outputId": "00b50778-ae4a-49f8-c676-2d5286d899c0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hit end!\n"
     ]
    }
   ],
   "source": [
    "#Trading period account value with tuned model\n",
    "df_account_value_tuned, df_actions_tuned = DRLAgent.DRL_prediction(\n",
    "    model=tuned_model_ddpg, \n",
    "    environment = e_trade_gym)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "8s5KNvVuvr2D",
    "outputId": "6a4e6933-84ca-4689-b8c3-042fc77e181c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==============Get Backtest Results===========\n",
      "Annual return          0.311337\n",
      "Cumulative returns     0.468121\n",
      "Annual volatility      0.143427\n",
      "Sharpe ratio           1.968012\n",
      "Calmar ratio           3.371128\n",
      "Stability              0.952728\n",
      "Max drawdown          -0.092354\n",
      "Omega ratio            1.404845\n",
      "Sortino ratio          2.816227\n",
      "Skew                        NaN\n",
      "Kurtosis                    NaN\n",
      "Tail ratio             1.161312\n",
      "Daily value at risk   -0.016950\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "#Backtesting with our pruned model\n",
    "print(\"==============Get Backtest Results===========\")\n",
    "now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')\n",
    "\n",
    "perf_stats_all_tuned = backtest_stats(account_value=df_account_value_tuned)\n",
    "perf_stats_all_tuned = pd.DataFrame(perf_stats_all_tuned)\n",
    "perf_stats_all_tuned.to_csv(\"./\"+config.RESULTS_DIR+\"/perf_stats_all_tuned_\"+now+'.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "tuGaI9lSvvVD",
    "outputId": "7e72920e-84e5-486f-e7ce-747d282783d2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'batch_size': 128, 'buffer_size': 50000, 'learning_rate': 0.001}\n",
      "Using cuda device\n",
      "Logging to tensorboard_log/ddpg/ddpg_15\n",
      "---------------------------------\n",
      "| time/              |          |\n",
      "|    episodes        | 4        |\n",
      "|    fps             | 55       |\n",
      "|    time_elapsed    | 209      |\n",
      "|    total timesteps | 11572    |\n",
      "| train/             |          |\n",
      "|    actor_loss      | 84       |\n",
      "|    critic_loss     | 1.89e+04 |\n",
      "|    learning_rate   | 0.001    |\n",
      "|    n_updates       | 8679     |\n",
      "|    reward          | 4.206319 |\n",
      "---------------------------------\n",
      "day: 2892, episode: 260\n",
      "begin_total_asset: 1000000.00\n",
      "end_total_asset: 3610867.99\n",
      "total_reward: 2610867.99\n",
      "total_cost: 999.00\n",
      "total_trades: 54928\n",
      "Sharpe: 0.685\n",
      "=================================\n",
      "---------------------------------\n",
      "| time/              |          |\n",
      "|    episodes        | 8        |\n",
      "|    fps             | 50       |\n",
      "|    time_elapsed    | 455      |\n",
      "|    total timesteps | 23144    |\n",
      "| train/             |          |\n",
      "|    actor_loss      | -29.6    |\n",
      "|    critic_loss     | 158      |\n",
      "|    learning_rate   | 0.001    |\n",
      "|    n_updates       | 20251    |\n",
      "|    reward          | 4.206319 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| time/              |          |\n",
      "|    episodes        | 12       |\n",
      "|    fps             | 49       |\n",
      "|    time_elapsed    | 702      |\n",
      "|    total timesteps | 34716    |\n",
      "| train/             |          |\n",
      "|    actor_loss      | -23.7    |\n",
      "|    critic_loss     | 37.4     |\n",
      "|    learning_rate   | 0.001    |\n",
      "|    n_updates       | 31823    |\n",
      "|    reward          | 4.206319 |\n",
      "---------------------------------\n",
      "---------------------------------\n",
      "| time/              |          |\n",
      "|    episodes        | 16       |\n",
      "|    fps             | 48       |\n",
      "|    time_elapsed    | 946      |\n",
      "|    total timesteps | 46288    |\n",
      "| train/             |          |\n",
      "|    actor_loss      | -18      |\n",
      "|    critic_loss     | 16.9     |\n",
      "|    learning_rate   | 0.001    |\n",
      "|    n_updates       | 43395    |\n",
      "|    reward          | 4.206319 |\n",
      "---------------------------------\n",
      "day: 2892, episode: 270\n",
      "begin_total_asset: 1000000.00\n",
      "end_total_asset: 3610867.99\n",
      "total_reward: 2610867.99\n",
      "total_cost: 999.00\n",
      "total_trades: 54928\n",
      "Sharpe: 0.685\n",
      "=================================\n"
     ]
    }
   ],
   "source": [
    "#Now train with not tuned hyperaparameters\n",
    "#Default config.ddpg_PARAMS\n",
    "non_tuned_model_ddpg = agent.get_model(\"ddpg\",model_kwargs = config.DDPG_PARAMS )\n",
    "trained_ddpg = agent.train_model(model=non_tuned_model_ddpg, \n",
    "                             tb_log_name='ddpg',\n",
    "                             total_timesteps=50000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yZbEYRQ1wBeC",
    "outputId": "c2aacb1f-9fc1-4816-c15a-da512df2b2df"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hit end!\n"
     ]
    }
   ],
   "source": [
    "df_account_value, df_actions = DRLAgent.DRL_prediction(\n",
    "    model=trained_ddpg, \n",
    "    environment = e_trade_gym)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pFdB4YM3wh0m",
    "outputId": "e31ca771-bf0f-4f5c-d385-839dfa215450"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==============Get Backtest Results===========\n",
      "Annual return          0.325768\n",
      "Cumulative returns     0.491062\n",
      "Annual volatility      0.169124\n",
      "Sharpe ratio           1.757594\n",
      "Calmar ratio           3.238950\n",
      "Stability              0.944550\n",
      "Max drawdown          -0.100578\n",
      "Omega ratio            1.355538\n",
      "Sortino ratio          2.590766\n",
      "Skew                        NaN\n",
      "Kurtosis                    NaN\n",
      "Tail ratio             1.171057\n",
      "Daily value at risk   -0.020128\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "#Backtesting for not tuned hyperparamters\n",
    "print(\"==============Get Backtest Results===========\")\n",
    "now = datetime.datetime.now().strftime('%Y%m%d-%Hh%M')\n",
    "\n",
    "perf_stats_all = backtest_stats(account_value=df_account_value)\n",
    "perf_stats_all = pd.DataFrame(perf_stats_all)\n",
    "# perf_stats_all.to_csv(\"./\"+config.RESULTS_DIR+\"/perf_stats_all_\"+now+'.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 542
    },
    "id": "12fxdvUZwi_W",
    "outputId": "56a25b27-dd6c-4620-acf1-76e68f252f32"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"08ae1a48-3f9f-46a6-9cf3-3756c1dd2622\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"08ae1a48-3f9f-46a6-9cf3-3756c1dd2622\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '08ae1a48-3f9f-46a6-9cf3-3756c1dd2622',\n",
       "                        [{\"mode\": \"markers\", \"name\": \"Objective Value\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], \"y\": [1.9336748564881099, 1.7103825303995113, 1.451824873143654, 1.399998769842667, 1.610091338541197, 1.9282760290455876, 1.9680115561891418, 1.7307168460129427, 1.8178755898363874, 1.4459502841509786, 1.6852943468510908, 1.8105525374061526, 1.9463535284637115]}, {\"name\": \"Best Value\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], \"y\": [1.9336748564881099, 1.9336748564881099, 1.9336748564881099, 1.9336748564881099, 1.9336748564881099, 1.9336748564881099, 1.9680115561891418, 1.9680115561891418, 1.9680115561891418, 1.9680115561891418, 1.9680115561891418, 1.9680115561891418, 1.9680115561891418]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Optimization History Plot\"}, \"xaxis\": {\"title\": {\"text\": \"#Trials\"}}, \"yaxis\": {\"title\": {\"text\": \"Objective Value\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('08ae1a48-3f9f-46a6-9cf3-3756c1dd2622');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#You can see with trial, our sharpe ratio is increasing\n",
    "#Certainly you can afford more number of trials for further optimization\n",
    "from optuna.visualization import plot_optimization_history\n",
    "plot_optimization_history(study)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_TUF2GvAx6-k"
   },
   "outputs": [],
   "source": [
    "from optuna.visualization import plot_contour\n",
    "from optuna.visualization import plot_edf\n",
    "from optuna.visualization import plot_intermediate_values\n",
    "from optuna.visualization import plot_optimization_history\n",
    "from optuna.visualization import plot_parallel_coordinate\n",
    "from optuna.visualization import plot_param_importances\n",
    "from optuna.visualization import plot_slice"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 542
    },
    "id": "2jkqeSUIyCT0",
    "outputId": "531590bb-e10c-4a96-b1d7-af57ff166030"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"b368844c-12b4-48a3-8214-3653f299217d\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"b368844c-12b4-48a3-8214-3653f299217d\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        'b368844c-12b4-48a3-8214-3653f299217d',\n",
       "                        [{\"cliponaxis\": false, \"hovertemplate\": [\"learning_rate (LogUniformDistribution): 0.039013664889828874<extra></extra>\", \"buffer_size (CategoricalDistribution): 0.18575466361275744<extra></extra>\", \"batch_size (CategoricalDistribution): 0.7752316714974136<extra></extra>\"], \"marker\": {\"color\": \"rgb(66,146,198)\"}, \"orientation\": \"h\", \"text\": [\"0.039013664889828874\", \"0.18575466361275744\", \"0.7752316714974136\"], \"textposition\": \"outside\", \"texttemplate\": \"%{text:.2f}\", \"type\": \"bar\", \"x\": [0.039013664889828874, 0.18575466361275744, 0.7752316714974136], \"y\": [\"learning_rate\", \"buffer_size\", \"batch_size\"]}],\n",
       "                        {\"showlegend\": false, \"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Hyperparameter Importances\"}, \"xaxis\": {\"title\": {\"text\": \"Importance for Objective Value\"}}, \"yaxis\": {\"title\": {\"text\": \"Hyperparameter\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('b368844c-12b4-48a3-8214-3653f299217d');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Hyperparamters importance\n",
    "#Ent_coef is the most important\n",
    "plot_param_importances(study)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fAD0MIAWukB9"
   },
   "source": [
    "## Further works\n",
    "\n",
    "1.   You can tune more critical hyperparameters\n",
    "2.   Multi-objective hyperparameter optimization using Optuna. Here we can maximize Sharpe and simultaneously minimize Volatility in our account value to tune our hyperparameters\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 542
    },
    "id": "_edBJqB8yEDr",
    "outputId": "2bf34851-7038-4341-b3cb-de7a85608c78"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<html>\n",
       "<head><meta charset=\"utf-8\" /></head>\n",
       "<body>\n",
       "    <div>\n",
       "            <script src=\"https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/MathJax.js?config=TeX-AMS-MML_SVG\"></script><script type=\"text/javascript\">if (window.MathJax) {MathJax.Hub.Config({SVG: {font: \"STIX-Web\"}});}</script>\n",
       "                <script type=\"text/javascript\">window.PlotlyConfig = {MathJaxConfig: 'local'};</script>\n",
       "        <script src=\"https://cdn.plot.ly/plotly-latest.min.js\"></script>    \n",
       "            <div id=\"21d54349-a5ac-4f06-adb0-e17b25a08512\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>\n",
       "            <script type=\"text/javascript\">\n",
       "                \n",
       "                    window.PLOTLYENV=window.PLOTLYENV || {};\n",
       "                    \n",
       "                if (document.getElementById(\"21d54349-a5ac-4f06-adb0-e17b25a08512\")) {\n",
       "                    Plotly.newPlot(\n",
       "                        '21d54349-a5ac-4f06-adb0-e17b25a08512',\n",
       "                        [{\"mode\": \"lines\", \"name\": \"ddpg_study\", \"type\": \"scatter\", \"x\": [1.399998769842667, 1.4057362727350557, 1.4114737756274442, 1.417211278519833, 1.4229487814122215, 1.4286862843046102, 1.434423787196999, 1.4401612900893874, 1.4458987929817761, 1.4516362958741647, 1.4573737987665534, 1.463111301658942, 1.4688488045513306, 1.4745863074437193, 1.4803238103361078, 1.4860613132284965, 1.4917988161208853, 1.4975363190132738, 1.5032738219056625, 1.509011324798051, 1.5147488276904397, 1.5204863305828284, 1.526223833475217, 1.5319613363676057, 1.5376988392599942, 1.543436342152383, 1.5491738450447716, 1.5549113479371601, 1.5606488508295489, 1.5663863537219374, 1.572123856614326, 1.5778613595067146, 1.5835988623991033, 1.589336365291492, 1.5950738681838805, 1.6008113710762693, 1.606548873968658, 1.6122863768610465, 1.6180238797534352, 1.6237613826458237, 1.6294988855382124, 1.635236388430601, 1.6409738913229897, 1.6467113942153784, 1.6524488971077669, 1.6581864000001556, 1.6639239028925443, 1.6696614057849328, 1.6753989086773216, 1.68113641156971, 1.6868739144620988, 1.6926114173544873, 1.698348920246876, 1.7040864231392647, 1.7098239260316532, 1.715561428924042, 1.7212989318164307, 1.7270364347088192, 1.732773937601208, 1.7385114404935964, 1.7442489433859851, 1.7499864462783736, 1.7557239491707624, 1.761461452063151, 1.7671989549555396, 1.7729364578479283, 1.778673960740317, 1.7844114636327055, 1.7901489665250943, 1.7958864694174828, 1.8016239723098715, 1.80736147520226, 1.8130989780946487, 1.8188364809870374, 1.824573983879426, 1.8303114867718147, 1.8360489896642034, 1.8417864925565919, 1.8475239954489804, 1.853261498341369, 1.8589990012337578, 1.8647365041261463, 1.870474007018535, 1.8762115099109238, 1.8819490128033123, 1.887686515695701, 1.8934240185880897, 1.8991615214804782, 1.9048990243728667, 1.9106365272652555, 1.9163740301576442, 1.9221115330500327, 1.9278490359424214, 1.9335865388348101, 1.9393240417271986, 1.9450615446195874, 1.950799047511976, 1.9565365504043646, 1.962274053296753, 1.9680115561891418], \"y\": [0.07692307692307693, 0.07692307692307693, 0.07692307692307693, 0.07692307692307693, 0.07692307692307693, 0.07692307692307693, 0.07692307692307693, 0.07692307692307693, 0.07692307692307693, 0.15384615384615385, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.23076923076923078, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.3076923076923077, 0.38461538461538464, 0.38461538461538464, 0.38461538461538464, 0.38461538461538464, 0.38461538461538464, 0.46153846153846156, 0.46153846153846156, 0.46153846153846156, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.5384615384615384, 0.6153846153846154, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.6923076923076923, 0.7692307692307693, 0.8461538461538461, 0.8461538461538461, 0.9230769230769231, 0.9230769230769231, 0.9230769230769231, 1.0]}],\n",
       "                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Empirical Distribution Function Plot\"}, \"xaxis\": {\"title\": {\"text\": \"Objective Value\"}}, \"yaxis\": {\"range\": [0, 1], \"title\": {\"text\": \"Cumulative Probability\"}}},\n",
       "                        {\"responsive\": true}\n",
       "                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('21d54349-a5ac-4f06-adb0-e17b25a08512');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })\n",
       "                };\n",
       "                \n",
       "            </script>\n",
       "        </div>\n",
       "</body>\n",
       "</html>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_edf(study)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pXTgG1yvgeyq"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "FinRL_Hyperparameter tuning using Optuna",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}