non_stationary_dgp_rl_sim.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "# import matplotlib.pyplot as plt\n",
    "\n",
    "import statsmodels.api as sm\n",
    "from statsmodels.tsa.stattools import adfuller\n",
    "\n",
    "import gymnasium as gym\n",
    "import gym_trading_env\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from RLTrade.agent import DQNAgent, XGBoostAgent\n",
    "from RLTrade.utils import ( \n",
    "    nonstationaryDGP, \n",
    "    FeatureEngineering\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = nonstationaryDGP()\n",
    "spread = y - x\n",
    "df = pd.DataFrame({'x': x, 'y': y, 'close': spread}) # close price of portfolio is the spread\n",
    "df_train = df.copy()\n",
    "\n",
    "feature_config = {\n",
    "    \"window\": 7, # 7 days window, pad with first value when window is not available\n",
    "    \"mode\": \"diff\", # difference between current and previous close price\n",
    "    \"subtract_mean\": True,\n",
    "}\n",
    "\n",
    "fe = FeatureEngineering(df_train, main_feature_col=\"close\")\n",
    "fe.rolling_feature(**feature_config)\n",
    "df_train = fe.df.copy()\n",
    "df_train\n",
    "\n",
    "env_action_space = [-1, 0, 1] # Positions : [-1=SHORT, 0=OUT, 1=LONG]\n",
    "# env_action_space = [i/10 for i in range(-10, 11)] # Positions : [-1, -0.9, -0.8, ..., 0, 0.1, 0.2, ..., 1]\n",
    "\n",
    "env = gym.make(\"TradingEnv\",\n",
    "        name= \"stationaryDGP\",\n",
    "        df = df_train, # Your dataset with your custom features\n",
    "        positions = env_action_space, # -1 (=SHORT), 0(=OUT), +1 (=LONG)\n",
    "        trading_fees = 0.01/100, # 0.01% per stock buy / sell (Binance fees)\n",
    "        borrow_interest_rate= 0.0003/100, # 0.0003% per timestep (one timestep = 1h here)\n",
    "        # max_episode_duration=180\n",
    "    )\n",
    "env.unwrapped.add_metric('Position Changes', lambda history : np.sum(np.diff(history['position']) != 0) )\n",
    "env.unwrapped.add_metric('Episode Length', lambda history : len(history['position']) )\n",
    "\n",
    "n_actions = env.action_space.n\n",
    "n_observations = len(env.reset()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Market Return : 20.21%   |   Portfolio Return : -46.42%   |   Position Changes : 5722   |   Episode Length : 10000   |   \n",
      "Market Return : 28.72%   |   Portfolio Return : -49.55%   |   Position Changes : 5232   |   Episode Length : 10000   |   \n",
      "Market Return :  2.01%   |   Portfolio Return : -49.98%   |   Position Changes : 4963   |   Episode Length : 10000   |   \n",
      "Market Return : -25.47%   |   Portfolio Return : -44.46%   |   Position Changes : 4823   |   Episode Length : 10000   |   \n",
      "Market Return :  1.79%   |   Portfolio Return : -45.15%   |   Position Changes : 4539   |   Episode Length : 10000   |   \n",
      "Market Return : 13.51%   |   Portfolio Return : -44.17%   |   Position Changes : 4515   |   Episode Length : 10000   |   \n",
      "Market Return : -15.91%   |   Portfolio Return : -43.11%   |   Position Changes : 4511   |   Episode Length : 10000   |   \n",
      "Market Return : 19.79%   |   Portfolio Return : -44.14%   |   Position Changes : 4416   |   Episode Length : 10000   |   \n",
      "Market Return : 12.48%   |   Portfolio Return : -39.36%   |   Position Changes : 4323   |   Episode Length : 10000   |   \n",
      "Market Return :  6.35%   |   Portfolio Return : -40.99%   |   Position Changes : 4151   |   Episode Length : 10000   |   \n",
      "Market Return :  0.88%   |   Portfolio Return : -39.38%   |   Position Changes : 4137   |   Episode Length : 10000   |   \n",
      "Market Return :  8.11%   |   Portfolio Return : -42.25%   |   Position Changes : 3994   |   Episode Length : 10000   |   \n",
      "Market Return : -8.09%   |   Portfolio Return : -36.93%   |   Position Changes : 3973   |   Episode Length : 10000   |   \n",
      "Market Return : -20.15%   |   Portfolio Return : -38.50%   |   Position Changes : 3806   |   Episode Length : 10000   |   \n",
      "Market Return : -0.07%   |   Portfolio Return : -28.87%   |   Position Changes : 3592   |   Episode Length : 10000   |   \n"
     ]
    }
   ],
   "source": [
    "# Train a DQN agent using training data from a stationary data generating process\n",
    "agent = DQNAgent(n_observations=n_observations, n_actions=n_actions)\n",
    "for i_simulation in range(15):\n",
    "    x, y = nonstationaryDGP()\n",
    "    spread = y - x\n",
    "    df = pd.DataFrame({'x': x, 'y': y, 'close': spread}) # close price of portfolio is the spread\n",
    "    df_train = df.copy()\n",
    "    \n",
    "    feature_config = {\n",
    "        \"window\": 7, # 7 days window, pad with first value when window is not available\n",
    "        \"mode\": \"diff\", # difference between current and previous close price\n",
    "        \"subtract_mean\": True,\n",
    "    }\n",
    "\n",
    "    fe = FeatureEngineering(df_train, main_feature_col=\"close\")\n",
    "    fe.rolling_feature(**feature_config)\n",
    "    df_train = fe.df.copy()\n",
    "\n",
    "    # Define the environment\n",
    "    env = gym.make(\"TradingEnv\",\n",
    "            name= \"stationaryDGP\",\n",
    "            df = df_train, # Your dataset with your custom features\n",
    "            positions = env_action_space, # -1 (=SHORT), 0(=OUT), +1 (=LONG)\n",
    "            trading_fees = 0.01/100, # 0.01% per stock buy / sell (Binance fees)\n",
    "            borrow_interest_rate= 0.0003/100, # 0.0003% per timestep (one timestep = 1h here)\n",
    "            # max_episode_duration=180\n",
    "        )\n",
    "    env.unwrapped.add_metric('Position Changes', lambda history : np.sum(np.diff(history['position']) != 0) )\n",
    "    env.unwrapped.add_metric('Episode Length', lambda history : len(history['position']) )\n",
    "\n",
    "    history_metrics, _ = agent.train(env, num_episodes=1) # One episode is one simulation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = nonstationaryDGP()\n",
    "spread = y - x\n",
    "df = pd.DataFrame({'x': x, 'y': y, 'close': spread}) # close price of portfolio is the spread\n",
    "df_train = df.copy()\n",
    "\n",
    "feature_config = {\n",
    "    \"window\": 100, # 7 days window, pad with first value when window is not available\n",
    "    \"mode\": \"diff\", # difference between current and previous close price\n",
    "}\n",
    "\n",
    "fe = FeatureEngineering(df_train, main_feature_col=\"close\")\n",
    "fe.rolling_mean_corrected_feature(**feature_config)\n",
    "df_train = fe.df.copy()\n",
    "\n",
    "env_action_space = [-1, 0, 1] # Positions : [-1=SHORT, 0=OUT, 1=LONG]\n",
    "# env_action_space = [i/10 for i in range(-10, 11)] # Positions : [-1, -0.9, -0.8, ..., 0, 0.1, 0.2, ..., 1]\n",
    "\n",
    "env = gym.make(\"TradingEnv\",\n",
    "        name= \"stationaryDGP\",\n",
    "        df = df_train, # Your dataset with your custom features\n",
    "        positions = env_action_space, # -1 (=SHORT), 0(=OUT), +1 (=LONG)\n",
    "        trading_fees = 0.01/100, # 0.01% per stock buy / sell (Binance fees)\n",
    "        borrow_interest_rate= 0.0003/100, # 0.0003% per timestep (one timestep = 1h here)\n",
    "        # max_episode_duration=180\n",
    "    )\n",
    "env.unwrapped.add_metric('Position Changes', lambda history : np.sum(np.diff(history['position']) != 0) )\n",
    "env.unwrapped.add_metric('Episode Length', lambda history : len(history['position']) )\n",
    "\n",
    "n_actions = env.action_space.n\n",
    "n_observations = len(env.reset()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Market Return : 10.49%   |   Portfolio Return : -51.26%   |   Position Changes : 6549   |   Episode Length : 10000   |   \n",
      "Market Return :  8.11%   |   Portfolio Return : -53.11%   |   Position Changes : 5612   |   Episode Length : 10000   |   \n",
      "Market Return :  7.35%   |   Portfolio Return : -31.04%   |   Position Changes : 3158   |   Episode Length : 10000   |   \n",
      "Market Return : -20.42%   |   Portfolio Return : -25.07%   |   Position Changes : 2097   |   Episode Length : 10000   |   \n",
      "Market Return : -19.97%   |   Portfolio Return : -19.56%   |   Position Changes : 1301   |   Episode Length : 10000   |   \n",
      "Market Return :  4.61%   |   Portfolio Return : -6.64%   |   Position Changes : 628   |   Episode Length : 10000   |   \n",
      "Market Return : 21.86%   |   Portfolio Return : -3.04%   |   Position Changes : 293   |   Episode Length : 10000   |   \n",
      "Market Return : -11.13%   |   Portfolio Return : -0.65%   |   Position Changes : 183   |   Episode Length : 10000   |   \n",
      "Market Return :  0.29%   |   Portfolio Return : -1.50%   |   Position Changes : 155   |   Episode Length : 10000   |   \n",
      "Market Return :  9.03%   |   Portfolio Return : -4.67%   |   Position Changes : 110   |   Episode Length : 10000   |   \n",
      "Market Return : -25.08%   |   Portfolio Return : -0.50%   |   Position Changes : 190   |   Episode Length : 10000   |   \n",
      "Market Return : -3.18%   |   Portfolio Return : -1.65%   |   Position Changes : 103   |   Episode Length : 10000   |   \n",
      "Market Return : 11.76%   |   Portfolio Return : -2.56%   |   Position Changes : 88   |   Episode Length : 10000   |   \n",
      "Market Return : 13.31%   |   Portfolio Return : -0.73%   |   Position Changes : 92   |   Episode Length : 10000   |   \n",
      "Market Return : -24.58%   |   Portfolio Return : -2.92%   |   Position Changes : 101   |   Episode Length : 10000   |   \n"
     ]
    }
   ],
   "source": [
    "# Train a DQN agent using training data from a stationary data generating process\n",
    "agent = XGBoostAgent(n_observations, n_actions)\n",
    "for i_simulation in range(15):\n",
    "    x, y = nonstationaryDGP()\n",
    "    spread = y - x\n",
    "    df = pd.DataFrame({'x': x, 'y': y, 'close': spread}) # close price of portfolio is the spread\n",
    "    df_train = df.copy()\n",
    "\n",
    "    feature_config = {\n",
    "        \"window\": 100, # 7 days window, pad with first value when window is not available\n",
    "        \"mode\": \"diff\", # difference between current and previous close price\n",
    "    }\n",
    "\n",
    "    fe = FeatureEngineering(df_train, main_feature_col=\"close\")\n",
    "    fe.rolling_mean_corrected_feature(**feature_config)\n",
    "    df_train = fe.df.copy()\n",
    "\n",
    "    # Define the environment\n",
    "    env = gym.make(\"TradingEnv\",\n",
    "            name= \"stationaryDGP\",\n",
    "            df = df_train, # Your dataset with your custom features\n",
    "            positions = env_action_space, # -1 (=SHORT), 0(=OUT), +1 (=LONG)\n",
    "            trading_fees = 0.01/100, # 0.01% per stock buy / sell (Binance fees)\n",
    "            borrow_interest_rate= 0.0003/100, # 0.0003% per timestep (one timestep = 1h here)\n",
    "            # max_episode_duration=180\n",
    "        )\n",
    "    env.unwrapped.add_metric('Position Changes', lambda history : np.sum(np.diff(history['position']) != 0) )\n",
    "    env.unwrapped.add_metric('Episode Length', lambda history : len(history['position']) )\n",
    "\n",
    "    history_metrics, _ = agent.train(env, num_episodes=1, replay_every=1) # One episode is one simulation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env_full_py3_9",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}