long_horizon_baselines.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "541cf383",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import time\n",
    "import argparse\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from hyperopt import fmin, tpe, hp, Trials, STATUS_OK\n",
    "\n",
    "from statsforecast import StatsForecast\n",
    "from statsforecast.models import (\n",
    "    naive,\n",
    "    seasonal_naive, \n",
    "    auto_arima, \n",
    "    auto_arima_f, \n",
    "    forecast_arima\n",
    ")\n",
    "\n",
    "from src.losses.numpy import mae, mse\n",
    "from src.experiments.utils import hyperopt_tunning\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "101fadec",
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(args):\n",
    "\n",
    "    #----------------------------------------------- Load Data -----------------------------------------------#\n",
    "    Y_df = pd.read_csv(f'./data/{args.dataset}/M/df_y.csv')\n",
    "\n",
    "    X_df = None\n",
    "    S_df = None\n",
    "\n",
    "    print('Y_df: ', Y_df.head())\n",
    "    if args.dataset == 'ETTm2':\n",
    "        len_val = 11520\n",
    "        len_test = 11520\n",
    "    if args.dataset == 'Exchange':\n",
    "        len_val = 760\n",
    "        len_test = 1517\n",
    "    if args.dataset == 'ECL':\n",
    "        len_val = 2632\n",
    "        len_test = 5260\n",
    "    if args.dataset == 'traffic':\n",
    "        len_val = 1756\n",
    "        len_test = 3508\n",
    "    if args.dataset == 'weather':\n",
    "        len_val = 5270\n",
    "        len_test = 10539\n",
    "    if args.dataset == 'ili':\n",
    "        len_val = 97\n",
    "        len_test = 193\n",
    "\n",
    "    #---------------------------------------------- Directories ----------------------------------------------#\n",
    "    output_dir = f'./results/multivariate/{args.dataset}_{args.horizon}/NHITS/'\n",
    "\n",
    "    os.makedirs(output_dir, exist_ok = True)\n",
    "    assert os.path.exists(output_dir), f'Output dir {output_dir} does not exist'\n",
    "    \n",
    "    hyperopt_file = output_dir + f'hyperopt_{args.experiment_id}.p'    \n",
    "    \n",
    "    n_series = len(Y_df.unique_id.unique())\n",
    "    Y_df['ds'] = pd.to_datetime(Y_df['ds'])\n",
    "    \n",
    "    # Test set predictions\n",
    "    model = StatsForecast(Y_df[['unique_id', 'ds', 'y']].set_index('unique_id'),\n",
    "                          models=[(seasonal_naive, args.seasonality)],\n",
    "                          freq=args.freq, n_jobs=-1)\n",
    "    forecasts = model.cross_validation(h=args.horizon, \n",
    "                                       n_windows=None, step_size=1,\n",
    "                                       test_size=len_test)\n",
    "    forecasts['unique_id'] = forecasts.index\n",
    "    forecasts = forecasts.reset_index(drop=True)\n",
    "    \n",
    "    y_true = forecasts['y'].values\n",
    "    y_hat  = forecasts[f'seasonal_naive_season_length-{args.seasonality}'].values\n",
    "    \n",
    "    y_true = y_true.reshape(n_series, -1, args.horizon)\n",
    "    y_hat  = y_hat.reshape(n_series, -1, args.horizon)\n",
    "    \n",
    "    #return forecasts, y_true, y_hat\n",
    "    losses= {'dataset': args.dataset,\n",
    "             'horizon': args.horizon,\n",
    "             'mse': mse(y=y_true, y_hat=y_hat),\n",
    "             'mae': mae(y=y_true, y_hat=y_hat)}\n",
    "    \n",
    "    #with open(hyperopt_file, \"wb\") as f:\n",
    "    #    pickle.dump(trials, f)    \n",
    "    \n",
    "    return losses\n",
    "    \n",
    "    #pickle.dump(trials, f)\n",
    "\n",
    "# source ~/anaconda3/etc/profile.d/conda.sh\n",
    "# conda activate nixtla\n",
    "# CUDA_VISIBLE_DEVICES=0 python nhits_multivariate.py --hyperopt_max_evals 10 --experiment_id \"eval_train\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "16180e29",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------- ETTm2 --------------------------------------------------\n",
      "-------------------------------------------------- 96 --------------------------------------------------\n",
      "Y_df:                      ds unique_id         y\n",
      "0  2016-07-01 00:00:00      HUFL -0.041413\n",
      "1  2016-07-01 00:00:00      HULL  0.040104\n",
      "2  2016-07-01 00:00:00      MUFL -0.599211\n",
      "3  2016-07-01 00:00:00      MULL -0.393536\n",
      "4  2016-07-01 00:00:00      LUFL  0.695804\n",
      "Time:  8.74078893661499\n",
      "-------------------------------------------------- ETTm2 --------------------------------------------------\n",
      "-------------------------------------------------- 192 --------------------------------------------------\n",
      "Y_df:                      ds unique_id         y\n",
      "0  2016-07-01 00:00:00      HUFL -0.041413\n",
      "1  2016-07-01 00:00:00      HULL  0.040104\n",
      "2  2016-07-01 00:00:00      MUFL -0.599211\n",
      "3  2016-07-01 00:00:00      MULL -0.393536\n",
      "4  2016-07-01 00:00:00      LUFL  0.695804\n",
      "Time:  12.760643243789673\n",
      "-------------------------------------------------- ETTm2 --------------------------------------------------\n",
      "-------------------------------------------------- 336 --------------------------------------------------\n",
      "Y_df:                      ds unique_id         y\n",
      "0  2016-07-01 00:00:00      HUFL -0.041413\n",
      "1  2016-07-01 00:00:00      HULL  0.040104\n",
      "2  2016-07-01 00:00:00      MUFL -0.599211\n",
      "3  2016-07-01 00:00:00      MULL -0.393536\n",
      "4  2016-07-01 00:00:00      LUFL  0.695804\n",
      "Time:  19.327486991882324\n",
      "-------------------------------------------------- ETTm2 --------------------------------------------------\n",
      "-------------------------------------------------- 720 --------------------------------------------------\n",
      "Y_df:                      ds unique_id         y\n",
      "0  2016-07-01 00:00:00      HUFL -0.041413\n",
      "1  2016-07-01 00:00:00      HULL  0.040104\n",
      "2  2016-07-01 00:00:00      MUFL -0.599211\n",
      "3  2016-07-01 00:00:00      MULL -0.393536\n",
      "4  2016-07-01 00:00:00      LUFL  0.695804\n",
      "Time:  43.292787075042725\n",
      "-------------------------------------------------- Exchange --------------------------------------------------\n",
      "-------------------------------------------------- 96 --------------------------------------------------\n",
      "Y_df:             ds unique_id         y\n",
      "0  1990-01-01         0  0.606785\n",
      "1  1990-01-01         1 -0.361671\n",
      "2  1990-01-01         2  0.735367\n",
      "3  1990-01-01         3 -1.164373\n",
      "4  1990-01-01         4  2.851890\n",
      "Time:  4.06845498085022\n",
      "-------------------------------------------------- Exchange --------------------------------------------------\n",
      "-------------------------------------------------- 192 --------------------------------------------------\n",
      "Y_df:             ds unique_id         y\n",
      "0  1990-01-01         0  0.606785\n",
      "1  1990-01-01         1 -0.361671\n",
      "2  1990-01-01         2  0.735367\n",
      "3  1990-01-01         3 -1.164373\n",
      "4  1990-01-01         4  2.851890\n",
      "Time:  3.395883083343506\n",
      "-------------------------------------------------- Exchange --------------------------------------------------\n",
      "-------------------------------------------------- 336 --------------------------------------------------\n",
      "Y_df:             ds unique_id         y\n",
      "0  1990-01-01         0  0.606785\n",
      "1  1990-01-01         1 -0.361671\n",
      "2  1990-01-01         2  0.735367\n",
      "3  1990-01-01         3 -1.164373\n",
      "4  1990-01-01         4  2.851890\n",
      "Time:  4.004759073257446\n",
      "-------------------------------------------------- Exchange --------------------------------------------------\n",
      "-------------------------------------------------- 720 --------------------------------------------------\n",
      "Y_df:             ds unique_id         y\n",
      "0  1990-01-01         0  0.606785\n",
      "1  1990-01-01         1 -0.361671\n",
      "2  1990-01-01         2  0.735367\n",
      "3  1990-01-01         3 -1.164373\n",
      "4  1990-01-01         4  2.851890\n",
      "Time:  5.576997995376587\n",
      "-------------------------------------------------- weather --------------------------------------------------\n",
      "-------------------------------------------------- 96 --------------------------------------------------\n",
      "Y_df:                      ds    unique_id         y\n",
      "0  2020-01-01 00:10:00     p (mbar)  2.114257\n",
      "1  2020-01-01 00:10:00     T (degC) -1.459980\n",
      "2  2020-01-01 00:10:00     Tpot (K) -1.607935\n",
      "3  2020-01-01 00:10:00  Tdew (degC) -1.052596\n",
      "4  2020-01-01 00:10:00       rh (%)  0.990128\n",
      "Time:  11.242947816848755\n",
      "-------------------------------------------------- weather --------------------------------------------------\n",
      "-------------------------------------------------- 192 --------------------------------------------------\n",
      "Y_df:                      ds    unique_id         y\n",
      "0  2020-01-01 00:10:00     p (mbar)  2.114257\n",
      "1  2020-01-01 00:10:00     T (degC) -1.459980\n",
      "2  2020-01-01 00:10:00     Tpot (K) -1.607935\n",
      "3  2020-01-01 00:10:00  Tdew (degC) -1.052596\n",
      "4  2020-01-01 00:10:00       rh (%)  0.990128\n",
      "Time:  18.620702266693115\n",
      "-------------------------------------------------- weather --------------------------------------------------\n",
      "-------------------------------------------------- 336 --------------------------------------------------\n",
      "Y_df:                      ds    unique_id         y\n",
      "0  2020-01-01 00:10:00     p (mbar)  2.114257\n",
      "1  2020-01-01 00:10:00     T (degC) -1.459980\n",
      "2  2020-01-01 00:10:00     Tpot (K) -1.607935\n",
      "3  2020-01-01 00:10:00  Tdew (degC) -1.052596\n",
      "4  2020-01-01 00:10:00       rh (%)  0.990128\n",
      "Time:  32.16367793083191\n",
      "-------------------------------------------------- weather --------------------------------------------------\n",
      "-------------------------------------------------- 720 --------------------------------------------------\n",
      "Y_df:                      ds    unique_id         y\n",
      "0  2020-01-01 00:10:00     p (mbar)  2.114257\n",
      "1  2020-01-01 00:10:00     T (degC) -1.459980\n",
      "2  2020-01-01 00:10:00     Tpot (K) -1.607935\n",
      "3  2020-01-01 00:10:00  Tdew (degC) -1.052596\n",
      "4  2020-01-01 00:10:00       rh (%)  0.990128\n",
      "Time:  78.94659304618835\n",
      "-------------------------------------------------- ili --------------------------------------------------\n",
      "-------------------------------------------------- 24 --------------------------------------------------\n",
      "Y_df:             ds        unique_id         y\n",
      "0  2002-01-01   % WEIGHTED ILI -0.421499\n",
      "1  2002-01-01  %UNWEIGHTED ILI -0.472442\n",
      "2  2002-01-01          AGE 0-4 -0.981641\n",
      "3  2002-01-01         AGE 5-24 -0.692621\n",
      "4  2002-01-01         ILITOTAL -0.819695\n",
      "Time:  2.3775720596313477\n",
      "-------------------------------------------------- ili --------------------------------------------------\n",
      "-------------------------------------------------- 36 --------------------------------------------------\n",
      "Y_df:             ds        unique_id         y\n",
      "0  2002-01-01   % WEIGHTED ILI -0.421499\n",
      "1  2002-01-01  %UNWEIGHTED ILI -0.472442\n",
      "2  2002-01-01          AGE 0-4 -0.981641\n",
      "3  2002-01-01         AGE 5-24 -0.692621\n",
      "4  2002-01-01         ILITOTAL -0.819695\n",
      "Time:  1.6622469425201416\n",
      "-------------------------------------------------- ili --------------------------------------------------\n",
      "-------------------------------------------------- 48 --------------------------------------------------\n",
      "Y_df:             ds        unique_id         y\n",
      "0  2002-01-01   % WEIGHTED ILI -0.421499\n",
      "1  2002-01-01  %UNWEIGHTED ILI -0.472442\n",
      "2  2002-01-01          AGE 0-4 -0.981641\n",
      "3  2002-01-01         AGE 5-24 -0.692621\n",
      "4  2002-01-01         ILITOTAL -0.819695\n",
      "Time:  1.6632211208343506\n",
      "-------------------------------------------------- ili --------------------------------------------------\n",
      "-------------------------------------------------- 60 --------------------------------------------------\n",
      "Y_df:             ds        unique_id         y\n",
      "0  2002-01-01   % WEIGHTED ILI -0.421499\n",
      "1  2002-01-01  %UNWEIGHTED ILI -0.472442\n",
      "2  2002-01-01          AGE 0-4 -0.981641\n",
      "3  2002-01-01         AGE 5-24 -0.692621\n",
      "4  2002-01-01         ILITOTAL -0.819695\n",
      "Time:  1.7644822597503662\n",
      "-------------------------------------------------- ECL --------------------------------------------------\n",
      "-------------------------------------------------- 96 --------------------------------------------------\n",
      "Y_df:                      ds unique_id         y\n",
      "0  2016-07-01 02:00:00         0 -0.449788\n",
      "1  2016-07-01 02:00:00         1 -1.748271\n",
      "2  2016-07-01 02:00:00         2  3.655108\n",
      "3  2016-07-01 02:00:00         3 -0.063269\n",
      "4  2016-07-01 02:00:00         4  0.214958\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time:  65.18087124824524\n",
      "-------------------------------------------------- ECL --------------------------------------------------\n",
      "-------------------------------------------------- 192 --------------------------------------------------\n",
      "Y_df:                      ds unique_id         y\n",
      "0  2016-07-01 02:00:00         0 -0.449788\n",
      "1  2016-07-01 02:00:00         1 -1.748271\n",
      "2  2016-07-01 02:00:00         2  3.655108\n",
      "3  2016-07-01 02:00:00         3 -0.063269\n",
      "4  2016-07-01 02:00:00         4  0.214958\n",
      "Time:  192.2269058227539\n",
      "-------------------------------------------------- ECL --------------------------------------------------\n",
      "-------------------------------------------------- 336 --------------------------------------------------\n",
      "Y_df:                      ds unique_id         y\n",
      "0  2016-07-01 02:00:00         0 -0.449788\n",
      "1  2016-07-01 02:00:00         1 -1.748271\n",
      "2  2016-07-01 02:00:00         2  3.655108\n",
      "3  2016-07-01 02:00:00         3 -0.063269\n",
      "4  2016-07-01 02:00:00         4  0.214958\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-07 18:12:54 root ERROR: Internal Python error in the inspect module.\n",
      "Below is the traceback from this internal error.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/opt/anaconda3/envs/neuralforecast/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3552, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_74450/386760276.py\", line 31, in <module>\n",
      "    losses = main(args)\n",
      "KeyboardInterrupt\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/anaconda3/envs/neuralforecast/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2098, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'KeyboardInterrupt' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/anaconda3/envs/neuralforecast/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 1101, in get_records\n",
      "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
      "  File \"/opt/anaconda3/envs/neuralforecast/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 248, in wrapped\n",
      "    return f(*args, **kwargs)\n",
      "  File \"/opt/anaconda3/envs/neuralforecast/lib/python3.7/site-packages/IPython/core/ultratb.py\", line 281, in _fixed_getinnerframes\n",
      "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
      "  File \"/opt/anaconda3/envs/neuralforecast/lib/python3.7/inspect.py\", line 1502, in getinnerframes\n",
      "    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n",
      "  File \"/opt/anaconda3/envs/neuralforecast/lib/python3.7/inspect.py\", line 1460, in getframeinfo\n",
      "    filename = getsourcefile(frame) or getfile(frame)\n",
      "  File \"/opt/anaconda3/envs/neuralforecast/lib/python3.7/inspect.py\", line 696, in getsourcefile\n",
      "    if getattr(getmodule(object, filename), '__loader__', None) is not None:\n",
      "  File \"/opt/anaconda3/envs/neuralforecast/lib/python3.7/inspect.py\", line 742, in getmodule\n",
      "    os.path.realpath(f)] = module.__name__\n",
      "  File \"/opt/anaconda3/envs/neuralforecast/lib/python3.7/posixpath.py\", line 395, in realpath\n",
      "    path, ok = _joinrealpath(filename[:0], filename, {})\n",
      "  File \"/opt/anaconda3/envs/neuralforecast/lib/python3.7/posixpath.py\", line 429, in _joinrealpath\n",
      "    if not islink(newpath):\n",
      "  File \"/opt/anaconda3/envs/neuralforecast/lib/python3.7/posixpath.py\", line 171, in islink\n",
      "    st = os.lstat(path)\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "object of type 'NoneType' has no len()",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "    \u001b[0;31m[... skipping hidden 1 frame]\u001b[0m\n",
      "\u001b[0;32m/var/folders/fy/h20z78md68jgtxrgcl3h1y4r0000gn/T/ipykernel_74450/386760276.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     30\u001b[0m             \u001b[0margs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfreq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfreq\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m             \u001b[0mlosses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m             \u001b[0mlosses_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlosses\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m/opt/anaconda3/envs/neuralforecast/lib/python3.7/site-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mshowtraceback\u001b[0;34m(self, exc_tuple, filename, tb_offset, exception_only, running_compiled_code)\u001b[0m\n\u001b[1;32m   2097\u001b[0m                         \u001b[0;31m# in the engines. This should return a list of strings.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2098\u001b[0;31m                         \u001b[0mstb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_render_traceback_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2099\u001b[0m                     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'KeyboardInterrupt' object has no attribute '_render_traceback_'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "    \u001b[0;31m[... skipping hidden 1 frame]\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/envs/neuralforecast/lib/python3.7/site-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mshowtraceback\u001b[0;34m(self, exc_tuple, filename, tb_offset, exception_only, running_compiled_code)\u001b[0m\n\u001b[1;32m   2099\u001b[0m                     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2100\u001b[0m                         stb = self.InteractiveTB.structured_traceback(etype,\n\u001b[0;32m-> 2101\u001b[0;31m                                             value, tb, tb_offset=tb_offset)\n\u001b[0m\u001b[1;32m   2102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2103\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_showtraceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0metype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/envs/neuralforecast/lib/python3.7/site-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mstructured_traceback\u001b[0;34m(self, etype, value, tb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1366\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1367\u001b[0m         return FormattedTB.structured_traceback(\n\u001b[0;32m-> 1368\u001b[0;31m             self, etype, value, tb, tb_offset, number_of_lines_of_context)\n\u001b[0m\u001b[1;32m   1369\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1370\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/envs/neuralforecast/lib/python3.7/site-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mstructured_traceback\u001b[0;34m(self, etype, value, tb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1266\u001b[0m             \u001b[0;31m# Verbose modes need a full traceback\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1267\u001b[0m             return VerboseTB.structured_traceback(\n\u001b[0;32m-> 1268\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0metype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtb_offset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnumber_of_lines_of_context\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1269\u001b[0m             )\n\u001b[1;32m   1270\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'Minimal'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/envs/neuralforecast/lib/python3.7/site-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mstructured_traceback\u001b[0;34m(self, etype, evalue, etb, tb_offset, number_of_lines_of_context)\u001b[0m\n\u001b[1;32m   1123\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1124\u001b[0m         formatted_exception = self.format_exception_as_a_whole(etype, evalue, etb, number_of_lines_of_context,\n\u001b[0;32m-> 1125\u001b[0;31m                                                                tb_offset)\n\u001b[0m\u001b[1;32m   1126\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1127\u001b[0m         \u001b[0mcolors\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mColors\u001b[0m  \u001b[0;31m# just a shorthand + quicker name lookup\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/envs/neuralforecast/lib/python3.7/site-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mformat_exception_as_a_whole\u001b[0;34m(self, etype, evalue, etb, number_of_lines_of_context, tb_offset)\u001b[0m\n\u001b[1;32m   1080\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1081\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1082\u001b[0;31m         \u001b[0mlast_unique\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecursion_repeat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfind_recursion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0morig_etype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecords\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1083\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1084\u001b[0m         \u001b[0mframes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat_records\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecords\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlast_unique\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecursion_repeat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/envs/neuralforecast/lib/python3.7/site-packages/IPython/core/ultratb.py\u001b[0m in \u001b[0;36mfind_recursion\u001b[0;34m(etype, value, records)\u001b[0m\n\u001b[1;32m    380\u001b[0m     \u001b[0;31m# first frame (from in to out) that looks different.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    381\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_recursion_error\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0metype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrecords\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 382\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrecords\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    383\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    384\u001b[0m     \u001b[0;31m# Select filename, lineno, func_name to track frames with\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: object of type 'NoneType' has no len()"
     ]
    }
   ],
   "source": [
    "args=pd.Series(dict(experiment_id='seasonal_naive'))\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    \n",
    "    horizons = [96, 192, 336, 720]\n",
    "    ILI_horizons = [24, 36, 48, 60]\n",
    "    datasets = ['ETTm2', 'Exchange', 'weather', 'ili', 'ECL', 'traffic']\n",
    "    seasonalities = [96, 7, 144, 52, 96, 24]\n",
    "    freqs = ['15min', 'D', '10min', 'W', '15min', 'H']\n",
    "    \n",
    "    #datasets = ['ili']\n",
    "    #seasonalities = [52]\n",
    "    #freqs = ['W']\n",
    "    \n",
    "    losses_list = []\n",
    "    for dataset, seasonality, freq in\\\n",
    "            zip(datasets, seasonalities, freqs):\n",
    "        # Horizon\n",
    "        if dataset == 'ili':\n",
    "            horizons_dataset = ILI_horizons\n",
    "        else:\n",
    "            horizons_dataset = horizons\n",
    "        for horizon in horizons_dataset:\n",
    "            print(50*'-', dataset, 50*'-')\n",
    "            print(50*'-', horizon, 50*'-')\n",
    "            start = time.time()\n",
    "            args.dataset = dataset\n",
    "            args.horizon = horizon\n",
    "            args.seasonality = seasonality\n",
    "            args.freq = freq\n",
    "            losses = main(args)\n",
    "            losses_list.append(losses)\n",
    "            print('Time: ', time.time() - start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a2a6588e",
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df = pd.DataFrame(losses_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f496d7de",
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ade7934c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46163796",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "neuralforecast",
   "language": "python",
   "name": "neuralforecast"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}