From 1f564535b3baeba26e04160496cd388f0ff794ca Mon Sep 17 00:00:00 2001 From: Antoine CARME Date: Wed, 8 Mar 2023 21:15:53 +0100 Subject: [PATCH] Investigate Large Horizon Models #213. Improved perf logging. Display the first and last multistep forecast perfs. Updated these perfs --- .../sampling/test_ozone_with_no_sampling.log | 122 +++++++++++------- .../sampling/test_ozone_with_sampling.log | 122 +++++++++++------- .../sampling/test_ozone_with_sampling_2.log | 118 ++++++++++------- .../sampling/test_ozone_with_sampling_3.log | 120 ++++++++++------- 4 files changed, 285 insertions(+), 197 deletions(-) diff --git a/tests/references/sampling/test_ozone_with_no_sampling.log b/tests/references/sampling/test_ozone_with_no_sampling.log index d7235c1e2..14497ed52 100644 --- a/tests/references/sampling/test_ozone_with_no_sampling.log +++ b/tests/references/sampling/test_ozone_with_no_sampling.log @@ -6,24 +6,24 @@ INFO:pyaf.timing:('OPERATION_START', ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 2 1955-03 3.6 1955-03-01 3 1955-04 5.0 1955-04-01 4 1955-05 6.5 1955-05-01 -INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'})) INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'})) INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 14.091, ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 17.589, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 17.593, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 21.61, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 21.881, ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', 'None', '_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', 'CumSum_', 'T+S+R')], 'Cores': 4})) +INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 22.203, ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 23.188, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 24.375, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 28.892, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 29.177, ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', 'None', '_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', 'CumSum_', 'T+S+R')], 'Cores': 4})) INFO:pyaf.timing:('OPERATION_START', ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1})) INFO:pyaf.timing:('OPERATION_START', ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.711, ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.52, ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]})) INFO:pyaf.timing:('OPERATION_START', ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 2.779, ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 1.88, ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR'})) INFO:pyaf.timing:('OPERATION_START', ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 2.704, ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 6.195, ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 28.081, ('TRAINING', {'Signals': ['Ozone'], 'Horizons': {'Ozone': 12}})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 1.724, ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 4.124, ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 33.306, ('TRAINING', {'Signals': ['Ozone'], 'Horizons': {'Ozone': 12}})) INFO:pyaf.std:TIME_DETAIL TimeVariable='Time' TimeMin=1955-01-01T00:00:00.000000 TimeMax=1967-09-01T00:00:00.000000 TimeDelta= Horizon=12 INFO:pyaf.std:SIGNAL_DETAIL_ORIG SignalVariable='Ozone' Length=204 Min=1.2 Max=8.7 Mean=3.835784 StdDev=1.491559 INFO:pyaf.std:SIGNAL_DETAIL_TRANSFORMED TransformedSignalVariable='_Ozone' Min=0.0 Max=1.0 Mean=0.351438 StdDev=0.198875 @@ -33,19 +33,12 @@ INFO:pyaf.std:BEST_DECOMPOSITION '_Ozone_LinearTrend_residue_Seasonal_MonthOfYe INFO:pyaf.std:TREND_DETAIL '_Ozone_LinearTrend' [LinearTrend] INFO:pyaf.std:CYCLE_DETAIL '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear' [Seasonal_MonthOfYear] INFO:pyaf.std:AUTOREG_DETAIL '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR' [NoAR] -INFO:pyaf.std:MODEL_MAPE MAPE_Fit=0.1761 MAPE_Forecast=0.1765 MAPE_Test=0.2209 -INFO:pyaf.std:MODEL_SMAPE SMAPE_Fit=0.1712 SMAPE_Forecast=0.1941 SMAPE_Test=0.2249 -INFO:pyaf.std:MODEL_DiffSMAPE DiffSMAPE_Fit=0.1688 DiffSMAPE_Forecast=0.1903 DiffSMAPE_Test=0.2196 -INFO:pyaf.std:MODEL_MASE MASE_Fit=0.7728 MASE_Forecast=0.7151 MASE_Test=1.0918 -INFO:pyaf.std:MODEL_CRPS CRPS_Fit=0.3308 CRPS_Forecast=0.2855 CRPS_Test=0.3409 -INFO:pyaf.std:MODEL_L1 L1_Fit=0.6792 L1_Forecast=0.5552 L1_Test=0.5161 -INFO:pyaf.std:MODEL_L2 L2_Fit=0.9118 L2_Forecast=0.663 L2_Test=0.5962 -INFO:pyaf.std:MODEL_LnQ LnQ_Fit=7.4081 LnQ_Forecast=2.1757 LnQ_Test=0.9116 -INFO:pyaf.std:MODEL_MEDIAN_AE MedAE_Fit=0.5329 MedAE_Forecast=0.5576 MedAE_Test=0.5692 -INFO:pyaf.std:MODEL_KENDALL_TAU KENDALL_TAU_Fit=0.6293 KENDALL_TAU_Forecast=0.7354 KENDALL_TAU_Test=0.6565 -INFO:pyaf.std:MODEL_KOLMOGOROV_SMIRNOV KS_Fit=0.0915 KS_Forecast=0.2051 KS_Test=0.25 -INFO:pyaf.std:MODEL_MANN_WHITNEY_U MWU_Fit=11828.5 MWU_Forecast=877.0 MWU_Test=68.0 -INFO:pyaf.std:MODEL_AUC AUC_Fit=0.5053 AUC_Forecast=0.5766 AUC_Test=0.4722 +INFO:pyaf.std:MODEL_PERFS Fit STEP=1 {'MAPE': 0.1761, 'RMSE': 0.9118, 'MAE': 0.6792, 'MASE': 0.7728} +INFO:pyaf.std:MODEL_PERFS Forecast STEP=1 {'MAPE': 0.1765, 'RMSE': 0.663, 'MAE': 0.5552, 'MASE': 0.7151} +INFO:pyaf.std:MODEL_PERFS Test STEP=1 {'MAPE': 0.2209, 'RMSE': 0.5962, 'MAE': 0.5161, 'MASE': 1.0918} +INFO:pyaf.std:MODEL_PERFS Fit STEP=12 {'MAPE': 0.1761, 'RMSE': 0.9118, 'MAE': 0.6792, 'MASE': 0.7728} +INFO:pyaf.std:MODEL_PERFS Forecast STEP=12 {'MAPE': 0.1765, 'RMSE': 0.663, 'MAE': 0.5552, 'MASE': 0.7151} +INFO:pyaf.std:MODEL_PERFS Test STEP=12 {'MAPE': 0.2209, 'RMSE': 0.5962, 'MAE': 0.5161, 'MASE': 1.0918} INFO:pyaf.std:MODEL_COMPLEXITY {'Decomposition': 'S', 'Transformation': 'S', 'Trend': 'S', 'Cycle': 'S', 'AR': 'S'} [SSSSS] INFO:pyaf.std:SIGNAL_TRANSFORMATION_DETAIL_START INFO:pyaf.std:SIGNAL_TRANSFORMATION_MODEL_VALUES NoTransf None @@ -58,9 +51,9 @@ INFO:pyaf.std:SEASONAL_MODEL_VALUES _Ozone_LinearTrend_residue_Seasonal_MonthOfY INFO:pyaf.std:CYCLE_MODEL_DETAIL_END INFO:pyaf.std:AR_MODEL_DETAIL_START INFO:pyaf.std:AR_MODEL_DETAIL_END -INFO:pyaf.std:TRAINING_TIME_IN_SECONDS 28.081 +INFO:pyaf.std:TRAINING_TIME_IN_SECONDS 33.306 INFO:pyaf.timing:('OPERATION_START', ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 1.198, ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.999, ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12})) Transformation DecompositionType ... Voting Complexity 1 _Ozone T+S+R ... 612.5000 SSSSS 0 Diff_Ozone T+S+R ... 617.1667 LMSSS @@ -120,14 +113,20 @@ Forecasts { "Ozone": { + "Complexity": { + "AR": "S", + "Cycle": "S", + "Decomposition": "S", + "Transformation": "S", + "Trend": "S" + }, "Dataset": { "Signal": "Ozone", "Time": { "Horizon": 12, - "TimeMinMax": [ - "1955-01-01 00:00:00", - "1971-12-01 00:00:00" - ], + "TimeDelta": "", + "TimeMax": "1971-12-01 00:00:00", + "TimeMin": "1955-01-01 00:00:00", "TimeVariable": "Time" }, "Training_Signal_Length": 204 @@ -141,27 +140,50 @@ Forecasts "Trend": "LinearTrend" }, "Model_Performance": { - "AUC": 0.5766, - "COMPLEXITY": { - "AR": "S", - "Cycle": "S", - "Decomposition": "S", - "Transformation": "S", - "Trend": "S" + "1": { + "AUC": 0.5766, + "DiffSMAPE": 0.1903, + "ErrorMean": -0.3403, + "ErrorStdDev": 0.569, + "KS": 0.2051, + "KendallTau": 0.7354, + "Length": 39, + "LnQ": 2.1757, + "MAE": 0.5552, + "MAPE": 0.1765, + "MASE": 0.7151, + "MannWhitneyU": 877.0, + "MedAE": 0.5576, + "Pearson": 0.858, + "R2": 0.6262, + "RMSE": 0.663, + "SMAPE": 0.1941, + "Signal": "Ozone_Forecast_1" }, - "CRPS": 0.2855, - "KS": 0.2051, - "KendallTau": 0.7354, - "LnQ": 2.1757, - "MAE": 0.5552, - "MAPE": 0.1765, - "MASE": 0.7151, - "MannWhitneyU": 877.0, - "MedAE": 0.5576, - "RMSE": 0.663 - } + "12": { + "AUC": 0.5766, + "DiffSMAPE": 0.1903, + "ErrorMean": -0.3403, + "ErrorStdDev": 0.569, + "KS": 0.2051, + "KendallTau": 0.7354, + "Length": 39, + "LnQ": 2.1757, + "MAE": 0.5552, + "MAPE": 0.1765, + "MASE": 0.7151, + "MannWhitneyU": 877.0, + "MedAE": 0.5576, + "Pearson": 0.858, + "R2": 0.6262, + "RMSE": 0.663, + "SMAPE": 0.1941, + "Signal": "Ozone_Forecast_12" + } + }, + "Model_Selection_Criterion": "MAPE" }, - "Training_Time": 28.081 + "Training_Time": 33.306 } diff --git a/tests/references/sampling/test_ozone_with_sampling.log b/tests/references/sampling/test_ozone_with_sampling.log index a5c662409..0f67452df 100644 --- a/tests/references/sampling/test_ozone_with_sampling.log +++ b/tests/references/sampling/test_ozone_with_sampling.log @@ -6,24 +6,24 @@ INFO:pyaf.timing:('OPERATION_START', ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 2 1955-03 3.6 1955-03-01 3 1955-04 5.0 1955-04-01 4 1955-05 6.5 1955-05-01 -INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'})) INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'})) INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 15.114, ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 15.207, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 15.515, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 22.308, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 22.495, ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', 'None', '_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', 'CumSum_', 'T+S+R')], 'Cores': 4})) +INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 23.564, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 24.103, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 24.425, ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 29.282, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 29.492, ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', 'None', '_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', 'CumSum_', 'T+S+R')], 'Cores': 4})) INFO:pyaf.timing:('OPERATION_START', ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1})) INFO:pyaf.timing:('OPERATION_START', ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.39, ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.403, ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]})) INFO:pyaf.timing:('OPERATION_START', ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 2.601, ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 1.997, ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR'})) INFO:pyaf.timing:('OPERATION_START', ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 2.69, ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 5.681, ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 28.18, ('TRAINING', {'Signals': ['Ozone'], 'Horizons': {'Ozone': 12}})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 1.92, ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 4.321, ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 33.818, ('TRAINING', {'Signals': ['Ozone'], 'Horizons': {'Ozone': 12}})) INFO:pyaf.std:TIME_DETAIL TimeVariable='Time' TimeMin=1955-01-01T00:00:00.000000 TimeMax=1967-09-01T00:00:00.000000 TimeDelta= Horizon=12 INFO:pyaf.std:SIGNAL_DETAIL_ORIG SignalVariable='Ozone' Length=204 Min=1.2 Max=8.7 Mean=3.835784 StdDev=1.491559 INFO:pyaf.std:SIGNAL_DETAIL_TRANSFORMED TransformedSignalVariable='_Ozone' Min=0.0 Max=1.0 Mean=0.351438 StdDev=0.198875 @@ -33,19 +33,12 @@ INFO:pyaf.std:BEST_DECOMPOSITION '_Ozone_LinearTrend_residue_Seasonal_MonthOfYe INFO:pyaf.std:TREND_DETAIL '_Ozone_LinearTrend' [LinearTrend] INFO:pyaf.std:CYCLE_DETAIL '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear' [Seasonal_MonthOfYear] INFO:pyaf.std:AUTOREG_DETAIL '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR' [NoAR] -INFO:pyaf.std:MODEL_MAPE MAPE_Fit=0.1761 MAPE_Forecast=0.1765 MAPE_Test=0.2209 -INFO:pyaf.std:MODEL_SMAPE SMAPE_Fit=0.1712 SMAPE_Forecast=0.1941 SMAPE_Test=0.2249 -INFO:pyaf.std:MODEL_DiffSMAPE DiffSMAPE_Fit=0.1688 DiffSMAPE_Forecast=0.1903 DiffSMAPE_Test=0.2196 -INFO:pyaf.std:MODEL_MASE MASE_Fit=0.7728 MASE_Forecast=0.7151 MASE_Test=1.0918 -INFO:pyaf.std:MODEL_CRPS CRPS_Fit=0.3308 CRPS_Forecast=0.2855 CRPS_Test=0.3409 -INFO:pyaf.std:MODEL_L1 L1_Fit=0.6792 L1_Forecast=0.5552 L1_Test=0.5161 -INFO:pyaf.std:MODEL_L2 L2_Fit=0.9118 L2_Forecast=0.663 L2_Test=0.5962 -INFO:pyaf.std:MODEL_LnQ LnQ_Fit=7.4081 LnQ_Forecast=2.1757 LnQ_Test=0.9116 -INFO:pyaf.std:MODEL_MEDIAN_AE MedAE_Fit=0.5329 MedAE_Forecast=0.5576 MedAE_Test=0.5692 -INFO:pyaf.std:MODEL_KENDALL_TAU KENDALL_TAU_Fit=0.6293 KENDALL_TAU_Forecast=0.7354 KENDALL_TAU_Test=0.6565 -INFO:pyaf.std:MODEL_KOLMOGOROV_SMIRNOV KS_Fit=0.0915 KS_Forecast=0.2051 KS_Test=0.25 -INFO:pyaf.std:MODEL_MANN_WHITNEY_U MWU_Fit=11828.5 MWU_Forecast=877.0 MWU_Test=68.0 -INFO:pyaf.std:MODEL_AUC AUC_Fit=0.5053 AUC_Forecast=0.5766 AUC_Test=0.4722 +INFO:pyaf.std:MODEL_PERFS Fit STEP=1 {'MAPE': 0.1761, 'RMSE': 0.9118, 'MAE': 0.6792, 'MASE': 0.7728} +INFO:pyaf.std:MODEL_PERFS Forecast STEP=1 {'MAPE': 0.1765, 'RMSE': 0.663, 'MAE': 0.5552, 'MASE': 0.7151} +INFO:pyaf.std:MODEL_PERFS Test STEP=1 {'MAPE': 0.2209, 'RMSE': 0.5962, 'MAE': 0.5161, 'MASE': 1.0918} +INFO:pyaf.std:MODEL_PERFS Fit STEP=12 {'MAPE': 0.1761, 'RMSE': 0.9118, 'MAE': 0.6792, 'MASE': 0.7728} +INFO:pyaf.std:MODEL_PERFS Forecast STEP=12 {'MAPE': 0.1765, 'RMSE': 0.663, 'MAE': 0.5552, 'MASE': 0.7151} +INFO:pyaf.std:MODEL_PERFS Test STEP=12 {'MAPE': 0.2209, 'RMSE': 0.5962, 'MAE': 0.5161, 'MASE': 1.0918} INFO:pyaf.std:MODEL_COMPLEXITY {'Decomposition': 'S', 'Transformation': 'S', 'Trend': 'S', 'Cycle': 'S', 'AR': 'S'} [SSSSS] INFO:pyaf.std:SIGNAL_TRANSFORMATION_DETAIL_START INFO:pyaf.std:SIGNAL_TRANSFORMATION_MODEL_VALUES NoTransf None @@ -58,9 +51,9 @@ INFO:pyaf.std:SEASONAL_MODEL_VALUES _Ozone_LinearTrend_residue_Seasonal_MonthOfY INFO:pyaf.std:CYCLE_MODEL_DETAIL_END INFO:pyaf.std:AR_MODEL_DETAIL_START INFO:pyaf.std:AR_MODEL_DETAIL_END -INFO:pyaf.std:TRAINING_TIME_IN_SECONDS 28.18 +INFO:pyaf.std:TRAINING_TIME_IN_SECONDS 33.818 INFO:pyaf.timing:('OPERATION_START', ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 1.392, ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 1.007, ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12})) Transformation DecompositionType ... Voting Complexity 1 _Ozone T+S+R ... 612.5000 SSSSS 0 Diff_Ozone T+S+R ... 617.1667 LMSSS @@ -120,14 +113,20 @@ Forecasts { "Ozone": { + "Complexity": { + "AR": "S", + "Cycle": "S", + "Decomposition": "S", + "Transformation": "S", + "Trend": "S" + }, "Dataset": { "Signal": "Ozone", "Time": { "Horizon": 12, - "TimeMinMax": [ - "1955-01-01 00:00:00", - "1971-12-01 00:00:00" - ], + "TimeDelta": "", + "TimeMax": "1971-12-01 00:00:00", + "TimeMin": "1955-01-01 00:00:00", "TimeVariable": "Time" }, "Training_Signal_Length": 204 @@ -141,27 +140,50 @@ Forecasts "Trend": "LinearTrend" }, "Model_Performance": { - "AUC": 0.5766, - "COMPLEXITY": { - "AR": "S", - "Cycle": "S", - "Decomposition": "S", - "Transformation": "S", - "Trend": "S" + "1": { + "AUC": 0.5766, + "DiffSMAPE": 0.1903, + "ErrorMean": -0.3403, + "ErrorStdDev": 0.569, + "KS": 0.2051, + "KendallTau": 0.7354, + "Length": 39, + "LnQ": 2.1757, + "MAE": 0.5552, + "MAPE": 0.1765, + "MASE": 0.7151, + "MannWhitneyU": 877.0, + "MedAE": 0.5576, + "Pearson": 0.858, + "R2": 0.6262, + "RMSE": 0.663, + "SMAPE": 0.1941, + "Signal": "Ozone_Forecast_1" }, - "CRPS": 0.2855, - "KS": 0.2051, - "KendallTau": 0.7354, - "LnQ": 2.1757, - "MAE": 0.5552, - "MAPE": 0.1765, - "MASE": 0.7151, - "MannWhitneyU": 877.0, - "MedAE": 0.5576, - "RMSE": 0.663 - } + "12": { + "AUC": 0.5766, + "DiffSMAPE": 0.1903, + "ErrorMean": -0.3403, + "ErrorStdDev": 0.569, + "KS": 0.2051, + "KendallTau": 0.7354, + "Length": 39, + "LnQ": 2.1757, + "MAE": 0.5552, + "MAPE": 0.1765, + "MASE": 0.7151, + "MannWhitneyU": 877.0, + "MedAE": 0.5576, + "Pearson": 0.858, + "R2": 0.6262, + "RMSE": 0.663, + "SMAPE": 0.1941, + "Signal": "Ozone_Forecast_12" + } + }, + "Model_Selection_Criterion": "MAPE" }, - "Training_Time": 28.18 + "Training_Time": 33.818 } diff --git a/tests/references/sampling/test_ozone_with_sampling_2.log b/tests/references/sampling/test_ozone_with_sampling_2.log index bae6c07a6..67a139f4e 100644 --- a/tests/references/sampling/test_ozone_with_sampling_2.log +++ b/tests/references/sampling/test_ozone_with_sampling_2.log @@ -10,20 +10,20 @@ INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transform INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'})) INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'})) INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 12.186, ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 12.5, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 12.59, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 16.29, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 16.459, ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', 'None', '_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', 'CumSum_', 'T+S+R')], 'Cores': 4})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 20.088, ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 21.409, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 22.391, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 29.182, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 29.405, ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', 'None', '_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', 'CumSum_', 'T+S+R')], 'Cores': 4})) INFO:pyaf.timing:('OPERATION_START', ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1})) INFO:pyaf.timing:('OPERATION_START', ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.215, ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.599, ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]})) INFO:pyaf.timing:('OPERATION_START', ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 1.384, ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 2.096, ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR'})) INFO:pyaf.timing:('OPERATION_START', ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 1.405, ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 3.005, ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 19.467, ('TRAINING', {'Signals': ['Ozone'], 'Horizons': {'Ozone': 12}})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 2.123, ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 4.819, ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 34.289, ('TRAINING', {'Signals': ['Ozone'], 'Horizons': {'Ozone': 12}})) INFO:pyaf.std:TIME_DETAIL TimeVariable='Time' TimeMin=1955-01-01T00:00:00.000000 TimeMax=1967-09-01T00:00:00.000000 TimeDelta= Horizon=12 INFO:pyaf.std:SIGNAL_DETAIL_ORIG SignalVariable='Ozone' Length=204 Min=1.2 Max=8.7 Mean=3.835784 StdDev=1.491559 INFO:pyaf.std:SIGNAL_DETAIL_TRANSFORMED TransformedSignalVariable='_Ozone' Min=0.0 Max=1.0 Mean=0.351438 StdDev=0.198875 @@ -33,19 +33,12 @@ INFO:pyaf.std:BEST_DECOMPOSITION '_Ozone_LinearTrend_residue_Seasonal_MonthOfYe INFO:pyaf.std:TREND_DETAIL '_Ozone_LinearTrend' [LinearTrend] INFO:pyaf.std:CYCLE_DETAIL '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear' [Seasonal_MonthOfYear] INFO:pyaf.std:AUTOREG_DETAIL '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR' [NoAR] -INFO:pyaf.std:MODEL_MAPE MAPE_Fit=0.1761 MAPE_Forecast=0.1765 MAPE_Test=0.2209 -INFO:pyaf.std:MODEL_SMAPE SMAPE_Fit=0.1712 SMAPE_Forecast=0.1941 SMAPE_Test=0.2249 -INFO:pyaf.std:MODEL_DiffSMAPE DiffSMAPE_Fit=0.1688 DiffSMAPE_Forecast=0.1903 DiffSMAPE_Test=0.2196 -INFO:pyaf.std:MODEL_MASE MASE_Fit=0.7728 MASE_Forecast=0.7151 MASE_Test=1.0918 -INFO:pyaf.std:MODEL_CRPS CRPS_Fit=0.3308 CRPS_Forecast=0.2855 CRPS_Test=0.3409 -INFO:pyaf.std:MODEL_L1 L1_Fit=0.6792 L1_Forecast=0.5552 L1_Test=0.5161 -INFO:pyaf.std:MODEL_L2 L2_Fit=0.9118 L2_Forecast=0.663 L2_Test=0.5962 -INFO:pyaf.std:MODEL_LnQ LnQ_Fit=7.4081 LnQ_Forecast=2.1757 LnQ_Test=0.9116 -INFO:pyaf.std:MODEL_MEDIAN_AE MedAE_Fit=0.5329 MedAE_Forecast=0.5576 MedAE_Test=0.5692 -INFO:pyaf.std:MODEL_KENDALL_TAU KENDALL_TAU_Fit=0.6293 KENDALL_TAU_Forecast=0.7354 KENDALL_TAU_Test=0.6565 -INFO:pyaf.std:MODEL_KOLMOGOROV_SMIRNOV KS_Fit=0.0915 KS_Forecast=0.2051 KS_Test=0.25 -INFO:pyaf.std:MODEL_MANN_WHITNEY_U MWU_Fit=11828.5 MWU_Forecast=877.0 MWU_Test=68.0 -INFO:pyaf.std:MODEL_AUC AUC_Fit=0.5053 AUC_Forecast=0.5766 AUC_Test=0.4722 +INFO:pyaf.std:MODEL_PERFS Fit STEP=1 {'MAPE': 0.1761, 'RMSE': 0.9118, 'MAE': 0.6792, 'MASE': 0.7728} +INFO:pyaf.std:MODEL_PERFS Forecast STEP=1 {'MAPE': 0.1765, 'RMSE': 0.663, 'MAE': 0.5552, 'MASE': 0.7151} +INFO:pyaf.std:MODEL_PERFS Test STEP=1 {'MAPE': 0.2209, 'RMSE': 0.5962, 'MAE': 0.5161, 'MASE': 1.0918} +INFO:pyaf.std:MODEL_PERFS Fit STEP=12 {'MAPE': 0.1761, 'RMSE': 0.9118, 'MAE': 0.6792, 'MASE': 0.7728} +INFO:pyaf.std:MODEL_PERFS Forecast STEP=12 {'MAPE': 0.1765, 'RMSE': 0.663, 'MAE': 0.5552, 'MASE': 0.7151} +INFO:pyaf.std:MODEL_PERFS Test STEP=12 {'MAPE': 0.2209, 'RMSE': 0.5962, 'MAE': 0.5161, 'MASE': 1.0918} INFO:pyaf.std:MODEL_COMPLEXITY {'Decomposition': 'S', 'Transformation': 'S', 'Trend': 'S', 'Cycle': 'S', 'AR': 'S'} [SSSSS] INFO:pyaf.std:SIGNAL_TRANSFORMATION_DETAIL_START INFO:pyaf.std:SIGNAL_TRANSFORMATION_MODEL_VALUES NoTransf None @@ -58,9 +51,9 @@ INFO:pyaf.std:SEASONAL_MODEL_VALUES _Ozone_LinearTrend_residue_Seasonal_MonthOfY INFO:pyaf.std:CYCLE_MODEL_DETAIL_END INFO:pyaf.std:AR_MODEL_DETAIL_START INFO:pyaf.std:AR_MODEL_DETAIL_END -INFO:pyaf.std:TRAINING_TIME_IN_SECONDS 19.467 +INFO:pyaf.std:TRAINING_TIME_IN_SECONDS 34.289 INFO:pyaf.timing:('OPERATION_START', ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.711, ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.698, ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12})) Transformation DecompositionType ... Voting Complexity 1 _Ozone T+S+R ... 612.5000 SSSSS 0 Diff_Ozone T+S+R ... 617.1667 LMSSS @@ -120,14 +113,20 @@ Forecasts { "Ozone": { + "Complexity": { + "AR": "S", + "Cycle": "S", + "Decomposition": "S", + "Transformation": "S", + "Trend": "S" + }, "Dataset": { "Signal": "Ozone", "Time": { "Horizon": 12, - "TimeMinMax": [ - "1955-01-01 00:00:00", - "1971-12-01 00:00:00" - ], + "TimeDelta": "", + "TimeMax": "1971-12-01 00:00:00", + "TimeMin": "1955-01-01 00:00:00", "TimeVariable": "Time" }, "Training_Signal_Length": 204 @@ -141,27 +140,50 @@ Forecasts "Trend": "LinearTrend" }, "Model_Performance": { - "AUC": 0.5766, - "COMPLEXITY": { - "AR": "S", - "Cycle": "S", - "Decomposition": "S", - "Transformation": "S", - "Trend": "S" + "1": { + "AUC": 0.5766, + "DiffSMAPE": 0.1903, + "ErrorMean": -0.3403, + "ErrorStdDev": 0.569, + "KS": 0.2051, + "KendallTau": 0.7354, + "Length": 39, + "LnQ": 2.1757, + "MAE": 0.5552, + "MAPE": 0.1765, + "MASE": 0.7151, + "MannWhitneyU": 877.0, + "MedAE": 0.5576, + "Pearson": 0.858, + "R2": 0.6262, + "RMSE": 0.663, + "SMAPE": 0.1941, + "Signal": "Ozone_Forecast_1" }, - "CRPS": 0.2855, - "KS": 0.2051, - "KendallTau": 0.7354, - "LnQ": 2.1757, - "MAE": 0.5552, - "MAPE": 0.1765, - "MASE": 0.7151, - "MannWhitneyU": 877.0, - "MedAE": 0.5576, - "RMSE": 0.663 - } + "12": { + "AUC": 0.5766, + "DiffSMAPE": 0.1903, + "ErrorMean": -0.3403, + "ErrorStdDev": 0.569, + "KS": 0.2051, + "KendallTau": 0.7354, + "Length": 39, + "LnQ": 2.1757, + "MAE": 0.5552, + "MAPE": 0.1765, + "MASE": 0.7151, + "MannWhitneyU": 877.0, + "MedAE": 0.5576, + "Pearson": 0.858, + "R2": 0.6262, + "RMSE": 0.663, + "SMAPE": 0.1941, + "Signal": "Ozone_Forecast_12" + } + }, + "Model_Selection_Criterion": "MAPE" }, - "Training_Time": 19.467 + "Training_Time": 34.289 } diff --git a/tests/references/sampling/test_ozone_with_sampling_3.log b/tests/references/sampling/test_ozone_with_sampling_3.log index b02736091..1b85d952c 100644 --- a/tests/references/sampling/test_ozone_with_sampling_3.log +++ b/tests/references/sampling/test_ozone_with_sampling_3.log @@ -7,23 +7,23 @@ INFO:pyaf.timing:('OPERATION_START', ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 3 1955-04 5.0 1955-04-01 4 1955-05 6.5 1955-05-01 INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'})) INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'})) INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 10.503, ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 10.899, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 11.408, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 13.896, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 14.091, ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', 'None', '_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', 'CumSum_', 'T+S+R')], 'Cores': 4})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 15.208, ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 16.496, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 16.999, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 21.207, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 21.491, ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', 'None', '_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', 'CumSum_', 'T+S+R')], 'Cores': 4})) INFO:pyaf.timing:('OPERATION_START', ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1})) INFO:pyaf.timing:('OPERATION_START', ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.601, ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.602, ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]})) INFO:pyaf.timing:('OPERATION_START', ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 2.585, ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 2.288, ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR'})) INFO:pyaf.timing:('OPERATION_START', ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 1.707, ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 4.893, ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 18.988, ('TRAINING', {'Signals': ['Ozone'], 'Horizons': {'Ozone': 12}})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 2.202, ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 5.093, ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 26.588, ('TRAINING', {'Signals': ['Ozone'], 'Horizons': {'Ozone': 12}})) INFO:pyaf.std:TIME_DETAIL TimeVariable='Time' TimeMin=1955-01-01T00:00:00.000000 TimeMax=1967-09-01T00:00:00.000000 TimeDelta= Horizon=12 INFO:pyaf.std:SIGNAL_DETAIL_ORIG SignalVariable='Ozone' Length=204 Min=1.2 Max=8.7 Mean=3.835784 StdDev=1.491559 INFO:pyaf.std:SIGNAL_DETAIL_TRANSFORMED TransformedSignalVariable='_Ozone' Min=0.0 Max=1.0 Mean=0.351438 StdDev=0.198875 @@ -33,19 +33,12 @@ INFO:pyaf.std:BEST_DECOMPOSITION '_Ozone_LinearTrend_residue_Seasonal_MonthOfYe INFO:pyaf.std:TREND_DETAIL '_Ozone_LinearTrend' [LinearTrend] INFO:pyaf.std:CYCLE_DETAIL '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear' [Seasonal_MonthOfYear] INFO:pyaf.std:AUTOREG_DETAIL '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR' [NoAR] -INFO:pyaf.std:MODEL_MAPE MAPE_Fit=0.1761 MAPE_Forecast=0.1765 MAPE_Test=0.2209 -INFO:pyaf.std:MODEL_SMAPE SMAPE_Fit=0.1712 SMAPE_Forecast=0.1941 SMAPE_Test=0.2249 -INFO:pyaf.std:MODEL_DiffSMAPE DiffSMAPE_Fit=0.1688 DiffSMAPE_Forecast=0.1903 DiffSMAPE_Test=0.2196 -INFO:pyaf.std:MODEL_MASE MASE_Fit=0.7728 MASE_Forecast=0.7151 MASE_Test=1.0918 -INFO:pyaf.std:MODEL_CRPS CRPS_Fit=0.3308 CRPS_Forecast=0.2855 CRPS_Test=0.3409 -INFO:pyaf.std:MODEL_L1 L1_Fit=0.6792 L1_Forecast=0.5552 L1_Test=0.5161 -INFO:pyaf.std:MODEL_L2 L2_Fit=0.9118 L2_Forecast=0.663 L2_Test=0.5962 -INFO:pyaf.std:MODEL_LnQ LnQ_Fit=7.4081 LnQ_Forecast=2.1757 LnQ_Test=0.9116 -INFO:pyaf.std:MODEL_MEDIAN_AE MedAE_Fit=0.5329 MedAE_Forecast=0.5576 MedAE_Test=0.5692 -INFO:pyaf.std:MODEL_KENDALL_TAU KENDALL_TAU_Fit=0.6293 KENDALL_TAU_Forecast=0.7354 KENDALL_TAU_Test=0.6565 -INFO:pyaf.std:MODEL_KOLMOGOROV_SMIRNOV KS_Fit=0.0915 KS_Forecast=0.2051 KS_Test=0.25 -INFO:pyaf.std:MODEL_MANN_WHITNEY_U MWU_Fit=11828.5 MWU_Forecast=877.0 MWU_Test=68.0 -INFO:pyaf.std:MODEL_AUC AUC_Fit=0.5053 AUC_Forecast=0.5766 AUC_Test=0.4722 +INFO:pyaf.std:MODEL_PERFS Fit STEP=1 {'MAPE': 0.1761, 'RMSE': 0.9118, 'MAE': 0.6792, 'MASE': 0.7728} +INFO:pyaf.std:MODEL_PERFS Forecast STEP=1 {'MAPE': 0.1765, 'RMSE': 0.663, 'MAE': 0.5552, 'MASE': 0.7151} +INFO:pyaf.std:MODEL_PERFS Test STEP=1 {'MAPE': 0.2209, 'RMSE': 0.5962, 'MAE': 0.5161, 'MASE': 1.0918} +INFO:pyaf.std:MODEL_PERFS Fit STEP=12 {'MAPE': 0.1761, 'RMSE': 0.9118, 'MAE': 0.6792, 'MASE': 0.7728} +INFO:pyaf.std:MODEL_PERFS Forecast STEP=12 {'MAPE': 0.1765, 'RMSE': 0.663, 'MAE': 0.5552, 'MASE': 0.7151} +INFO:pyaf.std:MODEL_PERFS Test STEP=12 {'MAPE': 0.2209, 'RMSE': 0.5962, 'MAE': 0.5161, 'MASE': 1.0918} INFO:pyaf.std:MODEL_COMPLEXITY {'Decomposition': 'S', 'Transformation': 'S', 'Trend': 'S', 'Cycle': 'S', 'AR': 'S'} [SSSSS] INFO:pyaf.std:SIGNAL_TRANSFORMATION_DETAIL_START INFO:pyaf.std:SIGNAL_TRANSFORMATION_MODEL_VALUES NoTransf None @@ -58,9 +51,9 @@ INFO:pyaf.std:SEASONAL_MODEL_VALUES _Ozone_LinearTrend_residue_Seasonal_MonthOfY INFO:pyaf.std:CYCLE_MODEL_DETAIL_END INFO:pyaf.std:AR_MODEL_DETAIL_START INFO:pyaf.std:AR_MODEL_DETAIL_END -INFO:pyaf.std:TRAINING_TIME_IN_SECONDS 18.988 +INFO:pyaf.std:TRAINING_TIME_IN_SECONDS 26.588 INFO:pyaf.timing:('OPERATION_START', ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12})) -INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.606, ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12})) +INFO:pyaf.timing:('OPERATION_END_ELAPSED', 1.199, ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12})) Transformation DecompositionType ... Voting Complexity 1 _Ozone T+S+R ... 612.5000 SSSSS 0 Diff_Ozone T+S+R ... 617.1667 LMSSS @@ -120,14 +113,20 @@ Forecasts { "Ozone": { + "Complexity": { + "AR": "S", + "Cycle": "S", + "Decomposition": "S", + "Transformation": "S", + "Trend": "S" + }, "Dataset": { "Signal": "Ozone", "Time": { "Horizon": 12, - "TimeMinMax": [ - "1955-01-01 00:00:00", - "1971-12-01 00:00:00" - ], + "TimeDelta": "", + "TimeMax": "1971-12-01 00:00:00", + "TimeMin": "1955-01-01 00:00:00", "TimeVariable": "Time" }, "Training_Signal_Length": 204 @@ -141,27 +140,50 @@ Forecasts "Trend": "LinearTrend" }, "Model_Performance": { - "AUC": 0.5766, - "COMPLEXITY": { - "AR": "S", - "Cycle": "S", - "Decomposition": "S", - "Transformation": "S", - "Trend": "S" + "1": { + "AUC": 0.5766, + "DiffSMAPE": 0.1903, + "ErrorMean": -0.3403, + "ErrorStdDev": 0.569, + "KS": 0.2051, + "KendallTau": 0.7354, + "Length": 39, + "LnQ": 2.1757, + "MAE": 0.5552, + "MAPE": 0.1765, + "MASE": 0.7151, + "MannWhitneyU": 877.0, + "MedAE": 0.5576, + "Pearson": 0.858, + "R2": 0.6262, + "RMSE": 0.663, + "SMAPE": 0.1941, + "Signal": "Ozone_Forecast_1" }, - "CRPS": 0.2855, - "KS": 0.2051, - "KendallTau": 0.7354, - "LnQ": 2.1757, - "MAE": 0.5552, - "MAPE": 0.1765, - "MASE": 0.7151, - "MannWhitneyU": 877.0, - "MedAE": 0.5576, - "RMSE": 0.663 - } + "12": { + "AUC": 0.5766, + "DiffSMAPE": 0.1903, + "ErrorMean": -0.3403, + "ErrorStdDev": 0.569, + "KS": 0.2051, + "KendallTau": 0.7354, + "Length": 39, + "LnQ": 2.1757, + "MAE": 0.5552, + "MAPE": 0.1765, + "MASE": 0.7151, + "MannWhitneyU": 877.0, + "MedAE": 0.5576, + "Pearson": 0.858, + "R2": 0.6262, + "RMSE": 0.663, + "SMAPE": 0.1941, + "Signal": "Ozone_Forecast_12" + } + }, + "Model_Selection_Criterion": "MAPE" }, - "Training_Time": 18.988 + "Training_Time": 26.588 }