Skip to content

Commit

Permalink
Investigate Large Horizon Models #213. Improved perf logging. Display…
Browse files Browse the repository at this point in the history 
… the first and last multistep forecast perfs. Updated these perfs
  • Loading branch information
@antoinecarme
antoinecarme committed Mar 8, 2023
1 parent 7b33074 commit 1f56453
Show file tree
Hide file tree
Showing 4 changed files with 285 additions and 197 deletions.
122 changes: 72 additions & 50 deletions tests/references/sampling/test_ozone_with_no_sampling.log
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,24 +6,24 @@ INFO:pyaf.timing:('OPERATION_START', ('SIGNAL_TRAINING', {'Signals': ['Ozone'],
2 1955-03 3.6 1955-03-01
3 1955-04 5.0 1955-04-01
4 1955-05 6.5 1955-05-01
INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 14.091, ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 17.589, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 17.593, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 21.61, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 21.881, ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', 'None', '_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', 'CumSum_', 'T+S+R')], 'Cores': 4}))
INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 22.203, ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 23.188, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 24.375, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 28.892, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 29.177, ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', 'None', '_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', 'CumSum_', 'T+S+R')], 'Cores': 4}))
INFO:pyaf.timing:('OPERATION_START', ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1}))
INFO:pyaf.timing:('OPERATION_START', ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.711, ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.52, ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]}))
INFO:pyaf.timing:('OPERATION_START', ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 2.779, ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 1.88, ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR'}))
INFO:pyaf.timing:('OPERATION_START', ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 2.704, ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 6.195, ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 28.081, ('TRAINING', {'Signals': ['Ozone'], 'Horizons': {'Ozone': 12}}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 1.724, ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 4.124, ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 33.306, ('TRAINING', {'Signals': ['Ozone'], 'Horizons': {'Ozone': 12}}))
INFO:pyaf.std:TIME_DETAIL TimeVariable='Time' TimeMin=1955-01-01T00:00:00.000000 TimeMax=1967-09-01T00:00:00.000000 TimeDelta=<DateOffset: months=1> Horizon=12
INFO:pyaf.std:SIGNAL_DETAIL_ORIG SignalVariable='Ozone' Length=204 Min=1.2 Max=8.7 Mean=3.835784 StdDev=1.491559
INFO:pyaf.std:SIGNAL_DETAIL_TRANSFORMED TransformedSignalVariable='_Ozone' Min=0.0 Max=1.0 Mean=0.351438 StdDev=0.198875
Expand All @@ -33,19 +33,12 @@ INFO:pyaf.std:BEST_DECOMPOSITION '_Ozone_LinearTrend_residue_Seasonal_MonthOfYe
INFO:pyaf.std:TREND_DETAIL '_Ozone_LinearTrend' [LinearTrend]
INFO:pyaf.std:CYCLE_DETAIL '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear' [Seasonal_MonthOfYear]
INFO:pyaf.std:AUTOREG_DETAIL '_Ozone_LinearTrend_residue_Seasonal_MonthOfYear_residue_NoAR' [NoAR]
INFO:pyaf.std:MODEL_MAPE MAPE_Fit=0.1761 MAPE_Forecast=0.1765 MAPE_Test=0.2209
INFO:pyaf.std:MODEL_SMAPE SMAPE_Fit=0.1712 SMAPE_Forecast=0.1941 SMAPE_Test=0.2249
INFO:pyaf.std:MODEL_DiffSMAPE DiffSMAPE_Fit=0.1688 DiffSMAPE_Forecast=0.1903 DiffSMAPE_Test=0.2196
INFO:pyaf.std:MODEL_MASE MASE_Fit=0.7728 MASE_Forecast=0.7151 MASE_Test=1.0918
INFO:pyaf.std:MODEL_CRPS CRPS_Fit=0.3308 CRPS_Forecast=0.2855 CRPS_Test=0.3409
INFO:pyaf.std:MODEL_L1 L1_Fit=0.6792 L1_Forecast=0.5552 L1_Test=0.5161
INFO:pyaf.std:MODEL_L2 L2_Fit=0.9118 L2_Forecast=0.663 L2_Test=0.5962
INFO:pyaf.std:MODEL_LnQ LnQ_Fit=7.4081 LnQ_Forecast=2.1757 LnQ_Test=0.9116
INFO:pyaf.std:MODEL_MEDIAN_AE MedAE_Fit=0.5329 MedAE_Forecast=0.5576 MedAE_Test=0.5692
INFO:pyaf.std:MODEL_KENDALL_TAU KENDALL_TAU_Fit=0.6293 KENDALL_TAU_Forecast=0.7354 KENDALL_TAU_Test=0.6565
INFO:pyaf.std:MODEL_KOLMOGOROV_SMIRNOV KS_Fit=0.0915 KS_Forecast=0.2051 KS_Test=0.25
INFO:pyaf.std:MODEL_MANN_WHITNEY_U MWU_Fit=11828.5 MWU_Forecast=877.0 MWU_Test=68.0
INFO:pyaf.std:MODEL_AUC AUC_Fit=0.5053 AUC_Forecast=0.5766 AUC_Test=0.4722
INFO:pyaf.std:MODEL_PERFS Fit STEP=1 {'MAPE': 0.1761, 'RMSE': 0.9118, 'MAE': 0.6792, 'MASE': 0.7728}
INFO:pyaf.std:MODEL_PERFS Forecast STEP=1 {'MAPE': 0.1765, 'RMSE': 0.663, 'MAE': 0.5552, 'MASE': 0.7151}
INFO:pyaf.std:MODEL_PERFS Test STEP=1 {'MAPE': 0.2209, 'RMSE': 0.5962, 'MAE': 0.5161, 'MASE': 1.0918}
INFO:pyaf.std:MODEL_PERFS Fit STEP=12 {'MAPE': 0.1761, 'RMSE': 0.9118, 'MAE': 0.6792, 'MASE': 0.7728}
INFO:pyaf.std:MODEL_PERFS Forecast STEP=12 {'MAPE': 0.1765, 'RMSE': 0.663, 'MAE': 0.5552, 'MASE': 0.7151}
INFO:pyaf.std:MODEL_PERFS Test STEP=12 {'MAPE': 0.2209, 'RMSE': 0.5962, 'MAE': 0.5161, 'MASE': 1.0918}
INFO:pyaf.std:MODEL_COMPLEXITY {'Decomposition': 'S', 'Transformation': 'S', 'Trend': 'S', 'Cycle': 'S', 'AR': 'S'} [SSSSS]
INFO:pyaf.std:SIGNAL_TRANSFORMATION_DETAIL_START
INFO:pyaf.std:SIGNAL_TRANSFORMATION_MODEL_VALUES NoTransf None
Expand All @@ -58,9 +51,9 @@ INFO:pyaf.std:SEASONAL_MODEL_VALUES _Ozone_LinearTrend_residue_Seasonal_MonthOfY
INFO:pyaf.std:CYCLE_MODEL_DETAIL_END
INFO:pyaf.std:AR_MODEL_DETAIL_START
INFO:pyaf.std:AR_MODEL_DETAIL_END
INFO:pyaf.std:TRAINING_TIME_IN_SECONDS 28.081
INFO:pyaf.std:TRAINING_TIME_IN_SECONDS 33.306
INFO:pyaf.timing:('OPERATION_START', ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 1.198, ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.999, ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12}))
Transformation DecompositionType ... Voting Complexity
1 _Ozone T+S+R ... 612.5000 SSSSS
0 Diff_Ozone T+S+R ... 617.1667 LMSSS
Expand Down Expand Up @@ -120,14 +113,20 @@ Forecasts
<ModelInfo>
{
"Ozone": {
"Complexity": {
"AR": "S",
"Cycle": "S",
"Decomposition": "S",
"Transformation": "S",
"Trend": "S"
},
"Dataset": {
"Signal": "Ozone",
"Time": {
"Horizon": 12,
"TimeMinMax": [
"1955-01-01 00:00:00",
"1971-12-01 00:00:00"
],
"TimeDelta": "<DateOffset: months=1>",
"TimeMax": "1971-12-01 00:00:00",
"TimeMin": "1955-01-01 00:00:00",
"TimeVariable": "Time"
},
"Training_Signal_Length": 204
Expand All @@ -141,27 +140,50 @@ Forecasts
"Trend": "LinearTrend"
},
"Model_Performance": {
"AUC": 0.5766,
"COMPLEXITY": {
"AR": "S",
"Cycle": "S",
"Decomposition": "S",
"Transformation": "S",
"Trend": "S"
"1": {
"AUC": 0.5766,
"DiffSMAPE": 0.1903,
"ErrorMean": -0.3403,
"ErrorStdDev": 0.569,
"KS": 0.2051,
"KendallTau": 0.7354,
"Length": 39,
"LnQ": 2.1757,
"MAE": 0.5552,
"MAPE": 0.1765,
"MASE": 0.7151,
"MannWhitneyU": 877.0,
"MedAE": 0.5576,
"Pearson": 0.858,
"R2": 0.6262,
"RMSE": 0.663,
"SMAPE": 0.1941,
"Signal": "Ozone_Forecast_1"
},
"CRPS": 0.2855,
"KS": 0.2051,
"KendallTau": 0.7354,
"LnQ": 2.1757,
"MAE": 0.5552,
"MAPE": 0.1765,
"MASE": 0.7151,
"MannWhitneyU": 877.0,
"MedAE": 0.5576,
"RMSE": 0.663
}
"12": {
"AUC": 0.5766,
"DiffSMAPE": 0.1903,
"ErrorMean": -0.3403,
"ErrorStdDev": 0.569,
"KS": 0.2051,
"KendallTau": 0.7354,
"Length": 39,
"LnQ": 2.1757,
"MAE": 0.5552,
"MAPE": 0.1765,
"MASE": 0.7151,
"MannWhitneyU": 877.0,
"MedAE": 0.5576,
"Pearson": 0.858,
"R2": 0.6262,
"RMSE": 0.663,
"SMAPE": 0.1941,
"Signal": "Ozone_Forecast_12"
}
},
"Model_Selection_Criterion": "MAPE"
},
"Training_Time": 28.081
"Training_Time": 33.306
}
</ModelInfo>

Expand Down
Loading

0 comments on commit 1f56453

Please sign in to comment.