Skip to content

Commit

Permalink
Investigate Large Horizon Models #213. Improved perf logging. Display…
Browse files Browse the repository at this point in the history 
… the first and last multistep forecast perfs. Updated these perfs
  • Loading branch information
@antoinecarme
antoinecarme committed Mar 8, 2023
1 parent e69d48e commit 7b33074
Show file tree
Hide file tree
Showing 4 changed files with 311 additions and 223 deletions.
134 changes: 78 additions & 56 deletions tests/references/perfs/test_ozone_perf_measure_L1.log
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,13 @@ INFO:pyaf.timing:('OPERATION_START', ('SIGNAL_TRAINING', {'Signals': ['Ozone'],
4 1955-05 6.5 1955-05-01
INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 16.599, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 17.007, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 18.498, ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 21.714, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 22.007, ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', 'None', '_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', 'CumSum_', 'T+S+R')], 'Cores': 4}))
INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 26.601, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'CumSum_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 26.608, ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 27.623, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'Diff_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 34.895, ('TRAINING', {'Signal': 'Ozone', 'Transformation': 'RelDiff_Ozone', 'DecompositionType': 'T+S+R'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 35.221, ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', 'None', '_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', 'CumSum_', 'T+S+R')], 'Cores': 4}))
INFO:pyaf.timing:('OPERATION_START', ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1}))
INFO:pyaf.timing:('OPERATION_START', ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]}))
INFO:pyaf.std:PERF_DUMP_START
Expand Down Expand Up @@ -98,29 +98,29 @@ INFO:pyaf.std:[
"Voting": 574.0
},
{
"Category": "Integration_Lag1Trend_Seasonal_MonthOfYear_NoAR",
"Category": "Integration_Lag1Trend_Cycle_12_NoAR",
"Complexity": "MSSSS",
"DetailedFormula": [
"CumSum_Ozone",
"T+S+R",
null,
"CumSum_Ozone_Lag1Trend_residue_Seasonal_MonthOfYear_residue_NoAR"
"CumSum_Ozone_Lag1Trend_residue_Cycle_12_residue_NoAR"
],
"ForecastL1": 0.7628205128205279,
"Model": "CumSum_Ozone_Lag1Trend_residue_Seasonal_MonthOfYear_residue_NoAR",
"Model": "CumSum_Ozone_Lag1Trend_residue_Cycle_12_residue_NoAR",
"Voting": 552.9167
},
{
"Category": "Integration_Lag1Trend_Cycle_12_NoAR",
"Category": "Integration_Lag1Trend_Seasonal_MonthOfYear_NoAR",
"Complexity": "MSSSS",
"DetailedFormula": [
"CumSum_Ozone",
"T+S+R",
null,
"CumSum_Ozone_Lag1Trend_residue_bestCycle_byMAPE_residue_NoAR"
"CumSum_Ozone_Lag1Trend_residue_Seasonal_MonthOfYear_residue_NoAR"
],
"ForecastL1": 0.7628205128205279,
"Model": "CumSum_Ozone_Lag1Trend_residue_bestCycle_byMAPE_residue_NoAR",
"Model": "CumSum_Ozone_Lag1Trend_residue_Seasonal_MonthOfYear_residue_NoAR",
"Voting": 552.9167
},
{
Expand Down Expand Up @@ -151,13 +151,13 @@ INFO:pyaf.std:[
}
]
INFO:pyaf.std:PERF_DUMP_END
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.787, ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.597, ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')]}))
INFO:pyaf.timing:('OPERATION_START', ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': 'Diff_Ozone_ConstantTrend_residue_Seasonal_MonthOfYear_residue_AR(51)'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 3.111, ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': 'Diff_Ozone_ConstantTrend_residue_Seasonal_MonthOfYear_residue_AR(51)'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 2.477, ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': 'Diff_Ozone_ConstantTrend_residue_Seasonal_MonthOfYear_residue_AR(51)'}))
INFO:pyaf.timing:('OPERATION_START', ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 2.985, ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 6.883, ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 28.895, ('TRAINING', {'Signals': ['Ozone'], 'Horizons': {'Ozone': 12}}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 2.82, ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 5.895, ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', 'CumSum_', 'T+S+R'), ('Ozone', 'None', 'Diff_', 'T+S+R'), ('Ozone', 'None', 'RelDiff_', 'T+S+R'), ('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 41.12, ('TRAINING', {'Signals': ['Ozone'], 'Horizons': {'Ozone': 12}}))
INFO:pyaf.std:TIME_DETAIL TimeVariable='Time' TimeMin=1955-01-01T00:00:00.000000 TimeMax=1967-09-01T00:00:00.000000 TimeDelta=<DateOffset: months=1> Horizon=12
INFO:pyaf.std:SIGNAL_DETAIL_ORIG SignalVariable='Ozone' Length=204 Min=1.2 Max=8.7 Mean=3.835784 StdDev=1.491559
INFO:pyaf.std:SIGNAL_DETAIL_TRANSFORMED TransformedSignalVariable='Diff_Ozone' Min=-0.573333 Max=0.466667 Mean=-0.00098 StdDev=0.147082
Expand All @@ -167,19 +167,12 @@ INFO:pyaf.std:BEST_DECOMPOSITION 'Diff_Ozone_ConstantTrend_residue_Seasonal_Mon
INFO:pyaf.std:TREND_DETAIL 'Diff_Ozone_ConstantTrend' [ConstantTrend]
INFO:pyaf.std:CYCLE_DETAIL 'Diff_Ozone_ConstantTrend_residue_Seasonal_MonthOfYear' [Seasonal_MonthOfYear]
INFO:pyaf.std:AUTOREG_DETAIL 'Diff_Ozone_ConstantTrend_residue_Seasonal_MonthOfYear_residue_AR(51)' [AR]
INFO:pyaf.std:MODEL_MAPE MAPE_Fit=0.213 MAPE_Forecast=0.1663 MAPE_Test=0.2094
INFO:pyaf.std:MODEL_SMAPE SMAPE_Fit=0.2436 SMAPE_Forecast=0.1688 SMAPE_Test=0.1758
INFO:pyaf.std:MODEL_DiffSMAPE DiffSMAPE_Fit=0.2403 DiffSMAPE_Forecast=0.1657 DiffSMAPE_Test=0.1722
INFO:pyaf.std:MODEL_MASE MASE_Fit=1.1186 MASE_Forecast=0.6614 MASE_Test=0.9625
INFO:pyaf.std:MODEL_CRPS CRPS_Fit=0.319 CRPS_Forecast=0.2169 CRPS_Test=0.2743
INFO:pyaf.std:MODEL_L1 L1_Fit=0.9832 L1_Forecast=0.5135 L1_Test=0.455
INFO:pyaf.std:MODEL_L2 L2_Fit=1.3481 L2_Forecast=0.6435 L2_Test=0.5937
INFO:pyaf.std:MODEL_LnQ LnQ_Fit=14.876 LnQ_Forecast=1.5569 LnQ_Test=0.6808
INFO:pyaf.std:MODEL_MEDIAN_AE MedAE_Fit=0.7351 MedAE_Forecast=0.4355 MedAE_Test=0.4789
INFO:pyaf.std:MODEL_KENDALL_TAU KENDALL_TAU_Fit=0.6026 KENDALL_TAU_Forecast=0.7354 KENDALL_TAU_Test=0.6565
INFO:pyaf.std:MODEL_KOLMOGOROV_SMIRNOV KS_Fit=0.3922 KS_Forecast=0.2051 KS_Test=0.4167
INFO:pyaf.std:MODEL_MANN_WHITNEY_U MWU_Fit=15690.0 MWU_Forecast=828.0 MWU_Test=48.0
INFO:pyaf.std:MODEL_AUC AUC_Fit=0.6703 AUC_Forecast=0.5444 AUC_Test=0.3333
INFO:pyaf.std:MODEL_PERFS Fit STEP=1 {'MAPE': 0.2891, 'RMSE': 1.7863, 'MAE': 1.3346, 'MASE': 1.5184, 'L1': 1.3346}
INFO:pyaf.std:MODEL_PERFS Forecast STEP=1 {'MAPE': 0.1792, 'RMSE': 0.7111, 'MAE': 0.56, 'MASE': 0.7214, 'L1': 0.56}
INFO:pyaf.std:MODEL_PERFS Test STEP=1 {'MAPE': 0.3045, 'RMSE': 0.7974, 'MAE': 0.6623, 'MASE': 1.4009, 'L1': 0.6623}
INFO:pyaf.std:MODEL_PERFS Fit STEP=12 {'MAPE': 0.213, 'RMSE': 1.3481, 'MAE': 0.9832, 'MASE': 1.1186, 'L1': 0.9832}
INFO:pyaf.std:MODEL_PERFS Forecast STEP=12 {'MAPE': 0.1663, 'RMSE': 0.6435, 'MAE': 0.5135, 'MASE': 0.6614, 'L1': 0.5135}
INFO:pyaf.std:MODEL_PERFS Test STEP=12 {'MAPE': 0.2094, 'RMSE': 0.5937, 'MAE': 0.455, 'MASE': 0.9625, 'L1': 0.455}
INFO:pyaf.std:MODEL_COMPLEXITY {'Decomposition': 'S', 'Transformation': 'M', 'Trend': 'S', 'Cycle': 'S', 'AR': 'L'} [LMSSS]
INFO:pyaf.std:SIGNAL_TRANSFORMATION_DETAIL_START
INFO:pyaf.std:DIFFERENCING_TRANSFORMATION Difference 0.20000000000000007
Expand All @@ -202,11 +195,11 @@ INFO:pyaf.std:AR_MODEL_COEFF 8 Diff_Ozone_ConstantTrend_residue_Seasonal_MonthOf
INFO:pyaf.std:AR_MODEL_COEFF 9 Diff_Ozone_ConstantTrend_residue_Seasonal_MonthOfYear_residue_Lag29 -0.062049
INFO:pyaf.std:AR_MODEL_COEFF 10 Diff_Ozone_ConstantTrend_residue_Seasonal_MonthOfYear_residue_Lag46 -0.061005
INFO:pyaf.std:AR_MODEL_DETAIL_END
INFO:pyaf.std:TRAINING_TIME_IN_SECONDS 28.895
INFO:pyaf.std:TRAINING_TIME_IN_SECONDS 41.12
INFO:pyaf.timing:('OPERATION_START', ('PLOTTING', {'Signals': ['Ozone']}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 15.994, ('PLOTTING', {'Signals': ['Ozone']}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 20.024, ('PLOTTING', {'Signals': ['Ozone']}))
INFO:pyaf.timing:('OPERATION_START', ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 1.007, ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12}))
INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.882, ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12}))
Transformation DecompositionType ... Voting Complexity
0 Diff_Ozone T+S+R ... 616.4167 LMSSS

Expand Down Expand Up @@ -265,14 +258,20 @@ Forecasts
<ModelInfo>
{
"Ozone": {
"Complexity": {
"AR": "L",
"Cycle": "S",
"Decomposition": "S",
"Transformation": "M",
"Trend": "S"
},
"Dataset": {
"Signal": "Ozone",
"Time": {
"Horizon": 12,
"TimeMinMax": [
"1955-01-01 00:00:00",
"1971-12-01 00:00:00"
],
"TimeDelta": "<DateOffset: months=1>",
"TimeMax": "1971-12-01 00:00:00",
"TimeMin": "1955-01-01 00:00:00",
"TimeVariable": "Time"
},
"Training_Signal_Length": 204
Expand All @@ -286,27 +285,50 @@ Forecasts
"Trend": "ConstantTrend"
},
"Model_Performance": {
"AUC": 0.5444,
"COMPLEXITY": {
"AR": "L",
"Cycle": "S",
"Decomposition": "S",
"Transformation": "M",
"Trend": "S"
"1": {
"AUC": 0.6003,
"DiffSMAPE": 0.1894,
"ErrorMean": -0.4037,
"ErrorStdDev": 0.5853,
"KS": 0.2051,
"KendallTau": 0.6917,
"Length": 39,
"LnQ": 2.1723,
"MAE": 0.56,
"MAPE": 0.1792,
"MASE": 0.7214,
"MannWhitneyU": 913.0,
"MedAE": 0.5111,
"Pearson": 0.843,
"R2": 0.57,
"RMSE": 0.7111,
"SMAPE": 0.1931,
"Signal": "Ozone_Forecast_1"
},
"CRPS": 0.2169,
"KS": 0.2051,
"KendallTau": 0.7354,
"LnQ": 1.5569,
"MAE": 0.5135,
"MAPE": 0.1663,
"MASE": 0.6614,
"MannWhitneyU": 828.0,
"MedAE": 0.4355,
"RMSE": 0.6435
}
"12": {
"AUC": 0.5444,
"DiffSMAPE": 0.1657,
"ErrorMean": -0.2368,
"ErrorStdDev": 0.5983,
"KS": 0.2051,
"KendallTau": 0.7354,
"Length": 39,
"LnQ": 1.5569,
"MAE": 0.5135,
"MAPE": 0.1663,
"MASE": 0.6614,
"MannWhitneyU": 828.0,
"MedAE": 0.4355,
"Pearson": 0.8427,
"R2": 0.6478,
"RMSE": 0.6435,
"SMAPE": 0.1688,
"Signal": "Ozone_Forecast_12"
}
},
"Model_Selection_Criterion": "L1"
},
"Training_Time": 28.895
"Training_Time": 41.12
}
</ModelInfo>

Expand Down
Loading

0 comments on commit 7b33074

Please sign in to comment.