Investigate Large Horizon Models #213. Improved perf logging. Display…

… the first and last multistep forecast perfs. Updated these perfs
antoinecarme · Mar 8, 2023 · f659796 · f659796
1 parent 664053c
commit f659796
Showing 1 changed file with 68 additions and 46 deletions.
diff --git a/tests/references/model_control/test_ozone_no_models_enabled.log b/tests/references/model_control/test_ozone_no_models_enabled.log
@@ -1,17 +1,17 @@
 INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signals': ['Ozone'], 'Horizons': {'Ozone': 12}}))
 INFO:pyaf.timing:('OPERATION_START', ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', 'None', '_', 'T+S+R')], 'Cores': 1}))
 INFO:pyaf.timing:('OPERATION_START', ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'}))
-INFO:pyaf.timing:('OPERATION_END_ELAPSED', 3.675, ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'}))
-INFO:pyaf.timing:('OPERATION_END_ELAPSED', 3.675, ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', 'None', '_', 'T+S+R')], 'Cores': 1}))
+INFO:pyaf.timing:('OPERATION_END_ELAPSED', 6.303, ('TRAINING', {'Signal': 'Ozone', 'Transformation': '_Ozone', 'DecompositionType': 'T+S+R'}))
+INFO:pyaf.timing:('OPERATION_END_ELAPSED', 6.304, ('SIGNAL_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', 'None', '_', 'T+S+R')], 'Cores': 1}))
 INFO:pyaf.timing:('OPERATION_START', ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1}))
 INFO:pyaf.timing:('OPERATION_START', ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', '_', 'T+S+R')]}))
-INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.501, ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', '_', 'T+S+R')]}))
+INFO:pyaf.timing:('OPERATION_END_ELAPSED', 0.301, ('MODEL_SELECTION', {'Signal': 'Ozone', 'Transformations': [('Ozone', 'None', '_', 'T+S+R')]}))
 INFO:pyaf.timing:('OPERATION_START', ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_ConstantTrend_residue_zeroCycle[0.0]_residue_NoAR'}))
-INFO:pyaf.timing:('OPERATION_END_ELAPSED', 4.51, ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_ConstantTrend_residue_zeroCycle[0.0]_residue_NoAR'}))
+INFO:pyaf.timing:('OPERATION_END_ELAPSED', 3.609, ('UPDATE_BEST_MODEL_PERFS', {'Signal': 'Ozone', 'Model': '_Ozone_ConstantTrend_residue_zeroCycle[0.0]_residue_NoAR'}))
 INFO:pyaf.timing:('OPERATION_START', ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'}))
-INFO:pyaf.timing:('OPERATION_END_ELAPSED', 5.005, ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'}))
-INFO:pyaf.timing:('OPERATION_END_ELAPSED', 10.121, ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1}))
-INFO:pyaf.timing:('OPERATION_END_ELAPSED', 13.803, ('TRAINING', {'Signals': ['Ozone'], 'Horizons': {'Ozone': 12}}))
+INFO:pyaf.timing:('OPERATION_END_ELAPSED', 3.592, ('COMPUTE_PREDICTION_INTERVALS', {'Signal': 'Ozone'}))
+INFO:pyaf.timing:('OPERATION_END_ELAPSED', 7.502, ('FINALIZE_TRAINING', {'Signals': ['Ozone'], 'Transformations': [('Ozone', [('Ozone', 'None', '_', 'T+S+R')])], 'Cores': 1}))
+INFO:pyaf.timing:('OPERATION_END_ELAPSED', 14.105, ('TRAINING', {'Signals': ['Ozone'], 'Horizons': {'Ozone': 12}}))
 INFO:pyaf.std:TIME_DETAIL TimeVariable='Time' TimeMin=1955-01-01T00:00:00.000000 TimeMax=1967-09-01T00:00:00.000000 TimeDelta=<DateOffset: months=1> Horizon=12
 INFO:pyaf.std:SIGNAL_DETAIL_ORIG SignalVariable='Ozone' Length=204  Min=1.2 Max=8.7  Mean=3.835784 StdDev=1.491559
 INFO:pyaf.std:SIGNAL_DETAIL_TRANSFORMED TransformedSignalVariable='_Ozone' Min=0.0 Max=1.0  Mean=0.351438 StdDev=0.198875
@@ -21,19 +21,12 @@ INFO:pyaf.std:BEST_DECOMPOSITION  '_Ozone_ConstantTrend_residue_zeroCycle[0.0]_r
 INFO:pyaf.std:TREND_DETAIL '_Ozone_ConstantTrend' [ConstantTrend]
 INFO:pyaf.std:CYCLE_DETAIL '_Ozone_ConstantTrend_residue_zeroCycle[0.0]' [NoCycle]
 INFO:pyaf.std:AUTOREG_DETAIL '_Ozone_ConstantTrend_residue_zeroCycle[0.0]_residue_NoAR' [NoAR]
-INFO:pyaf.std:MODEL_MAPE MAPE_Fit=0.3752 MAPE_Forecast=0.5344 MAPE_Test=0.8365
-INFO:pyaf.std:MODEL_SMAPE SMAPE_Fit=0.3108 SMAPE_Forecast=0.3669 SMAPE_Test=0.5274
-INFO:pyaf.std:MODEL_DiffSMAPE DiffSMAPE_Fit=0.3067 DiffSMAPE_Forecast=0.3613 DiffSMAPE_Test=0.5189
-INFO:pyaf.std:MODEL_MASE MASE_Fit=1.3939 MASE_Forecast=1.5593 MASE_Test=3.4569
-INFO:pyaf.std:MODEL_CRPS CRPS_Fit=0.1787 CRPS_Forecast=0.2519 CRPS_Test=0.6255
-INFO:pyaf.std:MODEL_L1 L1_Fit=1.2251 L1_Forecast=1.2105 L1_Test=1.6342
-INFO:pyaf.std:MODEL_L2 L2_Fit=1.5155 L2_Forecast=1.4267 L2_Test=1.7867
-INFO:pyaf.std:MODEL_LnQ LnQ_Fit=24.8433 LnQ_Forecast=9.2692 LnQ_Test=4.9059
-INFO:pyaf.std:MODEL_MEDIAN_AE MedAE_Fit=1.0908 MedAE_Forecast=1.1092 MedAE_Test=1.6592
-INFO:pyaf.std:MODEL_KENDALL_TAU KENDALL_TAU_Fit=nan KENDALL_TAU_Forecast=nan KENDALL_TAU_Test=nan
-INFO:pyaf.std:MODEL_KOLMOGOROV_SMIRNOV KS_Fit=0.5163 KS_Forecast=0.7949 KS_Test=1.0
-INFO:pyaf.std:MODEL_MANN_WHITNEY_U MWU_Fit=11322.0 MWU_Forecast=312.0 MWU_Test=0.0
-INFO:pyaf.std:MODEL_AUC AUC_Fit=0.4837 AUC_Forecast=0.2051 AUC_Test=0.0
+INFO:pyaf.std:MODEL_PERFS Fit STEP=1 {'MAPE': 0.3752, 'RMSE': 1.5155, 'MAE': 1.2251, 'MASE': 1.3939}
+INFO:pyaf.std:MODEL_PERFS Forecast STEP=1 {'MAPE': 0.5344, 'RMSE': 1.4267, 'MAE': 1.2105, 'MASE': 1.5593}
+INFO:pyaf.std:MODEL_PERFS Test STEP=1 {'MAPE': 0.8365, 'RMSE': 1.7867, 'MAE': 1.6342, 'MASE': 3.4569}
+INFO:pyaf.std:MODEL_PERFS Fit STEP=12 {'MAPE': 0.3752, 'RMSE': 1.5155, 'MAE': 1.2251, 'MASE': 1.3939}
+INFO:pyaf.std:MODEL_PERFS Forecast STEP=12 {'MAPE': 0.5344, 'RMSE': 1.4267, 'MAE': 1.2105, 'MASE': 1.5593}
+INFO:pyaf.std:MODEL_PERFS Test STEP=12 {'MAPE': 0.8365, 'RMSE': 1.7867, 'MAE': 1.6342, 'MASE': 3.4569}
 INFO:pyaf.std:MODEL_COMPLEXITY {'Decomposition': 'S', 'Transformation': 'S', 'Trend': 'S', 'Cycle': 'S', 'AR': 'S'} [SSSSS]
 INFO:pyaf.std:SIGNAL_TRANSFORMATION_DETAIL_START
 INFO:pyaf.std:SIGNAL_TRANSFORMATION_MODEL_VALUES NoTransf None
@@ -46,11 +39,11 @@ INFO:pyaf.std:ZERO_CYCLE_MODEL_VALUES _Ozone_ConstantTrend_residue_zeroCycle[0.0
 INFO:pyaf.std:CYCLE_MODEL_DETAIL_END
 INFO:pyaf.std:AR_MODEL_DETAIL_START
 INFO:pyaf.std:AR_MODEL_DETAIL_END
-INFO:pyaf.std:TRAINING_TIME_IN_SECONDS 13.803
+INFO:pyaf.std:TRAINING_TIME_IN_SECONDS 14.105
 INFO:pyaf.timing:('OPERATION_START', ('PLOTTING', {'Signals': ['Ozone']}))
-INFO:pyaf.timing:('OPERATION_END_ELAPSED', 54.499, ('PLOTTING', {'Signals': ['Ozone']}))
+INFO:pyaf.timing:('OPERATION_END_ELAPSED', 54.307, ('PLOTTING', {'Signals': ['Ozone']}))
 INFO:pyaf.timing:('OPERATION_START', ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12}))
-INFO:pyaf.timing:('OPERATION_END_ELAPSED', 1.995, ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12}))
+INFO:pyaf.timing:('OPERATION_END_ELAPSED', 1.203, ('FORECASTING', {'Signals': ['Ozone'], 'Horizon': 12}))
      Month  Ozone       Time
 0  1955-01    2.7 1955-01-01
 1  1955-02    2.0 1955-02-01
@@ -111,14 +104,20 @@ Forecasts
 <ModelInfo>
 {
     "Ozone": {
+        "Complexity": {
+            "AR": "S",
+            "Cycle": "S",
+            "Decomposition": "S",
+            "Transformation": "S",
+            "Trend": "S"
+        },
         "Dataset": {
             "Signal": "Ozone",
             "Time": {
                 "Horizon": 12,
-                "TimeMinMax": [
-                    "1955-01-01 00:00:00",
-                    "1971-12-01 00:00:00"
-                ],
+                "TimeDelta": "<DateOffset: months=1>",
+                "TimeMax": "1971-12-01 00:00:00",
+                "TimeMin": "1955-01-01 00:00:00",
                 "TimeVariable": "Time"
             },
             "Training_Signal_Length": 204
@@ -132,27 +131,50 @@ Forecasts
             "Trend": "ConstantTrend"
         },
         "Model_Performance": {
-            "AUC": 0.2051,
-            "COMPLEXITY": {
-                "AR": "S",
-                "Cycle": "S",
-                "Decomposition": "S",
-                "Transformation": "S",
-                "Trend": "S"
+            "1": {
+                "AUC": 0.2051,
+                "DiffSMAPE": 0.3613,
+                "ErrorMean": 0.9271,
+                "ErrorStdDev": 1.0844,
+                "KS": 0.7949,
+                "KendallTau": NaN,
+                "Length": 39,
+                "LnQ": 9.2692,
+                "MAE": 1.2105,
+                "MAPE": 0.5344,
+                "MASE": 1.5593,
+                "MannWhitneyU": 312.0,
+                "MedAE": 1.1092,
+                "Pearson": 0.0,
+                "R2": -0.731,
+                "RMSE": 1.4267,
+                "SMAPE": 0.3669,
+                "Signal": "Ozone_Forecast_1"
             },
-            "CRPS": 0.2519,
-            "KS": 0.7949,
-            "KendallTau": NaN,
-            "LnQ": 9.2692,
-            "MAE": 1.2105,
-            "MAPE": 0.5344,
-            "MASE": 1.5593,
-            "MannWhitneyU": 312.0,
-            "MedAE": 1.1092,
-            "RMSE": 1.4267
-        }
+            "12": {
+                "AUC": 0.2051,
+                "DiffSMAPE": 0.3613,
+                "ErrorMean": 0.9271,
+                "ErrorStdDev": 1.0844,
+                "KS": 0.7949,
+                "KendallTau": NaN,
+                "Length": 39,
+                "LnQ": 9.2692,
+                "MAE": 1.2105,
+                "MAPE": 0.5344,
+                "MASE": 1.5593,
+                "MannWhitneyU": 312.0,
+                "MedAE": 1.1092,
+                "Pearson": 0.0,
+                "R2": -0.731,
+                "RMSE": 1.4267,
+                "SMAPE": 0.3669,
+                "Signal": "Ozone_Forecast_12"
+            }
+        },
+        "Model_Selection_Criterion": "MAPE"
     },
-    "Training_Time": 13.803
+    "Training_Time": 14.105
 }
 </ModelInfo>