Chronos: Delete existing tags and replace with new Installation optio…

…ns (#5165) * remove skipif tf2 * delete skip_onnxrt tags * fix synatx error * add installation options * remove invalid options * fix known issues * fix known issues * rollback tf1 uts * rollback tf1 UTs * rollback invalid changes Co-authored-by: fuckyouNoob <[email protected]>
intel-analytics · Sep 2, 2022 · 1fffc81 · 1fffc81
1 parent ee7f694
commit 1fffc81
Show file tree

Hide file tree

Showing 12 changed files with 105 additions and 119 deletions.
diff --git a/python/chronos/dev/test/run-pytests-onnxrt16.sh b/python/chronos/dev/test/run-pytests-onnxrt16.sh
@@ -27,43 +27,9 @@ fi
 
 ray stop -f
 
-echo "Running chronos tests TF1 and Deprecated API"
-python -m pytest -v test/bigdl/chronos/autots/test_autotsestimator.py::TestAutoTrainer::test_fit_lstm_feature \
-                    test/bigdl/chronos/autots/test_autotsestimator.py::TestAutoTrainer::test_fit_tcn_feature \
-                    test/bigdl/chronos/autots/test_autotsestimator.py::TestAutoTrainer::test_fit_seq2seq_feature \
-                    test/bigdl/chronos/autots/test_tspipeline.py::TestTSPipeline::test_seq2seq_tsppl_support_dataloader \
-                    test/bigdl/chronos/autots/test_tspipeline.py::TestTSPipeline::test_tsppl_quantize_data_creator \
-                    test/bigdl/chronos/autots/model/test_auto_lstm.py::TestAutoLSTM::test_onnx_methods \
-                    test/bigdl/chronos/autots/model/test_auto_lstm.py::TestAutoLSTM::test_save_load \
-                    test/bigdl/chronos/autots/model/test_auto_tcn.py::TestAutoTCN::test_onnx_methods \
-                    test/bigdl/chronos/autots/model/test_auto_tcn.py::TestAutoTCN::test_save_load \
-                    test/bigdl/chronos/autots/model/test_auto_seq2seq.py::TestAutoSeq2Seq::test_onnx_methods \
-                    test/bigdl/chronos/autots/model/test_auto_seq2seq.py::TestAutoSeq2Seq::test_save_load \
-                    test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_onnx_methods \
-                    test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_quantization_onnx \
-                    test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_quantization_onnx_tuning \
-                    test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_distributed \
-                    test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_fit_loader \
-                    test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_forecaster_from_tsdataset_data_loader_onnx \
-                    test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_s2s_forecaster_onnx_methods \
-                    test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_s2s_forecaster_distributed \
-                    test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_s2s_forecaster_fit_loader \
-                    test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_forecaster_from_tsdataset_data_loader_onnx \
-                    test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_onnx_methods \
-                    test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_quantization_onnx \
-                    test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_quantization_onnx_tuning \
-                    test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_distributed \
-                    test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_fit_loader \
-                    test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_forecaster_from_tsdataset_data_loader_onnx \
-                    test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_multi_objective_tune_acceleration \
-                    test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_mo_tune_acceleration_fit_input \
-                    test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_mo_tune_acceleration_fit \
-                    test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_onnx_methods \
-                    test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_quantization_onnx \
-                    test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_quantization_onnx_tuning \
-                    test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_fit_loader \
-                    test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_distributed \
-                    test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_forecaster_from_tsdataset_data_loader_onnx
+echo "Running chronos tests onnxrt16"
+python -m pytest -v -m "onnxrt16" test/bigdl/chronos/forecaster \
+                                  test/bigdl/chronos/autots
 
 exit_status_0=$?
 if [ $exit_status_0 -ne 0 ];

diff --git a/python/chronos/dev/test/run-pytests.sh b/python/chronos/dev/test/run-pytests.sh
@@ -43,10 +43,10 @@ fi
 
 if [ $RUN_PART1 = 1 ]; then
 echo "Running chronos tests Part 1"
-python -m pytest -v test/bigdl/chronos/model \
-                    test/bigdl/chronos/forecaster \
-                    test/bigdl/chronos/metric \
-                    test/bigdl/chronos/pytorch \
+python -m pytest -v -m "not onnxrt16" test/bigdl/chronos/model \
+                                      test/bigdl/chronos/forecaster \
+                                      test/bigdl/chronos/metric \
+                                      test/bigdl/chronos/pytorch \
        -k "not test_forecast_tcmf_distributed"
 exit_status_0=$?
 if [ $exit_status_0 -ne 0 ];
@@ -57,10 +57,10 @@ fi
 
 if [ $RUN_PART2 = 1 ]; then
 echo "Running chronos tests Part 2"
-python -m pytest -v test/bigdl/chronos/autots\
-                    test/bigdl/chronos/data \
-                    test/bigdl/chronos/simulator \
-                    test/bigdl/chronos/detector \
+python -m pytest -v -m "not onnxrt16" test/bigdl/chronos/autots\
+                                      test/bigdl/chronos/data \
+                                      test/bigdl/chronos/simulator \
+                                      test/bigdl/chronos/detector \
         -k "not test_ae_fit_score_unrolled"
 exit_status_0=$?
 if [ $exit_status_0 -ne 0 ];

diff --git a/python/chronos/test/bigdl/chronos/__init__.py b/python/chronos/test/bigdl/chronos/__init__.py
@@ -13,3 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import pytest
+
+# framework
+op_torch = pytest.mark.torch
+op_tf2 = pytest.mark.tf2
+
+# distribution and automl
+op_automl = pytest.mark.automl
+op_distributed = pytest.mark.distributed
+
+# other mark
+op_all = pytest.mark.all
+op_onnxrt16 = pytest.mark.onnxrt16
diff --git a/python/chronos/test/bigdl/chronos/autots/model/test_auto_lstm.py b/python/chronos/test/bigdl/chronos/autots/model/test_auto_lstm.py
@@ -20,10 +20,8 @@
 from unittest import TestCase
 import pytest
 import tempfile
-import onnxruntime
 
-_onnxrt_ver = onnxruntime.__version__ != '1.6.0' #  Jenkins requires 1.6.0(chronos)
-skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0")
+from ... import op_all, op_onnxrt16
 
 from bigdl.chronos.autots.model.auto_lstm import AutoLSTM
 from bigdl.orca.automl import hp
@@ -144,7 +142,8 @@ def test_predict_evaluation(self):
         auto_lstm.predict(test_data_x)
         auto_lstm.evaluate((test_data_x, test_data_y))
 
-    @skip_onnxrt
+    @op_all
+    @op_onnxrt16
     def test_onnx_methods(self):
         auto_lstm = get_auto_estimator()
         auto_lstm.fit(data=train_dataloader_creator(config={"batch_size": 64}),
@@ -164,7 +163,8 @@ def test_onnx_methods(self):
         except ImportError:
             pass
 
-    @skip_onnxrt
+    @op_all
+    @op_onnxrt16
     def test_save_load(self):
         auto_lstm = get_auto_estimator()
         auto_lstm.fit(data=train_dataloader_creator(config={"batch_size": 64}),

diff --git a/python/chronos/test/bigdl/chronos/autots/model/test_auto_seq2seq.py b/python/chronos/test/bigdl/chronos/autots/model/test_auto_seq2seq.py
@@ -20,10 +20,8 @@
 from unittest import TestCase
 import pytest
 import tempfile
-import onnxruntime
 
-_onnxrt_ver = onnxruntime.__version__ != '1.6.0' #  Jenkins requires 1.6.0(chronos)
-skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0")
+from ... import op_all, op_onnxrt16
 
 from bigdl.chronos.autots.model.auto_seq2seq import AutoSeq2Seq
 from bigdl.orca.automl import hp
@@ -152,7 +150,8 @@ def test_predict_evaluation(self):
         auto_seq2seq.predict(test_data_x)
         auto_seq2seq.evaluate((test_data_x, test_data_y))
 
-    @skip_onnxrt
+    @op_all
+    @op_onnxrt16
     def test_onnx_methods(self):
         auto_seq2seq = get_auto_estimator()
         auto_seq2seq.fit(data=train_dataloader_creator(config={"batch_size": 64}),
@@ -172,7 +171,8 @@ def test_onnx_methods(self):
         except ImportError:
             pass
 
-    @skip_onnxrt
+    @op_all
+    @op_onnxrt16
     def test_save_load(self):
         auto_seq2seq = get_auto_estimator()
         auto_seq2seq.fit(data=train_dataloader_creator(config={"batch_size": 64}),

diff --git a/python/chronos/test/bigdl/chronos/autots/model/test_auto_tcn.py b/python/chronos/test/bigdl/chronos/autots/model/test_auto_tcn.py
@@ -20,10 +20,8 @@
 from unittest import TestCase
 import pytest
 import tempfile
-import onnxruntime
 
-_onnxrt_ver = onnxruntime.__version__ != '1.6.0' #  Jenkins requires 1.6.0(chronos)
-skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0")
+from ... import op_all, op_onnxrt16
 
 from bigdl.chronos.autots.model.auto_tcn import AutoTCN
 from bigdl.orca.automl import hp
@@ -187,7 +185,8 @@ def test_predict_evaluation(self):
         auto_tcn.predict(test_data_x)
         auto_tcn.evaluate((test_data_x, test_data_y))
 
-    @skip_onnxrt
+    @op_all
+    @op_onnxrt16
     def test_onnx_methods(self):
         auto_tcn = get_auto_estimator()
         auto_tcn.fit(data=train_dataloader_creator(config={"batch_size": 64}),
@@ -207,7 +206,8 @@ def test_onnx_methods(self):
         except ImportError:
             pass
 
-    @skip_onnxrt
+    @op_all
+    @op_onnxrt16
     def test_save_load(self):
         auto_tcn = get_auto_estimator()
         auto_tcn.fit(data=train_dataloader_creator(config={"batch_size": 64}),

diff --git a/python/chronos/test/bigdl/chronos/autots/test_autotsestimator.py b/python/chronos/test/bigdl/chronos/autots/test_autotsestimator.py
@@ -26,11 +26,8 @@
 from bigdl.orca.automl import hp
 import pandas as pd
 import tensorflow as tf
-import onnxruntime
-
-_onnxrt_ver = onnxruntime.__version__ != '1.6.0' #  Jenkins requires 1.6.0(chronos)
-skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0")
 
+from .. import op_all, op_onnxrt16
 
 def get_ts_df():
     sample_num = np.random.randint(100, 200)
@@ -311,7 +308,8 @@ def customized_metric(y_true, y_pred):
         best_model = auto_estimator._get_best_automl_model()
         assert 4 <= best_config["past_seq_len"] <= 6
 
-    @skip_onnxrt
+    @op_all
+    @op_onnxrt16
     def test_fit_lstm_feature(self):
         from sklearn.preprocessing import StandardScaler
         scaler = StandardScaler()
@@ -378,7 +376,8 @@ def test_fit_lstm_feature(self):
         # use tspipeline to incrementally train
         new_ts_pipeline.fit(tsdata_valid)
 
-    @skip_onnxrt
+    @op_all
+    @op_onnxrt16
     def test_fit_tcn_feature(self):
         from sklearn.preprocessing import StandardScaler
         scaler = StandardScaler()
@@ -446,7 +445,8 @@ def test_fit_tcn_feature(self):
         # use tspipeline to incrementally train
         new_ts_pipeline.fit(tsdata_valid)
 
-    @skip_onnxrt
+    @op_all
+    @op_onnxrt16
     def test_fit_seq2seq_feature(self):
         from sklearn.preprocessing import StandardScaler
         scaler = StandardScaler()

diff --git a/python/chronos/test/bigdl/chronos/autots/test_tspipeline.py b/python/chronos/test/bigdl/chronos/autots/test_tspipeline.py
@@ -26,11 +26,7 @@
 from bigdl.chronos.autots import TSPipeline
 from bigdl.chronos.data import TSDataset
 
-import onnxruntime
-
-_onnxrt_ver = onnxruntime.__version__ != '1.6.0' #  Jenkins requires 1.6.0(chronos)
-skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0")
-
+from .. import op_all, op_onnxrt16
 
 def train_data_creator(config):
     return DataLoader(TensorDataset(torch.randn(1000,
@@ -76,7 +72,8 @@ def setUp(self) -> None:
     def tearDown(self) -> None:
         pass
 
-    @skip_onnxrt
+    @op_all
+    @op_onnxrt16
     def test_seq2seq_tsppl_support_dataloader(self):
         # load
         tsppl_seq2seq = TSPipeline.load(
@@ -180,7 +177,8 @@ def test_tsppl_mixed_data_type_usage(self):
         with pytest.raises(RuntimeError):
             yhat = tsppl_lstm.predict(data=get_test_tsdataset(), batch_size=16)
 
-    @skip_onnxrt
+    @op_all
+    @op_onnxrt16
     def test_tsppl_quantize_data_creator(self):
         # s2s not support quantize
         with pytest.raises(RuntimeError):

diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_lstm_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_lstm_forecaster.py
@@ -22,11 +22,7 @@
 from bigdl.chronos.forecaster.lstm_forecaster import LSTMForecaster
 from unittest import TestCase
 import pytest
-import onnxruntime
-
-_onnxrt_ver = onnxruntime.__version__ != '1.6.0' #  Jenkins requires 1.6.0(chronos)
-skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0")
-
+from .. import op_all, op_onnxrt16
 
 def create_data(loader=False):
     num_train_samples = 1000
@@ -103,7 +99,8 @@ def test_lstm_forecaster_fit_eva_pred(self):
         test_mse = forecaster.evaluate(test_data)
         assert test_mse[0].shape == test_data[1].shape[1:]
 
-    @skip_onnxrt
+    @op_all
+    @op_onnxrt16
     def test_lstm_forecaster_fit_loader(self):
         train_loader, val_loader, test_loader = create_data(loader=True)
         forecaster = LSTMForecaster(past_seq_len=24,
@@ -127,7 +124,8 @@ def test_lstm_forecaster_fit_loader(self):
         forecaster.evaluate_with_onnx(test_loader)
         forecaster.evaluate_with_onnx(test_loader, batch_size=32, quantize=True)
 
-    @skip_onnxrt
+    @op_all
+    @op_onnxrt16
     def test_lstm_forecaster_onnx_methods(self):
         train_data, val_data, test_data = create_data()
         forecaster = LSTMForecaster(past_seq_len=24,
@@ -204,7 +202,8 @@ def test_lstm_forecaster_quantization(self):
         np.testing.assert_almost_equal(test_pred_save, test_pred_load)
         np.testing.assert_almost_equal(test_pred_save_q, test_pred_load_q)
 
-    @skip_onnxrt
+    @op_all
+    @op_onnxrt16  
     def test_lstm_forecaster_quantization_onnx(self):
         train_data, val_data, test_data = create_data()
         forecaster = LSTMForecaster(past_seq_len=24,
@@ -218,7 +217,8 @@ def test_lstm_forecaster_quantization_onnx(self):
         pred_q = forecaster.predict_with_onnx(test_data[0], quantize=True)
         eval_q = forecaster.evaluate_with_onnx(test_data, quantize=True)
 
-    @skip_onnxrt
+    @op_all
+    @op_onnxrt16
     def test_lstm_forecaster_quantization_onnx_tuning(self):
         train_data, val_data, test_data = create_data()
         forecaster = LSTMForecaster(past_seq_len=24,
@@ -307,7 +307,8 @@ def transform_to_dict_x(data):
             distributed_eval = forecaster.evaluate(val_data)
         stop_orca_context()
 
-    @skip_onnxrt
+    @op_all
+    @op_onnxrt16
     def test_lstm_forecaster_distributed(self):
         from bigdl.orca import init_orca_context, stop_orca_context
         train_data, val_data, test_data = create_data()
@@ -447,7 +448,8 @@ def test_forecaster_from_tsdataset(self):
         _, y_test = test.to_numpy()
         assert yhat.shape == y_test.shape
 
-    @skip_onnxrt
+    @op_all
+    @op_onnxrt16
     def test_forecaster_from_tsdataset_data_loader_onnx(self):
         train, test = create_tsdataset(roll=False)
         train.gen_dt_feature(one_hot_features=['WEEK'])