Optimized uts script

intel-analytics · Jul 22, 2022 · 6349247 · 6349247
1 parent 603c230
commit 6349247
Show file tree

Hide file tree

Showing 14 changed files with 39 additions and 35 deletions.
diff --git a/python/chronos/dev/test/run-pytests-onnxrt16.sh b/python/chronos/dev/test/run-pytests-onnxrt16.sh
@@ -27,40 +27,8 @@ fi
 
 ray stop -f
 
-echo "Running chronos tests TF1 and Deprecated API"
-python -m pytest -v test/bigdl/chronos/autots/test_autotsestimator.py::TestAutoTrainer::test_fit_lstm_feature \
-                    test/bigdl/chronos/autots/test_autotsestimator.py::TestAutoTrainer::test_fit_tcn_feature \
-                    test/bigdl/chronos/autots/test_autotsestimator.py::TestAutoTrainer::test_fit_seq2seq_feature \
-                    test/bigdl/chronos/autots/test_tspipeline.py::TestTSPipeline::test_seq2seq_tsppl_support_dataloader \
-                    test/bigdl/chronos/autots/test_tspipeline.py::TestTSPipeline::test_tsppl_quantize_data_creator \
-                    test/bigdl/chronos/autots/model/test_auto_lstm.py::TestAutoLSTM::test_onnx_methods \
-                    test/bigdl/chronos/autots/model/test_auto_lstm.py::TestAutoLSTM::test_save_load \
-                    test/bigdl/chronos/autots/model/test_auto_tcn.py::TestAutoTCN::test_onnx_methods \
-                    test/bigdl/chronos/autots/model/test_auto_tcn.py::TestAutoTCN::test_save_load \
-                    test/bigdl/chronos/autots/model/test_auto_seq2seq.py::TestAutoSeq2Seq::test_onnx_methods \
-                    test/bigdl/chronos/autots/model/test_auto_seq2seq.py::TestAutoSeq2Seq::test_save_load \
-                    test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_onnx_methods \
-                    test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_quantization_onnx \
-                    test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_quantization_onnx_tuning \
-                    test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_distributed \
-                    test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_fit_loader \
-                    test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_forecaster_from_tsdataset_data_loader_onnx \
-                    test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_s2s_forecaster_onnx_methods \
-                    test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_s2s_forecaster_distributed \
-                    test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_s2s_forecaster_fit_loader \
-                    test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_forecaster_from_tsdataset_data_loader_onnx \
-                    test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_onnx_methods \
-                    test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_quantization_onnx \
-                    test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_quantization_onnx_tuning \
-                    test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_distributed \
-                    test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_fit_loader \
-                    test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_forecaster_from_tsdataset_data_loader_onnx \
-                    test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_onnx_methods \
-                    test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_quantization_onnx \
-                    test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_quantization_onnx_tuning \
-                    test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_fit_loader \
-                    test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_distributed \
-                    test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_forecaster_from_tsdataset_data_loader_onnx
+echo "Running chronos tests onnxrt16"
+python -m pytest -v -m "skipif and onnxrt16" test/bigdl/chronos/
 
 exit_status_0=$?
 if [ $exit_status_0 -ne 0 ];

diff --git a/python/chronos/dev/test/run-pytests-tf1.sh b/python/chronos/dev/test/run-pytests-tf1.sh
@@ -28,7 +28,7 @@ fi
 ray stop -f
 
 echo "Running chronos tests TF1 and Deprecated API"
-python -m pytest -v test/bigdl/chronos/model/tf1
+python -m pytest -v -m "skipif and tf1" test/bigdl/chronos/model/
 
 exit_status_0=$?
 if [ $exit_status_0 -ne 0 ];

diff --git a/python/chronos/test/bigdl/chronos/autots/model/test_auto_lstm.py b/python/chronos/test/bigdl/chronos/autots/model/test_auto_lstm.py
@@ -145,6 +145,7 @@ def test_predict_evaluation(self):
         auto_lstm.evaluate((test_data_x, test_data_y))
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_onnx_methods(self):
         auto_lstm = get_auto_estimator()
         auto_lstm.fit(data=train_dataloader_creator(config={"batch_size": 64}),
@@ -165,6 +166,7 @@ def test_onnx_methods(self):
             pass
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_save_load(self):
         auto_lstm = get_auto_estimator()
         auto_lstm.fit(data=train_dataloader_creator(config={"batch_size": 64}),

diff --git a/python/chronos/test/bigdl/chronos/autots/model/test_auto_seq2seq.py b/python/chronos/test/bigdl/chronos/autots/model/test_auto_seq2seq.py
@@ -153,6 +153,7 @@ def test_predict_evaluation(self):
         auto_seq2seq.evaluate((test_data_x, test_data_y))
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_onnx_methods(self):
         auto_seq2seq = get_auto_estimator()
         auto_seq2seq.fit(data=train_dataloader_creator(config={"batch_size": 64}),
@@ -173,6 +174,7 @@ def test_onnx_methods(self):
             pass
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_save_load(self):
         auto_seq2seq = get_auto_estimator()
         auto_seq2seq.fit(data=train_dataloader_creator(config={"batch_size": 64}),

diff --git a/python/chronos/test/bigdl/chronos/autots/model/test_auto_tcn.py b/python/chronos/test/bigdl/chronos/autots/model/test_auto_tcn.py
@@ -188,6 +188,7 @@ def test_predict_evaluation(self):
         auto_tcn.evaluate((test_data_x, test_data_y))
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_onnx_methods(self):
         auto_tcn = get_auto_estimator()
         auto_tcn.fit(data=train_dataloader_creator(config={"batch_size": 64}),
@@ -208,6 +209,7 @@ def test_onnx_methods(self):
             pass
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_save_load(self):
         auto_tcn = get_auto_estimator()
         auto_tcn.fit(data=train_dataloader_creator(config={"batch_size": 64}),

diff --git a/python/chronos/test/bigdl/chronos/autots/test_autotsestimator.py b/python/chronos/test/bigdl/chronos/autots/test_autotsestimator.py
@@ -312,6 +312,7 @@ def customized_metric(y_true, y_pred):
         assert 4 <= best_config["past_seq_len"] <= 6
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_fit_lstm_feature(self):
         from sklearn.preprocessing import StandardScaler
         scaler = StandardScaler()
@@ -379,6 +380,7 @@ def test_fit_lstm_feature(self):
         new_ts_pipeline.fit(tsdata_valid)
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_fit_tcn_feature(self):
         from sklearn.preprocessing import StandardScaler
         scaler = StandardScaler()
@@ -447,6 +449,7 @@ def test_fit_tcn_feature(self):
         new_ts_pipeline.fit(tsdata_valid)
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_fit_seq2seq_feature(self):
         from sklearn.preprocessing import StandardScaler
         scaler = StandardScaler()

diff --git a/python/chronos/test/bigdl/chronos/autots/test_tspipeline.py b/python/chronos/test/bigdl/chronos/autots/test_tspipeline.py
@@ -77,6 +77,7 @@ def tearDown(self) -> None:
         pass
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_seq2seq_tsppl_support_dataloader(self):
         # load
         tsppl_seq2seq = TSPipeline.load(
@@ -181,6 +182,7 @@ def test_tsppl_mixed_data_type_usage(self):
             yhat = tsppl_lstm.predict(data=get_test_tsdataset(), batch_size=16)
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_tsppl_quantize_data_creator(self):
         # s2s not support quantize
         with pytest.raises(RuntimeError):

diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_lstm_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_lstm_forecaster.py
@@ -104,6 +104,7 @@ def test_lstm_forecaster_fit_eva_pred(self):
         assert test_mse[0].shape == test_data[1].shape[1:]
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_lstm_forecaster_fit_loader(self):
         train_loader, val_loader, test_loader = create_data(loader=True)
         forecaster = LSTMForecaster(past_seq_len=24,
@@ -128,6 +129,7 @@ def test_lstm_forecaster_fit_loader(self):
         forecaster.evaluate_with_onnx(test_loader, batch_size=32, quantize=True)
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_lstm_forecaster_onnx_methods(self):
         train_data, val_data, test_data = create_data()
         forecaster = LSTMForecaster(past_seq_len=24,
@@ -205,6 +207,7 @@ def test_lstm_forecaster_quantization(self):
         np.testing.assert_almost_equal(test_pred_save_q, test_pred_load_q)
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_lstm_forecaster_quantization_onnx(self):
         train_data, val_data, test_data = create_data()
         forecaster = LSTMForecaster(past_seq_len=24,
@@ -219,6 +222,7 @@ def test_lstm_forecaster_quantization_onnx(self):
         eval_q = forecaster.evaluate_with_onnx(test_data, quantize=True)
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_lstm_forecaster_quantization_onnx_tuning(self):
         train_data, val_data, test_data = create_data()
         forecaster = LSTMForecaster(past_seq_len=24,
@@ -308,6 +312,7 @@ def transform_to_dict_x(data):
         stop_orca_context()
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_lstm_forecaster_distributed(self):
         from bigdl.orca import init_orca_context, stop_orca_context
         train_data, val_data, test_data = create_data()
@@ -448,6 +453,7 @@ def test_forecaster_from_tsdataset(self):
         assert yhat.shape == y_test.shape
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_forecaster_from_tsdataset_data_loader_onnx(self):
         train, test = create_tsdataset(roll=False)
         train.gen_dt_feature(one_hot_features=['WEEK'])

diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_nbeats_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_nbeats_forecaster.py
@@ -102,6 +102,7 @@ def test_nbeats_forecaster_fit_pred_eva(self):
         assert eva[0].shape == test_data[1].shape[1:]
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_nbeats_forecaster_fit_loader(self):
         train_loader, val_loader, test_loader = create_data(loader=True)
         forecaster = NBeatsForecaster(past_seq_len=24,
@@ -123,6 +124,7 @@ def test_nbeats_forecaster_fit_loader(self):
         forecaster.evaluate_with_onnx(test_loader, batch_size=32, quantize=True)
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_nbeats_forecaster_onnx_methods(self):
         train_data, val_data, test_data = create_data()
         forecaster = NBeatsForecaster(past_seq_len=24,
@@ -199,6 +201,7 @@ def test_nbeats_forecaster_quantization_tuning(self):
         np.testing.assert_almost_equal(test_pred_save_q, test_pred_load_q)
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_nbeats_forecaster_quantization_onnx(self):
         train_data, val_data, test_data = create_data()
         forecaster = NBeatsForecaster(past_seq_len=24,
@@ -212,6 +215,7 @@ def test_nbeats_forecaster_quantization_onnx(self):
         eval_q = forecaster.evaluate_with_onnx(test_data, quantize=True)
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_nbeats_forecaster_quantization_onnx_tuning(self):
         train_data, val_data, test_data = create_data()
         forecaster = NBeatsForecaster(past_seq_len=24,
@@ -289,6 +293,7 @@ def transform_to_dict_x(data):
         stop_orca_context()
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_nbeats_forecaster_distributed(self):
         train_data, val_data, test_data = create_data()
         _train_loader, _, _test_loader = create_data(loader=True)
@@ -434,6 +439,7 @@ def test_forecaster_from_tsdataset(self):
         assert yhat.shape == y_test.shape
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_forecaster_from_tsdataset_data_loader_onnx(self):
         train, test = create_tsdataset(roll=False)
         loader = train.to_torch_data_loader(roll=True,

diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py
@@ -102,6 +102,7 @@ def test_s2s_forecaster_fit_eva_pred(self):
         assert test_mse[0].shape == test_data[1].shape[1:]
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_s2s_forecaster_fit_loader(self):
         train_loader, val_loader, test_loader = create_data(loader=True)
         forecaster = Seq2SeqForecaster(past_seq_len=24,
@@ -120,6 +121,7 @@ def test_s2s_forecaster_fit_loader(self):
 
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_s2s_forecaster_onnx_methods(self):
         train_data, val_data, test_data = create_data()
         forecaster = Seq2SeqForecaster(past_seq_len=24,
@@ -250,6 +252,7 @@ def transform_to_dict_x(data):
         stop_orca_context()
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_s2s_forecaster_distributed(self):
         from bigdl.orca import init_orca_context, stop_orca_context
         train_data, val_data, test_data = create_data()
@@ -391,6 +394,7 @@ def test_forecaster_from_tsdataset(self):
         assert yhat.shape == y_test.shape
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_forecaster_from_tsdataset_data_loader_onnx(self):
         train, test = create_tsdataset(roll=False)
         train.gen_dt_feature(one_hot_features=['WEEK'])

diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py
@@ -105,6 +105,7 @@ def test_tcn_forecaster_fit_eva_pred(self):
         assert test_mse[0].shape == test_data[1].shape[1:]
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_tcn_forecaster_fit_loader(self):
         train_loader, val_loader, test_loader = create_data(loader=True)
         forecaster = TCNForecaster(past_seq_len=24,
@@ -149,6 +150,7 @@ def test_tcn_forecaster_tune(self):
         assert train_loss > 10
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_tcn_forecaster_onnx_methods(self):
         train_data, val_data, test_data = create_data()
         forecaster = TCNForecaster(past_seq_len=24,
@@ -250,6 +252,7 @@ def test_tcn_forecaster_quantization(self):
         np.testing.assert_almost_equal(test_pred_save_q, test_pred_load_q)
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_tcn_forecaster_quantization_onnx(self):
         train_data, val_data, test_data = create_data()
         forecaster = TCNForecaster(past_seq_len=24,
@@ -266,6 +269,7 @@ def test_tcn_forecaster_quantization_onnx(self):
         eval_q = forecaster.evaluate_with_onnx(test_data, quantize=True)
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_tcn_forecaster_quantization_onnx_tuning(self):
         train_data, val_data, test_data = create_data()
         forecaster = TCNForecaster(past_seq_len=24,
@@ -362,6 +366,7 @@ def transform_to_dict_x(data):
         stop_orca_context()
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_tcn_forecaster_distributed(self):
         from bigdl.orca import init_orca_context, stop_orca_context
         train_data, val_data, test_data = create_data()
@@ -506,6 +511,7 @@ def test_forecaster_from_tsdataset(self):
         assert yhat.shape == y_test.shape
 
     @skip_onnxrt
+    @pytest.mark.onnxrt16
     def test_forecaster_from_tsdataset_data_loader_onnx(self):
         train, test = create_tsdataset(roll=False)
         train.gen_dt_feature(one_hot_features=['WEEK'])

diff --git a/python/chronos/test/bigdl/chronos/model/tf1/test_Seq2Seq_keras.py b/python/chronos/test/bigdl/chronos/model/tf1/test_Seq2Seq_keras.py
@@ -26,6 +26,7 @@
 
 
 @pytest.mark.skipif(tf.__version__ > '2.0.0', reason="Run only when tf==1.15.0.")
+@pytest.mark.tf1
 class TestSeq2Seq(ZooTestCase):
 
     def setup_method(self, method):

diff --git a/python/chronos/test/bigdl/chronos/model/tf1/test_VanillaLSTM_keras.py b/python/chronos/test/bigdl/chronos/model/tf1/test_VanillaLSTM_keras.py
@@ -42,6 +42,7 @@ def get_x_y(num_samples):
     return train_data, val_data, test_data
 
 @pytest.mark.skipif(tf.__version__ > '2.0.0', reason="Run only when tf==1.15.0.")
+@pytest.mark.tf1
 class TestVanillaLSTM(TestCase):
     train_data, val_data, test_data = create_data()
     model = VanillaLSTM()

diff --git a/python/chronos/test/bigdl/chronos/model/tf1/test_mtnet_keras.py b/python/chronos/test/bigdl/chronos/model/tf1/test_mtnet_keras.py
@@ -53,6 +53,7 @@ def get_data(num_samples):
 
 
 @pytest.mark.skipif(tf.__version__ > '2.0.0', reason="Run only when tf==1.15.0.")
+@pytest.mark.tf1
 class TestMTNetKeras(ZooTestCase):
 
     def setup_method(self, method):