From 63492472d0601d34f220452101539ad804fa7dc2 Mon Sep 17 00:00:00 2001 From: liangs6212 Date: Fri, 22 Jul 2022 04:17:08 +0800 Subject: [PATCH] Optimized uts script --- .../chronos/dev/test/run-pytests-onnxrt16.sh | 36 ++----------------- python/chronos/dev/test/run-pytests-tf1.sh | 2 +- .../chronos/autots/model/test_auto_lstm.py | 2 ++ .../chronos/autots/model/test_auto_seq2seq.py | 2 ++ .../chronos/autots/model/test_auto_tcn.py | 2 ++ .../chronos/autots/test_autotsestimator.py | 3 ++ .../bigdl/chronos/autots/test_tspipeline.py | 2 ++ .../forecaster/test_lstm_forecaster.py | 6 ++++ .../forecaster/test_nbeats_forecaster.py | 6 ++++ .../forecaster/test_seq2seq_forecaster.py | 4 +++ .../chronos/forecaster/test_tcn_forecaster.py | 6 ++++ .../chronos/model/tf1/test_Seq2Seq_keras.py | 1 + .../model/tf1/test_VanillaLSTM_keras.py | 1 + .../chronos/model/tf1/test_mtnet_keras.py | 1 + 14 files changed, 39 insertions(+), 35 deletions(-) diff --git a/python/chronos/dev/test/run-pytests-onnxrt16.sh b/python/chronos/dev/test/run-pytests-onnxrt16.sh index 67b59df5df4a..2d6ca4d0f1cb 100644 --- a/python/chronos/dev/test/run-pytests-onnxrt16.sh +++ b/python/chronos/dev/test/run-pytests-onnxrt16.sh @@ -27,40 +27,8 @@ fi ray stop -f -echo "Running chronos tests TF1 and Deprecated API" -python -m pytest -v test/bigdl/chronos/autots/test_autotsestimator.py::TestAutoTrainer::test_fit_lstm_feature \ - test/bigdl/chronos/autots/test_autotsestimator.py::TestAutoTrainer::test_fit_tcn_feature \ - test/bigdl/chronos/autots/test_autotsestimator.py::TestAutoTrainer::test_fit_seq2seq_feature \ - test/bigdl/chronos/autots/test_tspipeline.py::TestTSPipeline::test_seq2seq_tsppl_support_dataloader \ - test/bigdl/chronos/autots/test_tspipeline.py::TestTSPipeline::test_tsppl_quantize_data_creator \ - test/bigdl/chronos/autots/model/test_auto_lstm.py::TestAutoLSTM::test_onnx_methods \ - test/bigdl/chronos/autots/model/test_auto_lstm.py::TestAutoLSTM::test_save_load \ - test/bigdl/chronos/autots/model/test_auto_tcn.py::TestAutoTCN::test_onnx_methods \ - test/bigdl/chronos/autots/model/test_auto_tcn.py::TestAutoTCN::test_save_load \ - test/bigdl/chronos/autots/model/test_auto_seq2seq.py::TestAutoSeq2Seq::test_onnx_methods \ - test/bigdl/chronos/autots/model/test_auto_seq2seq.py::TestAutoSeq2Seq::test_save_load \ - test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_onnx_methods \ - test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_quantization_onnx \ - test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_quantization_onnx_tuning \ - test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_distributed \ - test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_fit_loader \ - test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_forecaster_from_tsdataset_data_loader_onnx \ - test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_s2s_forecaster_onnx_methods \ - test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_s2s_forecaster_distributed \ - test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_s2s_forecaster_fit_loader \ - test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_forecaster_from_tsdataset_data_loader_onnx \ - test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_onnx_methods \ - test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_quantization_onnx \ - test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_quantization_onnx_tuning \ - test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_distributed \ - test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_fit_loader \ - test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_forecaster_from_tsdataset_data_loader_onnx \ - test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_onnx_methods \ - test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_quantization_onnx \ - test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_quantization_onnx_tuning \ - test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_fit_loader \ - test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_distributed \ - test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_forecaster_from_tsdataset_data_loader_onnx +echo "Running chronos tests onnxrt16" +python -m pytest -v -m "skipif and onnxrt16" test/bigdl/chronos/ exit_status_0=$? if [ $exit_status_0 -ne 0 ]; diff --git a/python/chronos/dev/test/run-pytests-tf1.sh b/python/chronos/dev/test/run-pytests-tf1.sh index 545318ff1f38..72bc242f7432 100755 --- a/python/chronos/dev/test/run-pytests-tf1.sh +++ b/python/chronos/dev/test/run-pytests-tf1.sh @@ -28,7 +28,7 @@ fi ray stop -f echo "Running chronos tests TF1 and Deprecated API" -python -m pytest -v test/bigdl/chronos/model/tf1 +python -m pytest -v -m "skipif and tf1" test/bigdl/chronos/model/ exit_status_0=$? if [ $exit_status_0 -ne 0 ]; diff --git a/python/chronos/test/bigdl/chronos/autots/model/test_auto_lstm.py b/python/chronos/test/bigdl/chronos/autots/model/test_auto_lstm.py index 731d555a7b3a..21c0a04f2279 100644 --- a/python/chronos/test/bigdl/chronos/autots/model/test_auto_lstm.py +++ b/python/chronos/test/bigdl/chronos/autots/model/test_auto_lstm.py @@ -145,6 +145,7 @@ def test_predict_evaluation(self): auto_lstm.evaluate((test_data_x, test_data_y)) @skip_onnxrt + @pytest.mark.onnxrt16 def test_onnx_methods(self): auto_lstm = get_auto_estimator() auto_lstm.fit(data=train_dataloader_creator(config={"batch_size": 64}), @@ -165,6 +166,7 @@ def test_onnx_methods(self): pass @skip_onnxrt + @pytest.mark.onnxrt16 def test_save_load(self): auto_lstm = get_auto_estimator() auto_lstm.fit(data=train_dataloader_creator(config={"batch_size": 64}), diff --git a/python/chronos/test/bigdl/chronos/autots/model/test_auto_seq2seq.py b/python/chronos/test/bigdl/chronos/autots/model/test_auto_seq2seq.py index 1bedbc90d3d7..de0fdf778b41 100644 --- a/python/chronos/test/bigdl/chronos/autots/model/test_auto_seq2seq.py +++ b/python/chronos/test/bigdl/chronos/autots/model/test_auto_seq2seq.py @@ -153,6 +153,7 @@ def test_predict_evaluation(self): auto_seq2seq.evaluate((test_data_x, test_data_y)) @skip_onnxrt + @pytest.mark.onnxrt16 def test_onnx_methods(self): auto_seq2seq = get_auto_estimator() auto_seq2seq.fit(data=train_dataloader_creator(config={"batch_size": 64}), @@ -173,6 +174,7 @@ def test_onnx_methods(self): pass @skip_onnxrt + @pytest.mark.onnxrt16 def test_save_load(self): auto_seq2seq = get_auto_estimator() auto_seq2seq.fit(data=train_dataloader_creator(config={"batch_size": 64}), diff --git a/python/chronos/test/bigdl/chronos/autots/model/test_auto_tcn.py b/python/chronos/test/bigdl/chronos/autots/model/test_auto_tcn.py index 45d74a3c2ad7..0693cd6185c6 100644 --- a/python/chronos/test/bigdl/chronos/autots/model/test_auto_tcn.py +++ b/python/chronos/test/bigdl/chronos/autots/model/test_auto_tcn.py @@ -188,6 +188,7 @@ def test_predict_evaluation(self): auto_tcn.evaluate((test_data_x, test_data_y)) @skip_onnxrt + @pytest.mark.onnxrt16 def test_onnx_methods(self): auto_tcn = get_auto_estimator() auto_tcn.fit(data=train_dataloader_creator(config={"batch_size": 64}), @@ -208,6 +209,7 @@ def test_onnx_methods(self): pass @skip_onnxrt + @pytest.mark.onnxrt16 def test_save_load(self): auto_tcn = get_auto_estimator() auto_tcn.fit(data=train_dataloader_creator(config={"batch_size": 64}), diff --git a/python/chronos/test/bigdl/chronos/autots/test_autotsestimator.py b/python/chronos/test/bigdl/chronos/autots/test_autotsestimator.py index 039dff206e28..f800749f3d13 100644 --- a/python/chronos/test/bigdl/chronos/autots/test_autotsestimator.py +++ b/python/chronos/test/bigdl/chronos/autots/test_autotsestimator.py @@ -312,6 +312,7 @@ def customized_metric(y_true, y_pred): assert 4 <= best_config["past_seq_len"] <= 6 @skip_onnxrt + @pytest.mark.onnxrt16 def test_fit_lstm_feature(self): from sklearn.preprocessing import StandardScaler scaler = StandardScaler() @@ -379,6 +380,7 @@ def test_fit_lstm_feature(self): new_ts_pipeline.fit(tsdata_valid) @skip_onnxrt + @pytest.mark.onnxrt16 def test_fit_tcn_feature(self): from sklearn.preprocessing import StandardScaler scaler = StandardScaler() @@ -447,6 +449,7 @@ def test_fit_tcn_feature(self): new_ts_pipeline.fit(tsdata_valid) @skip_onnxrt + @pytest.mark.onnxrt16 def test_fit_seq2seq_feature(self): from sklearn.preprocessing import StandardScaler scaler = StandardScaler() diff --git a/python/chronos/test/bigdl/chronos/autots/test_tspipeline.py b/python/chronos/test/bigdl/chronos/autots/test_tspipeline.py index 45f26944b6f8..172dce92a6ba 100644 --- a/python/chronos/test/bigdl/chronos/autots/test_tspipeline.py +++ b/python/chronos/test/bigdl/chronos/autots/test_tspipeline.py @@ -77,6 +77,7 @@ def tearDown(self) -> None: pass @skip_onnxrt + @pytest.mark.onnxrt16 def test_seq2seq_tsppl_support_dataloader(self): # load tsppl_seq2seq = TSPipeline.load( @@ -181,6 +182,7 @@ def test_tsppl_mixed_data_type_usage(self): yhat = tsppl_lstm.predict(data=get_test_tsdataset(), batch_size=16) @skip_onnxrt + @pytest.mark.onnxrt16 def test_tsppl_quantize_data_creator(self): # s2s not support quantize with pytest.raises(RuntimeError): diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_lstm_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_lstm_forecaster.py index 186a725079ec..1fc83c069f6f 100644 --- a/python/chronos/test/bigdl/chronos/forecaster/test_lstm_forecaster.py +++ b/python/chronos/test/bigdl/chronos/forecaster/test_lstm_forecaster.py @@ -104,6 +104,7 @@ def test_lstm_forecaster_fit_eva_pred(self): assert test_mse[0].shape == test_data[1].shape[1:] @skip_onnxrt + @pytest.mark.onnxrt16 def test_lstm_forecaster_fit_loader(self): train_loader, val_loader, test_loader = create_data(loader=True) forecaster = LSTMForecaster(past_seq_len=24, @@ -128,6 +129,7 @@ def test_lstm_forecaster_fit_loader(self): forecaster.evaluate_with_onnx(test_loader, batch_size=32, quantize=True) @skip_onnxrt + @pytest.mark.onnxrt16 def test_lstm_forecaster_onnx_methods(self): train_data, val_data, test_data = create_data() forecaster = LSTMForecaster(past_seq_len=24, @@ -205,6 +207,7 @@ def test_lstm_forecaster_quantization(self): np.testing.assert_almost_equal(test_pred_save_q, test_pred_load_q) @skip_onnxrt + @pytest.mark.onnxrt16 def test_lstm_forecaster_quantization_onnx(self): train_data, val_data, test_data = create_data() forecaster = LSTMForecaster(past_seq_len=24, @@ -219,6 +222,7 @@ def test_lstm_forecaster_quantization_onnx(self): eval_q = forecaster.evaluate_with_onnx(test_data, quantize=True) @skip_onnxrt + @pytest.mark.onnxrt16 def test_lstm_forecaster_quantization_onnx_tuning(self): train_data, val_data, test_data = create_data() forecaster = LSTMForecaster(past_seq_len=24, @@ -308,6 +312,7 @@ def transform_to_dict_x(data): stop_orca_context() @skip_onnxrt + @pytest.mark.onnxrt16 def test_lstm_forecaster_distributed(self): from bigdl.orca import init_orca_context, stop_orca_context train_data, val_data, test_data = create_data() @@ -448,6 +453,7 @@ def test_forecaster_from_tsdataset(self): assert yhat.shape == y_test.shape @skip_onnxrt + @pytest.mark.onnxrt16 def test_forecaster_from_tsdataset_data_loader_onnx(self): train, test = create_tsdataset(roll=False) train.gen_dt_feature(one_hot_features=['WEEK']) diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_nbeats_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_nbeats_forecaster.py index abb0c1f8e87b..3838a80adc40 100644 --- a/python/chronos/test/bigdl/chronos/forecaster/test_nbeats_forecaster.py +++ b/python/chronos/test/bigdl/chronos/forecaster/test_nbeats_forecaster.py @@ -102,6 +102,7 @@ def test_nbeats_forecaster_fit_pred_eva(self): assert eva[0].shape == test_data[1].shape[1:] @skip_onnxrt + @pytest.mark.onnxrt16 def test_nbeats_forecaster_fit_loader(self): train_loader, val_loader, test_loader = create_data(loader=True) forecaster = NBeatsForecaster(past_seq_len=24, @@ -123,6 +124,7 @@ def test_nbeats_forecaster_fit_loader(self): forecaster.evaluate_with_onnx(test_loader, batch_size=32, quantize=True) @skip_onnxrt + @pytest.mark.onnxrt16 def test_nbeats_forecaster_onnx_methods(self): train_data, val_data, test_data = create_data() forecaster = NBeatsForecaster(past_seq_len=24, @@ -199,6 +201,7 @@ def test_nbeats_forecaster_quantization_tuning(self): np.testing.assert_almost_equal(test_pred_save_q, test_pred_load_q) @skip_onnxrt + @pytest.mark.onnxrt16 def test_nbeats_forecaster_quantization_onnx(self): train_data, val_data, test_data = create_data() forecaster = NBeatsForecaster(past_seq_len=24, @@ -212,6 +215,7 @@ def test_nbeats_forecaster_quantization_onnx(self): eval_q = forecaster.evaluate_with_onnx(test_data, quantize=True) @skip_onnxrt + @pytest.mark.onnxrt16 def test_nbeats_forecaster_quantization_onnx_tuning(self): train_data, val_data, test_data = create_data() forecaster = NBeatsForecaster(past_seq_len=24, @@ -289,6 +293,7 @@ def transform_to_dict_x(data): stop_orca_context() @skip_onnxrt + @pytest.mark.onnxrt16 def test_nbeats_forecaster_distributed(self): train_data, val_data, test_data = create_data() _train_loader, _, _test_loader = create_data(loader=True) @@ -434,6 +439,7 @@ def test_forecaster_from_tsdataset(self): assert yhat.shape == y_test.shape @skip_onnxrt + @pytest.mark.onnxrt16 def test_forecaster_from_tsdataset_data_loader_onnx(self): train, test = create_tsdataset(roll=False) loader = train.to_torch_data_loader(roll=True, diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py index d8624123a233..3fbe74b02f9c 100644 --- a/python/chronos/test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py +++ b/python/chronos/test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py @@ -102,6 +102,7 @@ def test_s2s_forecaster_fit_eva_pred(self): assert test_mse[0].shape == test_data[1].shape[1:] @skip_onnxrt + @pytest.mark.onnxrt16 def test_s2s_forecaster_fit_loader(self): train_loader, val_loader, test_loader = create_data(loader=True) forecaster = Seq2SeqForecaster(past_seq_len=24, @@ -120,6 +121,7 @@ def test_s2s_forecaster_fit_loader(self): @skip_onnxrt + @pytest.mark.onnxrt16 def test_s2s_forecaster_onnx_methods(self): train_data, val_data, test_data = create_data() forecaster = Seq2SeqForecaster(past_seq_len=24, @@ -250,6 +252,7 @@ def transform_to_dict_x(data): stop_orca_context() @skip_onnxrt + @pytest.mark.onnxrt16 def test_s2s_forecaster_distributed(self): from bigdl.orca import init_orca_context, stop_orca_context train_data, val_data, test_data = create_data() @@ -391,6 +394,7 @@ def test_forecaster_from_tsdataset(self): assert yhat.shape == y_test.shape @skip_onnxrt + @pytest.mark.onnxrt16 def test_forecaster_from_tsdataset_data_loader_onnx(self): train, test = create_tsdataset(roll=False) train.gen_dt_feature(one_hot_features=['WEEK']) diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py index 16306da60bd9..f763fb60063a 100644 --- a/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py +++ b/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py @@ -105,6 +105,7 @@ def test_tcn_forecaster_fit_eva_pred(self): assert test_mse[0].shape == test_data[1].shape[1:] @skip_onnxrt + @pytest.mark.onnxrt16 def test_tcn_forecaster_fit_loader(self): train_loader, val_loader, test_loader = create_data(loader=True) forecaster = TCNForecaster(past_seq_len=24, @@ -149,6 +150,7 @@ def test_tcn_forecaster_tune(self): assert train_loss > 10 @skip_onnxrt + @pytest.mark.onnxrt16 def test_tcn_forecaster_onnx_methods(self): train_data, val_data, test_data = create_data() forecaster = TCNForecaster(past_seq_len=24, @@ -250,6 +252,7 @@ def test_tcn_forecaster_quantization(self): np.testing.assert_almost_equal(test_pred_save_q, test_pred_load_q) @skip_onnxrt + @pytest.mark.onnxrt16 def test_tcn_forecaster_quantization_onnx(self): train_data, val_data, test_data = create_data() forecaster = TCNForecaster(past_seq_len=24, @@ -266,6 +269,7 @@ def test_tcn_forecaster_quantization_onnx(self): eval_q = forecaster.evaluate_with_onnx(test_data, quantize=True) @skip_onnxrt + @pytest.mark.onnxrt16 def test_tcn_forecaster_quantization_onnx_tuning(self): train_data, val_data, test_data = create_data() forecaster = TCNForecaster(past_seq_len=24, @@ -362,6 +366,7 @@ def transform_to_dict_x(data): stop_orca_context() @skip_onnxrt + @pytest.mark.onnxrt16 def test_tcn_forecaster_distributed(self): from bigdl.orca import init_orca_context, stop_orca_context train_data, val_data, test_data = create_data() @@ -506,6 +511,7 @@ def test_forecaster_from_tsdataset(self): assert yhat.shape == y_test.shape @skip_onnxrt + @pytest.mark.onnxrt16 def test_forecaster_from_tsdataset_data_loader_onnx(self): train, test = create_tsdataset(roll=False) train.gen_dt_feature(one_hot_features=['WEEK']) diff --git a/python/chronos/test/bigdl/chronos/model/tf1/test_Seq2Seq_keras.py b/python/chronos/test/bigdl/chronos/model/tf1/test_Seq2Seq_keras.py index b7f80212858c..3f4450260b29 100644 --- a/python/chronos/test/bigdl/chronos/model/tf1/test_Seq2Seq_keras.py +++ b/python/chronos/test/bigdl/chronos/model/tf1/test_Seq2Seq_keras.py @@ -26,6 +26,7 @@ @pytest.mark.skipif(tf.__version__ > '2.0.0', reason="Run only when tf==1.15.0.") +@pytest.mark.tf1 class TestSeq2Seq(ZooTestCase): def setup_method(self, method): diff --git a/python/chronos/test/bigdl/chronos/model/tf1/test_VanillaLSTM_keras.py b/python/chronos/test/bigdl/chronos/model/tf1/test_VanillaLSTM_keras.py index 236185ef0a48..d38a21c4af1c 100644 --- a/python/chronos/test/bigdl/chronos/model/tf1/test_VanillaLSTM_keras.py +++ b/python/chronos/test/bigdl/chronos/model/tf1/test_VanillaLSTM_keras.py @@ -42,6 +42,7 @@ def get_x_y(num_samples): return train_data, val_data, test_data @pytest.mark.skipif(tf.__version__ > '2.0.0', reason="Run only when tf==1.15.0.") +@pytest.mark.tf1 class TestVanillaLSTM(TestCase): train_data, val_data, test_data = create_data() model = VanillaLSTM() diff --git a/python/chronos/test/bigdl/chronos/model/tf1/test_mtnet_keras.py b/python/chronos/test/bigdl/chronos/model/tf1/test_mtnet_keras.py index f566dfe643eb..7c9c9240a6de 100644 --- a/python/chronos/test/bigdl/chronos/model/tf1/test_mtnet_keras.py +++ b/python/chronos/test/bigdl/chronos/model/tf1/test_mtnet_keras.py @@ -53,6 +53,7 @@ def get_data(num_samples): @pytest.mark.skipif(tf.__version__ > '2.0.0', reason="Run only when tf==1.15.0.") +@pytest.mark.tf1 class TestMTNetKeras(ZooTestCase): def setup_method(self, method):