From c7cc0d6d36bfd8510622e0936d56612ebfd66f0c Mon Sep 17 00:00:00 2001 From: fuckyouNoob Date: Sat, 6 Aug 2022 07:38:37 +0800 Subject: [PATCH 01/10] remove skipif tf2 --- .../dev/test/run-pytests-horovod-pytorch.sh | 2 +- .../chronos/dev/test/run-pytests-onnxrt16.sh | 39 +------------------ python/chronos/src/bigdl/chronos/__init__.py | 3 ++ .../chronos/autots/model/test_auto_lstm.py | 2 + .../chronos/autots/model/test_auto_seq2seq.py | 2 + .../chronos/autots/model/test_auto_tcn.py | 2 + .../chronos/autots/test_autotsestimator.py | 3 ++ .../bigdl/chronos/autots/test_tspipeline.py | 2 + .../forecaster/test_lstm_forecaster.py | 6 +++ .../forecaster/test_nbeats_forecaster.py | 6 +++ .../forecaster/test_seq2seq_forecaster.py | 4 ++ .../chronos/forecaster/test_tcn_forecaster.py | 6 +++ .../chronos/model/tf2/test_mtnet_keras.py | 1 + 13 files changed, 40 insertions(+), 38 deletions(-) diff --git a/python/chronos/dev/test/run-pytests-horovod-pytorch.sh b/python/chronos/dev/test/run-pytests-horovod-pytorch.sh index 853cdd63030..465094598f9 100755 --- a/python/chronos/dev/test/run-pytests-horovod-pytorch.sh +++ b/python/chronos/dev/test/run-pytests-horovod-pytorch.sh @@ -28,4 +28,4 @@ ray stop -f cd ../../ echo "Running chronos tcmf distributed test" -python -m pytest -v test/bigdl/chronos/forecaster/test_tcmf_forecaster.py::TestChronosModelTCMFForecaster::test_forecast_tcmf_distributed +python -m pytest -v test/bigdl/chronos/forecaster/test_tcmf_forecaster.py::TestChronosModelTCMFForecaster::test_forecast_tcmf_distributed \ No newline at end of file diff --git a/python/chronos/dev/test/run-pytests-onnxrt16.sh b/python/chronos/dev/test/run-pytests-onnxrt16.sh index 4545257bea6..2d6ca4d0f1c 100644 --- a/python/chronos/dev/test/run-pytests-onnxrt16.sh +++ b/python/chronos/dev/test/run-pytests-onnxrt16.sh @@ -27,43 +27,8 @@ fi ray stop -f -echo "Running chronos tests TF1 and Deprecated API" -python -m pytest -v test/bigdl/chronos/autots/test_autotsestimator.py::TestAutoTrainer::test_fit_lstm_feature \ - test/bigdl/chronos/autots/test_autotsestimator.py::TestAutoTrainer::test_fit_tcn_feature \ - test/bigdl/chronos/autots/test_autotsestimator.py::TestAutoTrainer::test_fit_seq2seq_feature \ - test/bigdl/chronos/autots/test_tspipeline.py::TestTSPipeline::test_seq2seq_tsppl_support_dataloader \ - test/bigdl/chronos/autots/test_tspipeline.py::TestTSPipeline::test_tsppl_quantize_data_creator \ - test/bigdl/chronos/autots/model/test_auto_lstm.py::TestAutoLSTM::test_onnx_methods \ - test/bigdl/chronos/autots/model/test_auto_lstm.py::TestAutoLSTM::test_save_load \ - test/bigdl/chronos/autots/model/test_auto_tcn.py::TestAutoTCN::test_onnx_methods \ - test/bigdl/chronos/autots/model/test_auto_tcn.py::TestAutoTCN::test_save_load \ - test/bigdl/chronos/autots/model/test_auto_seq2seq.py::TestAutoSeq2Seq::test_onnx_methods \ - test/bigdl/chronos/autots/model/test_auto_seq2seq.py::TestAutoSeq2Seq::test_save_load \ - test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_onnx_methods \ - test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_quantization_onnx \ - test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_quantization_onnx_tuning \ - test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_distributed \ - test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_fit_loader \ - test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_forecaster_from_tsdataset_data_loader_onnx \ - test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_s2s_forecaster_onnx_methods \ - test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_s2s_forecaster_distributed \ - test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_s2s_forecaster_fit_loader \ - test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_forecaster_from_tsdataset_data_loader_onnx \ - test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_onnx_methods \ - test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_quantization_onnx \ - test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_quantization_onnx_tuning \ - test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_distributed \ - test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_fit_loader \ - test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_forecaster_from_tsdataset_data_loader_onnx \ - test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_multi_objective_tune_acceleration \ - test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_mo_tune_acceleration_fit_input \ - test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_mo_tune_acceleration_fit \ - test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_onnx_methods \ - test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_quantization_onnx \ - test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_quantization_onnx_tuning \ - test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_fit_loader \ - test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_distributed \ - test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_forecaster_from_tsdataset_data_loader_onnx +echo "Running chronos tests onnxrt16" +python -m pytest -v -m "skipif and onnxrt16" test/bigdl/chronos/ exit_status_0=$? if [ $exit_status_0 -ne 0 ]; diff --git a/python/chronos/src/bigdl/chronos/__init__.py b/python/chronos/src/bigdl/chronos/__init__.py index 2151a805423..543c9c15e77 100644 --- a/python/chronos/src/bigdl/chronos/__init__.py +++ b/python/chronos/src/bigdl/chronos/__init__.py @@ -13,3 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import pytest +import tensorflow as tf +skip_tf = pytest.mark.skipif(tf.__version__ > '2.0.0', reason="Run only when tf==1.15.0.") diff --git a/python/chronos/test/bigdl/chronos/autots/model/test_auto_lstm.py b/python/chronos/test/bigdl/chronos/autots/model/test_auto_lstm.py index 731d555a7b3..21c0a04f227 100644 --- a/python/chronos/test/bigdl/chronos/autots/model/test_auto_lstm.py +++ b/python/chronos/test/bigdl/chronos/autots/model/test_auto_lstm.py @@ -145,6 +145,7 @@ def test_predict_evaluation(self): auto_lstm.evaluate((test_data_x, test_data_y)) @skip_onnxrt + @pytest.mark.onnxrt16 def test_onnx_methods(self): auto_lstm = get_auto_estimator() auto_lstm.fit(data=train_dataloader_creator(config={"batch_size": 64}), @@ -165,6 +166,7 @@ def test_onnx_methods(self): pass @skip_onnxrt + @pytest.mark.onnxrt16 def test_save_load(self): auto_lstm = get_auto_estimator() auto_lstm.fit(data=train_dataloader_creator(config={"batch_size": 64}), diff --git a/python/chronos/test/bigdl/chronos/autots/model/test_auto_seq2seq.py b/python/chronos/test/bigdl/chronos/autots/model/test_auto_seq2seq.py index 1bedbc90d3d..de0fdf778b4 100644 --- a/python/chronos/test/bigdl/chronos/autots/model/test_auto_seq2seq.py +++ b/python/chronos/test/bigdl/chronos/autots/model/test_auto_seq2seq.py @@ -153,6 +153,7 @@ def test_predict_evaluation(self): auto_seq2seq.evaluate((test_data_x, test_data_y)) @skip_onnxrt + @pytest.mark.onnxrt16 def test_onnx_methods(self): auto_seq2seq = get_auto_estimator() auto_seq2seq.fit(data=train_dataloader_creator(config={"batch_size": 64}), @@ -173,6 +174,7 @@ def test_onnx_methods(self): pass @skip_onnxrt + @pytest.mark.onnxrt16 def test_save_load(self): auto_seq2seq = get_auto_estimator() auto_seq2seq.fit(data=train_dataloader_creator(config={"batch_size": 64}), diff --git a/python/chronos/test/bigdl/chronos/autots/model/test_auto_tcn.py b/python/chronos/test/bigdl/chronos/autots/model/test_auto_tcn.py index 45d74a3c2ad..0693cd6185c 100644 --- a/python/chronos/test/bigdl/chronos/autots/model/test_auto_tcn.py +++ b/python/chronos/test/bigdl/chronos/autots/model/test_auto_tcn.py @@ -188,6 +188,7 @@ def test_predict_evaluation(self): auto_tcn.evaluate((test_data_x, test_data_y)) @skip_onnxrt + @pytest.mark.onnxrt16 def test_onnx_methods(self): auto_tcn = get_auto_estimator() auto_tcn.fit(data=train_dataloader_creator(config={"batch_size": 64}), @@ -208,6 +209,7 @@ def test_onnx_methods(self): pass @skip_onnxrt + @pytest.mark.onnxrt16 def test_save_load(self): auto_tcn = get_auto_estimator() auto_tcn.fit(data=train_dataloader_creator(config={"batch_size": 64}), diff --git a/python/chronos/test/bigdl/chronos/autots/test_autotsestimator.py b/python/chronos/test/bigdl/chronos/autots/test_autotsestimator.py index 039dff206e2..f800749f3d1 100644 --- a/python/chronos/test/bigdl/chronos/autots/test_autotsestimator.py +++ b/python/chronos/test/bigdl/chronos/autots/test_autotsestimator.py @@ -312,6 +312,7 @@ def customized_metric(y_true, y_pred): assert 4 <= best_config["past_seq_len"] <= 6 @skip_onnxrt + @pytest.mark.onnxrt16 def test_fit_lstm_feature(self): from sklearn.preprocessing import StandardScaler scaler = StandardScaler() @@ -379,6 +380,7 @@ def test_fit_lstm_feature(self): new_ts_pipeline.fit(tsdata_valid) @skip_onnxrt + @pytest.mark.onnxrt16 def test_fit_tcn_feature(self): from sklearn.preprocessing import StandardScaler scaler = StandardScaler() @@ -447,6 +449,7 @@ def test_fit_tcn_feature(self): new_ts_pipeline.fit(tsdata_valid) @skip_onnxrt + @pytest.mark.onnxrt16 def test_fit_seq2seq_feature(self): from sklearn.preprocessing import StandardScaler scaler = StandardScaler() diff --git a/python/chronos/test/bigdl/chronos/autots/test_tspipeline.py b/python/chronos/test/bigdl/chronos/autots/test_tspipeline.py index 36a39e85641..a6f9fb97fea 100644 --- a/python/chronos/test/bigdl/chronos/autots/test_tspipeline.py +++ b/python/chronos/test/bigdl/chronos/autots/test_tspipeline.py @@ -77,6 +77,7 @@ def tearDown(self) -> None: pass @skip_onnxrt + @pytest.mark.onnxrt16 def test_seq2seq_tsppl_support_dataloader(self): # load tsppl_seq2seq = TSPipeline.load( @@ -181,6 +182,7 @@ def test_tsppl_mixed_data_type_usage(self): yhat = tsppl_lstm.predict(data=get_test_tsdataset(), batch_size=16) @skip_onnxrt + @pytest.mark.onnxrt16 def test_tsppl_quantize_data_creator(self): # s2s not support quantize with pytest.raises(RuntimeError): diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_lstm_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_lstm_forecaster.py index 59d44c2649e..05324206b7a 100644 --- a/python/chronos/test/bigdl/chronos/forecaster/test_lstm_forecaster.py +++ b/python/chronos/test/bigdl/chronos/forecaster/test_lstm_forecaster.py @@ -104,6 +104,7 @@ def test_lstm_forecaster_fit_eva_pred(self): assert test_mse[0].shape == test_data[1].shape[1:] @skip_onnxrt + @pytest.mark.onnxrt16 def test_lstm_forecaster_fit_loader(self): train_loader, val_loader, test_loader = create_data(loader=True) forecaster = LSTMForecaster(past_seq_len=24, @@ -128,6 +129,7 @@ def test_lstm_forecaster_fit_loader(self): forecaster.evaluate_with_onnx(test_loader, batch_size=32, quantize=True) @skip_onnxrt + @pytest.mark.onnxrt16 def test_lstm_forecaster_onnx_methods(self): train_data, val_data, test_data = create_data() forecaster = LSTMForecaster(past_seq_len=24, @@ -205,6 +207,7 @@ def test_lstm_forecaster_quantization(self): np.testing.assert_almost_equal(test_pred_save_q, test_pred_load_q) @skip_onnxrt + @pytest.mark.onnxrt16 def test_lstm_forecaster_quantization_onnx(self): train_data, val_data, test_data = create_data() forecaster = LSTMForecaster(past_seq_len=24, @@ -219,6 +222,7 @@ def test_lstm_forecaster_quantization_onnx(self): eval_q = forecaster.evaluate_with_onnx(test_data, quantize=True) @skip_onnxrt + @pytest.mark.onnxrt16 def test_lstm_forecaster_quantization_onnx_tuning(self): train_data, val_data, test_data = create_data() forecaster = LSTMForecaster(past_seq_len=24, @@ -308,6 +312,7 @@ def transform_to_dict_x(data): stop_orca_context() @skip_onnxrt + @pytest.mark.onnxrt16 def test_lstm_forecaster_distributed(self): from bigdl.orca import init_orca_context, stop_orca_context train_data, val_data, test_data = create_data() @@ -448,6 +453,7 @@ def test_forecaster_from_tsdataset(self): assert yhat.shape == y_test.shape @skip_onnxrt + @pytest.mark.onnxrt16 def test_forecaster_from_tsdataset_data_loader_onnx(self): train, test = create_tsdataset(roll=False) train.gen_dt_feature(one_hot_features=['WEEK']) diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_nbeats_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_nbeats_forecaster.py index b5f8812ae68..852c14463bf 100644 --- a/python/chronos/test/bigdl/chronos/forecaster/test_nbeats_forecaster.py +++ b/python/chronos/test/bigdl/chronos/forecaster/test_nbeats_forecaster.py @@ -102,6 +102,7 @@ def test_nbeats_forecaster_fit_pred_eva(self): assert eva[0].shape == test_data[1].shape[1:] @skip_onnxrt + @pytest.mark.onnxrt16 def test_nbeats_forecaster_fit_loader(self): train_loader, val_loader, test_loader = create_data(loader=True) forecaster = NBeatsForecaster(past_seq_len=24, @@ -123,6 +124,7 @@ def test_nbeats_forecaster_fit_loader(self): forecaster.evaluate_with_onnx(test_loader, batch_size=32, quantize=True) @skip_onnxrt + @pytest.mark.onnxrt16 def test_nbeats_forecaster_onnx_methods(self): train_data, val_data, test_data = create_data() forecaster = NBeatsForecaster(past_seq_len=24, @@ -199,6 +201,7 @@ def test_nbeats_forecaster_quantization_tuning(self): np.testing.assert_almost_equal(test_pred_save_q, test_pred_load_q) @skip_onnxrt + @pytest.mark.onnxrt16 def test_nbeats_forecaster_quantization_onnx(self): train_data, val_data, test_data = create_data() forecaster = NBeatsForecaster(past_seq_len=24, @@ -212,6 +215,7 @@ def test_nbeats_forecaster_quantization_onnx(self): eval_q = forecaster.evaluate_with_onnx(test_data, quantize=True) @skip_onnxrt + @pytest.mark.onnxrt16 def test_nbeats_forecaster_quantization_onnx_tuning(self): train_data, val_data, test_data = create_data() forecaster = NBeatsForecaster(past_seq_len=24, @@ -289,6 +293,7 @@ def transform_to_dict_x(data): stop_orca_context() @skip_onnxrt + @pytest.mark.onnxrt16 def test_nbeats_forecaster_distributed(self): train_data, val_data, test_data = create_data() _train_loader, _, _test_loader = create_data(loader=True) @@ -434,6 +439,7 @@ def test_forecaster_from_tsdataset(self): assert yhat.shape == y_test.shape @skip_onnxrt + @pytest.mark.onnxrt16 def test_forecaster_from_tsdataset_data_loader_onnx(self): train, test = create_tsdataset(roll=False) loader = train.to_torch_data_loader(roll=True, diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py index ed70a870238..cbd23357f30 100644 --- a/python/chronos/test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py +++ b/python/chronos/test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py @@ -102,6 +102,7 @@ def test_s2s_forecaster_fit_eva_pred(self): assert test_mse[0].shape == test_data[1].shape[1:] @skip_onnxrt + @pytest.mark.onnxrt16 def test_s2s_forecaster_fit_loader(self): train_loader, val_loader, test_loader = create_data(loader=True) forecaster = Seq2SeqForecaster(past_seq_len=24, @@ -120,6 +121,7 @@ def test_s2s_forecaster_fit_loader(self): @skip_onnxrt + @pytest.mark.onnxrt16 def test_s2s_forecaster_onnx_methods(self): train_data, val_data, test_data = create_data() forecaster = Seq2SeqForecaster(past_seq_len=24, @@ -250,6 +252,7 @@ def transform_to_dict_x(data): stop_orca_context() @skip_onnxrt + @pytest.mark.onnxrt16 def test_s2s_forecaster_distributed(self): from bigdl.orca import init_orca_context, stop_orca_context train_data, val_data, test_data = create_data() @@ -391,6 +394,7 @@ def test_forecaster_from_tsdataset(self): assert yhat.shape == y_test.shape @skip_onnxrt + @pytest.mark.onnxrt16 def test_forecaster_from_tsdataset_data_loader_onnx(self): train, test = create_tsdataset(roll=False) train.gen_dt_feature(one_hot_features=['WEEK']) diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py index aa5245f3c5a..2a333473a3a 100644 --- a/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py +++ b/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py @@ -105,6 +105,7 @@ def test_tcn_forecaster_fit_eva_pred(self): assert test_mse[0].shape == test_data[1].shape[1:] @skip_onnxrt + @pytest.mark.onnxrt16 def test_tcn_forecaster_fit_loader(self): train_loader, val_loader, test_loader = create_data(loader=True) forecaster = TCNForecaster(past_seq_len=24, @@ -227,6 +228,7 @@ def test_tcn_forecaster_mo_tune_acceleration_fit(self): forecaster.fit(train_data, epochs=2, use_trial_id=0) @skip_onnxrt + @pytest.mark.onnxrt16 def test_tcn_forecaster_onnx_methods(self): train_data, val_data, test_data = create_data() forecaster = TCNForecaster(past_seq_len=24, @@ -328,6 +330,7 @@ def test_tcn_forecaster_quantization(self): np.testing.assert_almost_equal(test_pred_save_q, test_pred_load_q) @skip_onnxrt + @pytest.mark.onnxrt16 def test_tcn_forecaster_quantization_onnx(self): train_data, val_data, test_data = create_data() forecaster = TCNForecaster(past_seq_len=24, @@ -344,6 +347,7 @@ def test_tcn_forecaster_quantization_onnx(self): eval_q = forecaster.evaluate_with_onnx(test_data, quantize=True) @skip_onnxrt + @pytest.mark.onnxrt16 def test_tcn_forecaster_quantization_onnx_tuning(self): train_data, val_data, test_data = create_data() forecaster = TCNForecaster(past_seq_len=24, @@ -440,6 +444,7 @@ def transform_to_dict_x(data): stop_orca_context() @skip_onnxrt + @pytest.mark.onnxrt16 def test_tcn_forecaster_distributed(self): from bigdl.orca import init_orca_context, stop_orca_context train_data, val_data, test_data = create_data() @@ -584,6 +589,7 @@ def test_forecaster_from_tsdataset(self): assert yhat.shape == y_test.shape @skip_onnxrt + @pytest.mark.onnxrt16 def test_forecaster_from_tsdataset_data_loader_onnx(self): train, test = create_tsdataset(roll=False) train.gen_dt_feature(one_hot_features=['WEEK']) diff --git a/python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py b/python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py index dea6ca7cf2b..6d468ac769b 100644 --- a/python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py +++ b/python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py @@ -24,6 +24,7 @@ import numpy as np import tensorflow as tf from numpy.testing import assert_array_almost_equal +from ... import op_tf2 def create_data(): From 442870532c14cdb53aa2b91cc2add19b4a5fca0c Mon Sep 17 00:00:00 2001 From: liangs6212 Date: Fri, 12 Aug 2022 07:14:54 +0800 Subject: [PATCH 02/10] delete skip_onnxrt tags --- .../chronos/dev/test/run-pytests-onnxrt16.sh | 2 +- python/chronos/src/bigdl/chronos/__init__.py | 3 -- python/chronos/test/bigdl/chronos/__init__.py | 13 +++++++ .../chronos/autots/model/test_auto_lstm.py | 12 +++---- .../chronos/autots/model/test_auto_seq2seq.py | 12 +++---- .../chronos/autots/model/test_auto_tcn.py | 12 +++---- .../chronos/autots/test_autotsestimator.py | 17 ++++----- .../bigdl/chronos/autots/test_tspipeline.py | 14 +++----- .../forecaster/test_lstm_forecaster.py | 30 +++++++--------- .../forecaster/test_nbeats_forecaster.py | 30 +++++++--------- .../forecaster/test_seq2seq_forecaster.py | 22 +++++------- .../chronos/forecaster/test_tcn_forecaster.py | 36 +++++++++---------- .../chronos/model/tf2/test_mtnet_keras.py | 4 +-- 13 files changed, 94 insertions(+), 113 deletions(-) diff --git a/python/chronos/dev/test/run-pytests-onnxrt16.sh b/python/chronos/dev/test/run-pytests-onnxrt16.sh index 2d6ca4d0f1c..3d92921c122 100644 --- a/python/chronos/dev/test/run-pytests-onnxrt16.sh +++ b/python/chronos/dev/test/run-pytests-onnxrt16.sh @@ -28,7 +28,7 @@ fi ray stop -f echo "Running chronos tests onnxrt16" -python -m pytest -v -m "skipif and onnxrt16" test/bigdl/chronos/ +python -m pytest -v -m "all and onnxrt16" test/bigdl/chronos/ exit_status_0=$? if [ $exit_status_0 -ne 0 ]; diff --git a/python/chronos/src/bigdl/chronos/__init__.py b/python/chronos/src/bigdl/chronos/__init__.py index 543c9c15e77..2151a805423 100644 --- a/python/chronos/src/bigdl/chronos/__init__.py +++ b/python/chronos/src/bigdl/chronos/__init__.py @@ -13,6 +13,3 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import pytest -import tensorflow as tf -skip_tf = pytest.mark.skipif(tf.__version__ > '2.0.0', reason="Run only when tf==1.15.0.") diff --git a/python/chronos/test/bigdl/chronos/__init__.py b/python/chronos/test/bigdl/chronos/__init__.py index 2151a805423..aa96d7f1ee1 100644 --- a/python/chronos/test/bigdl/chronos/__init__.py +++ b/python/chronos/test/bigdl/chronos/__init__.py @@ -13,3 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import pytest + +# framework +op_torch = pytest.mark.torch +op_tf2 = pytest.mark.tf2 + +# distribution and automl +op_automl = pytest.mark.automlop_distributed = pytest.mark.distributed + +# other mark +op_tf1 = pytest.mark.tf1 +op_all = pytest.mark.all +op_onnxrt16 = pytest.mark.onnxrt16 diff --git a/python/chronos/test/bigdl/chronos/autots/model/test_auto_lstm.py b/python/chronos/test/bigdl/chronos/autots/model/test_auto_lstm.py index 21c0a04f227..9874106e4c8 100644 --- a/python/chronos/test/bigdl/chronos/autots/model/test_auto_lstm.py +++ b/python/chronos/test/bigdl/chronos/autots/model/test_auto_lstm.py @@ -20,10 +20,8 @@ from unittest import TestCase import pytest import tempfile -import onnxruntime -_onnxrt_ver = onnxruntime.__version__ != '1.6.0' # Jenkins requires 1.6.0(chronos) -skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0") +from ... import op_all, op_onnxrt16 from bigdl.chronos.autots.model.auto_lstm import AutoLSTM from bigdl.orca.automl import hp @@ -144,8 +142,8 @@ def test_predict_evaluation(self): auto_lstm.predict(test_data_x) auto_lstm.evaluate((test_data_x, test_data_y)) - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_onnx_methods(self): auto_lstm = get_auto_estimator() auto_lstm.fit(data=train_dataloader_creator(config={"batch_size": 64}), @@ -165,8 +163,8 @@ def test_onnx_methods(self): except ImportError: pass - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_save_load(self): auto_lstm = get_auto_estimator() auto_lstm.fit(data=train_dataloader_creator(config={"batch_size": 64}), diff --git a/python/chronos/test/bigdl/chronos/autots/model/test_auto_seq2seq.py b/python/chronos/test/bigdl/chronos/autots/model/test_auto_seq2seq.py index de0fdf778b4..960e799f326 100644 --- a/python/chronos/test/bigdl/chronos/autots/model/test_auto_seq2seq.py +++ b/python/chronos/test/bigdl/chronos/autots/model/test_auto_seq2seq.py @@ -20,10 +20,8 @@ from unittest import TestCase import pytest import tempfile -import onnxruntime -_onnxrt_ver = onnxruntime.__version__ != '1.6.0' # Jenkins requires 1.6.0(chronos) -skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0") +from ... import op_all, op_onnxrt16 from bigdl.chronos.autots.model.auto_seq2seq import AutoSeq2Seq from bigdl.orca.automl import hp @@ -152,8 +150,8 @@ def test_predict_evaluation(self): auto_seq2seq.predict(test_data_x) auto_seq2seq.evaluate((test_data_x, test_data_y)) - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_onnx_methods(self): auto_seq2seq = get_auto_estimator() auto_seq2seq.fit(data=train_dataloader_creator(config={"batch_size": 64}), @@ -173,8 +171,8 @@ def test_onnx_methods(self): except ImportError: pass - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_save_load(self): auto_seq2seq = get_auto_estimator() auto_seq2seq.fit(data=train_dataloader_creator(config={"batch_size": 64}), diff --git a/python/chronos/test/bigdl/chronos/autots/model/test_auto_tcn.py b/python/chronos/test/bigdl/chronos/autots/model/test_auto_tcn.py index 0693cd6185c..c89c61b0f7a 100644 --- a/python/chronos/test/bigdl/chronos/autots/model/test_auto_tcn.py +++ b/python/chronos/test/bigdl/chronos/autots/model/test_auto_tcn.py @@ -20,10 +20,8 @@ from unittest import TestCase import pytest import tempfile -import onnxruntime -_onnxrt_ver = onnxruntime.__version__ != '1.6.0' # Jenkins requires 1.6.0(chronos) -skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0") +from ... import op_all, op_onnxrt16 from bigdl.chronos.autots.model.auto_tcn import AutoTCN from bigdl.orca.automl import hp @@ -187,8 +185,8 @@ def test_predict_evaluation(self): auto_tcn.predict(test_data_x) auto_tcn.evaluate((test_data_x, test_data_y)) - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_onnx_methods(self): auto_tcn = get_auto_estimator() auto_tcn.fit(data=train_dataloader_creator(config={"batch_size": 64}), @@ -208,8 +206,8 @@ def test_onnx_methods(self): except ImportError: pass - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_save_load(self): auto_tcn = get_auto_estimator() auto_tcn.fit(data=train_dataloader_creator(config={"batch_size": 64}), diff --git a/python/chronos/test/bigdl/chronos/autots/test_autotsestimator.py b/python/chronos/test/bigdl/chronos/autots/test_autotsestimator.py index f800749f3d1..68a4c1df8d6 100644 --- a/python/chronos/test/bigdl/chronos/autots/test_autotsestimator.py +++ b/python/chronos/test/bigdl/chronos/autots/test_autotsestimator.py @@ -26,11 +26,8 @@ from bigdl.orca.automl import hp import pandas as pd import tensorflow as tf -import onnxruntime - -_onnxrt_ver = onnxruntime.__version__ != '1.6.0' # Jenkins requires 1.6.0(chronos) -skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0") +from .. import op_all, op_onnxrt16 def get_ts_df(): sample_num = np.random.randint(100, 200) @@ -311,8 +308,8 @@ def customized_metric(y_true, y_pred): best_model = auto_estimator._get_best_automl_model() assert 4 <= best_config["past_seq_len"] <= 6 - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_fit_lstm_feature(self): from sklearn.preprocessing import StandardScaler scaler = StandardScaler() @@ -379,8 +376,8 @@ def test_fit_lstm_feature(self): # use tspipeline to incrementally train new_ts_pipeline.fit(tsdata_valid) - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_fit_tcn_feature(self): from sklearn.preprocessing import StandardScaler scaler = StandardScaler() @@ -448,8 +445,8 @@ def test_fit_tcn_feature(self): # use tspipeline to incrementally train new_ts_pipeline.fit(tsdata_valid) - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_fit_seq2seq_feature(self): from sklearn.preprocessing import StandardScaler scaler = StandardScaler() diff --git a/python/chronos/test/bigdl/chronos/autots/test_tspipeline.py b/python/chronos/test/bigdl/chronos/autots/test_tspipeline.py index a6f9fb97fea..a89e52a3403 100644 --- a/python/chronos/test/bigdl/chronos/autots/test_tspipeline.py +++ b/python/chronos/test/bigdl/chronos/autots/test_tspipeline.py @@ -26,11 +26,7 @@ from bigdl.chronos.autots import TSPipeline from bigdl.chronos.data import TSDataset -import onnxruntime - -_onnxrt_ver = onnxruntime.__version__ != '1.6.0' # Jenkins requires 1.6.0(chronos) -skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0") - +from .. import op_all, op_onnxrt16 def train_data_creator(config): return DataLoader(TensorDataset(torch.randn(1000, @@ -76,8 +72,8 @@ def setUp(self) -> None: def tearDown(self) -> None: pass - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_seq2seq_tsppl_support_dataloader(self): # load tsppl_seq2seq = TSPipeline.load( @@ -181,8 +177,8 @@ def test_tsppl_mixed_data_type_usage(self): with pytest.raises(RuntimeError): yhat = tsppl_lstm.predict(data=get_test_tsdataset(), batch_size=16) - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_tsppl_quantize_data_creator(self): # s2s not support quantize with pytest.raises(RuntimeError): diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_lstm_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_lstm_forecaster.py index 05324206b7a..0780d94ec18 100644 --- a/python/chronos/test/bigdl/chronos/forecaster/test_lstm_forecaster.py +++ b/python/chronos/test/bigdl/chronos/forecaster/test_lstm_forecaster.py @@ -22,11 +22,7 @@ from bigdl.chronos.forecaster.lstm_forecaster import LSTMForecaster from unittest import TestCase import pytest -import onnxruntime - -_onnxrt_ver = onnxruntime.__version__ != '1.6.0' # Jenkins requires 1.6.0(chronos) -skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0") - +from .. import op_all, op_onnxrt16 def create_data(loader=False): num_train_samples = 1000 @@ -103,8 +99,8 @@ def test_lstm_forecaster_fit_eva_pred(self): test_mse = forecaster.evaluate(test_data) assert test_mse[0].shape == test_data[1].shape[1:] - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_lstm_forecaster_fit_loader(self): train_loader, val_loader, test_loader = create_data(loader=True) forecaster = LSTMForecaster(past_seq_len=24, @@ -128,8 +124,8 @@ def test_lstm_forecaster_fit_loader(self): forecaster.evaluate_with_onnx(test_loader) forecaster.evaluate_with_onnx(test_loader, batch_size=32, quantize=True) - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_lstm_forecaster_onnx_methods(self): train_data, val_data, test_data = create_data() forecaster = LSTMForecaster(past_seq_len=24, @@ -206,8 +202,8 @@ def test_lstm_forecaster_quantization(self): np.testing.assert_almost_equal(test_pred_save, test_pred_load) np.testing.assert_almost_equal(test_pred_save_q, test_pred_load_q) - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_lstm_forecaster_quantization_onnx(self): train_data, val_data, test_data = create_data() forecaster = LSTMForecaster(past_seq_len=24, @@ -221,8 +217,8 @@ def test_lstm_forecaster_quantization_onnx(self): pred_q = forecaster.predict_with_onnx(test_data[0], quantize=True) eval_q = forecaster.evaluate_with_onnx(test_data, quantize=True) - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_lstm_forecaster_quantization_onnx_tuning(self): train_data, val_data, test_data = create_data() forecaster = LSTMForecaster(past_seq_len=24, @@ -311,8 +307,8 @@ def transform_to_dict_x(data): distributed_eval = forecaster.evaluate(val_data) stop_orca_context() - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_lstm_forecaster_distributed(self): from bigdl.orca import init_orca_context, stop_orca_context train_data, val_data, test_data = create_data() @@ -452,8 +448,8 @@ def test_forecaster_from_tsdataset(self): _, y_test = test.to_numpy() assert yhat.shape == y_test.shape - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_forecaster_from_tsdataset_data_loader_onnx(self): train, test = create_tsdataset(roll=False) train.gen_dt_feature(one_hot_features=['WEEK']) diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_nbeats_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_nbeats_forecaster.py index 852c14463bf..91b8d27d2b1 100644 --- a/python/chronos/test/bigdl/chronos/forecaster/test_nbeats_forecaster.py +++ b/python/chronos/test/bigdl/chronos/forecaster/test_nbeats_forecaster.py @@ -22,11 +22,7 @@ import pytest from bigdl.chronos.forecaster.nbeats_forecaster import NBeatsForecaster -import onnxruntime - -_onnxrt_ver = onnxruntime.__version__ != '1.6.0' # Jenkins requires 1.6.0(chronos) -skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0") - +from .. import op_all, op_onnxrt16 def create_data(loader=False): num_train_samples = 1000 @@ -101,8 +97,8 @@ def test_nbeats_forecaster_fit_pred_eva(self): eva = forecaster.evaluate(test_data) assert eva[0].shape == test_data[1].shape[1:] - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_nbeats_forecaster_fit_loader(self): train_loader, val_loader, test_loader = create_data(loader=True) forecaster = NBeatsForecaster(past_seq_len=24, @@ -123,8 +119,8 @@ def test_nbeats_forecaster_fit_loader(self): forecaster.evaluate_with_onnx(test_loader) forecaster.evaluate_with_onnx(test_loader, batch_size=32, quantize=True) - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_nbeats_forecaster_onnx_methods(self): train_data, val_data, test_data = create_data() forecaster = NBeatsForecaster(past_seq_len=24, @@ -200,8 +196,8 @@ def test_nbeats_forecaster_quantization_tuning(self): np.testing.assert_almost_equal(test_pred_save, test_pred_load) np.testing.assert_almost_equal(test_pred_save_q, test_pred_load_q) - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_nbeats_forecaster_quantization_onnx(self): train_data, val_data, test_data = create_data() forecaster = NBeatsForecaster(past_seq_len=24, @@ -214,8 +210,8 @@ def test_nbeats_forecaster_quantization_onnx(self): pred_q = forecaster.predict_with_onnx(test_data[0], quantize=True) eval_q = forecaster.evaluate_with_onnx(test_data, quantize=True) - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_nbeats_forecaster_quantization_onnx_tuning(self): train_data, val_data, test_data = create_data() forecaster = NBeatsForecaster(past_seq_len=24, @@ -292,8 +288,8 @@ def transform_to_dict_x(data): distributed_eval = forecaster.evaluate(val_data) stop_orca_context() - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_nbeats_forecaster_distributed(self): train_data, val_data, test_data = create_data() _train_loader, _, _test_loader = create_data(loader=True) @@ -438,8 +434,8 @@ def test_forecaster_from_tsdataset(self): _, y_test = test.to_numpy() assert yhat.shape == y_test.shape - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_forecaster_from_tsdataset_data_loader_onnx(self): train, test = create_tsdataset(roll=False) loader = train.to_torch_data_loader(roll=True, diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py index cbd23357f30..30ba19b2660 100644 --- a/python/chronos/test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py +++ b/python/chronos/test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py @@ -22,11 +22,8 @@ from bigdl.chronos.forecaster.seq2seq_forecaster import Seq2SeqForecaster from unittest import TestCase import pytest -import onnxruntime - -_onnxrt_ver = onnxruntime.__version__ != '1.6.0' # Jenkins requires 1.6.0(chronos) -skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0") +from .. import op_all, op_onnxrt16 def create_data(loader=False): num_train_samples = 1000 @@ -101,8 +98,8 @@ def test_s2s_forecaster_fit_eva_pred(self): test_mse = forecaster.evaluate(test_data) assert test_mse[0].shape == test_data[1].shape[1:] - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_s2s_forecaster_fit_loader(self): train_loader, val_loader, test_loader = create_data(loader=True) forecaster = Seq2SeqForecaster(past_seq_len=24, @@ -119,9 +116,8 @@ def test_s2s_forecaster_fit_loader(self): forecaster.evaluate_with_onnx(test_loader) forecaster.evaluate_with_onnx(test_loader, batch_size=32) - - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_s2s_forecaster_onnx_methods(self): train_data, val_data, test_data = create_data() forecaster = Seq2SeqForecaster(past_seq_len=24, @@ -251,8 +247,8 @@ def transform_to_dict_x(data): distributed_eval = forecaster.evaluate(val_data) stop_orca_context() - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_s2s_forecaster_distributed(self): from bigdl.orca import init_orca_context, stop_orca_context train_data, val_data, test_data = create_data() @@ -393,8 +389,8 @@ def test_forecaster_from_tsdataset(self): _, y_test = test.to_numpy() assert yhat.shape == y_test.shape - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_forecaster_from_tsdataset_data_loader_onnx(self): train, test = create_tsdataset(roll=False) train.gen_dt_feature(one_hot_features=['WEEK']) diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py index 2a333473a3a..14df0945cb4 100644 --- a/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py +++ b/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py @@ -22,11 +22,7 @@ from bigdl.chronos.forecaster.tcn_forecaster import TCNForecaster from unittest import TestCase import pytest -import onnxruntime - -_onnxrt_ver = onnxruntime.__version__ != '1.6.0' # Jenkins requires 1.6.0(chronos) -skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0") - +from .. import op_all, op_onnxrt16 def create_data(loader=False): num_train_samples = 1000 @@ -104,8 +100,8 @@ def test_tcn_forecaster_fit_eva_pred(self): test_mse = forecaster.evaluate(test_data) assert test_mse[0].shape == test_data[1].shape[1:] - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_tcn_forecaster_fit_loader(self): train_loader, val_loader, test_loader = create_data(loader=True) forecaster = TCNForecaster(past_seq_len=24, @@ -167,7 +163,7 @@ def test_tcn_forecaster_multi_objective_tune(self): direction=None, directions=["minimize", "minimize"]) - @skip_onnxrt + @op_all def test_tcn_forecaster_multi_objective_tune_acceleration(self): import bigdl.nano.automl.hpo.space as space train_data, val_data, _ = create_data(loader=False) @@ -186,7 +182,7 @@ def test_tcn_forecaster_multi_objective_tune_acceleration(self): directions=["minimize", "minimize"], acceleration=True, direction=None) - @skip_onnxrt + @op_all def test_tcn_forecaster_mo_tune_acceleration_fit_input(self): import bigdl.nano.automl.hpo.space as space train_data, val_data, _ = create_data(loader=False) @@ -207,7 +203,7 @@ def test_tcn_forecaster_mo_tune_acceleration_fit_input(self): with self.assertRaises(Exception): forecaster.fit(train_data, epochs=2) - @skip_onnxrt + @op_all def test_tcn_forecaster_mo_tune_acceleration_fit(self): import bigdl.nano.automl.hpo.space as space train_data, val_data, _ = create_data(loader=False) @@ -227,8 +223,8 @@ def test_tcn_forecaster_mo_tune_acceleration_fit(self): acceleration=True, direction=None) forecaster.fit(train_data, epochs=2, use_trial_id=0) - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_tcn_forecaster_onnx_methods(self): train_data, val_data, test_data = create_data() forecaster = TCNForecaster(past_seq_len=24, @@ -329,8 +325,8 @@ def test_tcn_forecaster_quantization(self): np.testing.assert_almost_equal(test_pred_save, test_pred_load) np.testing.assert_almost_equal(test_pred_save_q, test_pred_load_q) - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_tcn_forecaster_quantization_onnx(self): train_data, val_data, test_data = create_data() forecaster = TCNForecaster(past_seq_len=24, @@ -346,8 +342,8 @@ def test_tcn_forecaster_quantization_onnx(self): pred_q = forecaster.predict_with_onnx(test_data[0], quantize=True) eval_q = forecaster.evaluate_with_onnx(test_data, quantize=True) - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_tcn_forecaster_quantization_onnx_tuning(self): train_data, val_data, test_data = create_data() forecaster = TCNForecaster(past_seq_len=24, @@ -443,8 +439,8 @@ def transform_to_dict_x(data): distributed_eval = forecaster.evaluate(val_data) stop_orca_context() - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_tcn_forecaster_distributed(self): from bigdl.orca import init_orca_context, stop_orca_context train_data, val_data, test_data = create_data() @@ -588,8 +584,8 @@ def test_forecaster_from_tsdataset(self): _, y_test = test.to_numpy() assert yhat.shape == y_test.shape - @skip_onnxrt - @pytest.mark.onnxrt16 + @op_all + @op_onnxrt16 def test_forecaster_from_tsdataset_data_loader_onnx(self): train, test = create_tsdataset(roll=False) train.gen_dt_feature(one_hot_features=['WEEK']) diff --git a/python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py b/python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py index 6d468ac769b..bfeaf22a298 100644 --- a/python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py +++ b/python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py @@ -22,9 +22,8 @@ from bigdl.chronos.data import TSDataset import pandas as pd import numpy as np -import tensorflow as tf from numpy.testing import assert_array_almost_equal -from ... import op_tf2 +import tensorflow as tf def create_data(): @@ -57,6 +56,7 @@ def get_data(num_samples): class TestMTNetKeras(TestCase): def setup_method(self, method): + import tensorflow as tf tf.keras.backend.clear_session() train_data, test_data = create_data() self.x_train, y_train = train_data.to_numpy() From bd09b0a2807ac840a06c83dd7f31149538a23986 Mon Sep 17 00:00:00 2001 From: liangs6212 Date: Fri, 12 Aug 2022 07:16:19 +0800 Subject: [PATCH 03/10] fix synatx error --- python/chronos/test/bigdl/chronos/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/chronos/test/bigdl/chronos/__init__.py b/python/chronos/test/bigdl/chronos/__init__.py index aa96d7f1ee1..5d7f8a72db2 100644 --- a/python/chronos/test/bigdl/chronos/__init__.py +++ b/python/chronos/test/bigdl/chronos/__init__.py @@ -20,7 +20,8 @@ op_tf2 = pytest.mark.tf2 # distribution and automl -op_automl = pytest.mark.automlop_distributed = pytest.mark.distributed +op_automl = pytest.mark.automl +op_distributed = pytest.mark.distributed # other mark op_tf1 = pytest.mark.tf1 From 75241dba5dcd97aaeb61ef7f0e0864af80a837b0 Mon Sep 17 00:00:00 2001 From: liangs6212 Date: Mon, 15 Aug 2022 18:22:53 +0800 Subject: [PATCH 04/10] add installation options --- python/chronos/dev/test/run-pytests.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python/chronos/dev/test/run-pytests.sh b/python/chronos/dev/test/run-pytests.sh index 3c9fa5061b1..888f583383a 100755 --- a/python/chronos/dev/test/run-pytests.sh +++ b/python/chronos/dev/test/run-pytests.sh @@ -43,10 +43,10 @@ fi if [ $RUN_PART1 = 1 ]; then echo "Running chronos tests Part 1" -python -m pytest -v test/bigdl/chronos/model \ - test/bigdl/chronos/forecaster \ - test/bigdl/chronos/metric \ - test/bigdl/chronos/pytorch \ +python -m pytest -v -m "not tf1 and not onnxrt16" test/bigdl/chronos/model \ + test/bigdl/chronos/forecaster \ + test/bigdl/chronos/metric \ + test/bigdl/chronos/pytorch \ -k "not test_forecast_tcmf_distributed" exit_status_0=$? if [ $exit_status_0 -ne 0 ]; @@ -57,10 +57,10 @@ fi if [ $RUN_PART2 = 1 ]; then echo "Running chronos tests Part 2" -python -m pytest -v test/bigdl/chronos/autots\ - test/bigdl/chronos/data \ - test/bigdl/chronos/simulator \ - test/bigdl/chronos/detector \ +python -m pytest -v -m "not tf1 and not onnxrt16" test/bigdl/chronos/autots\ + test/bigdl/chronos/data \ + test/bigdl/chronos/simulator \ + test/bigdl/chronos/detector \ -k "not test_ae_fit_score_unrolled" exit_status_0=$? if [ $exit_status_0 -ne 0 ]; From fbbae2392e66943c25e7036c749d606ca418c2b8 Mon Sep 17 00:00:00 2001 From: liangs6212 Date: Mon, 15 Aug 2022 23:01:11 +0800 Subject: [PATCH 05/10] remove invalid options --- python/chronos/dev/test/run-pytests-onnxrt16.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/chronos/dev/test/run-pytests-onnxrt16.sh b/python/chronos/dev/test/run-pytests-onnxrt16.sh index 3d92921c122..5a1278d2bff 100644 --- a/python/chronos/dev/test/run-pytests-onnxrt16.sh +++ b/python/chronos/dev/test/run-pytests-onnxrt16.sh @@ -28,7 +28,7 @@ fi ray stop -f echo "Running chronos tests onnxrt16" -python -m pytest -v -m "all and onnxrt16" test/bigdl/chronos/ +python -m pytest -v -m "onnxrt16" test/bigdl/chronos/ exit_status_0=$? if [ $exit_status_0 -ne 0 ]; From 797d1b840e92e1ec2a96efface6369c53bc39a19 Mon Sep 17 00:00:00 2001 From: liangs6212 Date: Tue, 16 Aug 2022 18:43:47 +0800 Subject: [PATCH 06/10] fix known issues --- .../test/bigdl/chronos/forecaster/test_tcn_forecaster.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py b/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py index 14df0945cb4..1aaddc70c55 100644 --- a/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py +++ b/python/chronos/test/bigdl/chronos/forecaster/test_tcn_forecaster.py @@ -164,6 +164,7 @@ def test_tcn_forecaster_multi_objective_tune(self): directions=["minimize", "minimize"]) @op_all + @op_onnxrt16 def test_tcn_forecaster_multi_objective_tune_acceleration(self): import bigdl.nano.automl.hpo.space as space train_data, val_data, _ = create_data(loader=False) @@ -183,6 +184,7 @@ def test_tcn_forecaster_multi_objective_tune_acceleration(self): acceleration=True, direction=None) @op_all + @op_onnxrt16 def test_tcn_forecaster_mo_tune_acceleration_fit_input(self): import bigdl.nano.automl.hpo.space as space train_data, val_data, _ = create_data(loader=False) @@ -204,6 +206,7 @@ def test_tcn_forecaster_mo_tune_acceleration_fit_input(self): forecaster.fit(train_data, epochs=2) @op_all + @op_onnxrt16 def test_tcn_forecaster_mo_tune_acceleration_fit(self): import bigdl.nano.automl.hpo.space as space train_data, val_data, _ = create_data(loader=False) From c7e512eefa39d902d671a57d81fb9bf4eeb7e638 Mon Sep 17 00:00:00 2001 From: liangs6212 Date: Fri, 19 Aug 2022 15:22:05 +0800 Subject: [PATCH 07/10] fix known issues --- python/chronos/dev/test/run-pytests-onnxrt16.sh | 3 ++- python/chronos/dev/test/run-pytests.sh | 4 ++-- python/chronos/test/bigdl/chronos/__init__.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/python/chronos/dev/test/run-pytests-onnxrt16.sh b/python/chronos/dev/test/run-pytests-onnxrt16.sh index 5a1278d2bff..78115697876 100644 --- a/python/chronos/dev/test/run-pytests-onnxrt16.sh +++ b/python/chronos/dev/test/run-pytests-onnxrt16.sh @@ -28,7 +28,8 @@ fi ray stop -f echo "Running chronos tests onnxrt16" -python -m pytest -v -m "onnxrt16" test/bigdl/chronos/ +python -m pytest -v -m "onnxrt16" test/bigdl/chronos/forecaster \ + test/bigdl/chronos/autots exit_status_0=$? if [ $exit_status_0 -ne 0 ]; diff --git a/python/chronos/dev/test/run-pytests.sh b/python/chronos/dev/test/run-pytests.sh index 888f583383a..8e522e1df65 100755 --- a/python/chronos/dev/test/run-pytests.sh +++ b/python/chronos/dev/test/run-pytests.sh @@ -43,7 +43,7 @@ fi if [ $RUN_PART1 = 1 ]; then echo "Running chronos tests Part 1" -python -m pytest -v -m "not tf1 and not onnxrt16" test/bigdl/chronos/model \ +python -m pytest -v -m "not onnxrt16 and not tf1" test/bigdl/chronos/model \ test/bigdl/chronos/forecaster \ test/bigdl/chronos/metric \ test/bigdl/chronos/pytorch \ @@ -57,7 +57,7 @@ fi if [ $RUN_PART2 = 1 ]; then echo "Running chronos tests Part 2" -python -m pytest -v -m "not tf1 and not onnxrt16" test/bigdl/chronos/autots\ +python -m pytest -v -m "not onnxrt16 and not tf1" test/bigdl/chronos/autots\ test/bigdl/chronos/data \ test/bigdl/chronos/simulator \ test/bigdl/chronos/detector \ diff --git a/python/chronos/test/bigdl/chronos/__init__.py b/python/chronos/test/bigdl/chronos/__init__.py index 5d7f8a72db2..6bf488c336a 100644 --- a/python/chronos/test/bigdl/chronos/__init__.py +++ b/python/chronos/test/bigdl/chronos/__init__.py @@ -24,6 +24,6 @@ op_distributed = pytest.mark.distributed # other mark -op_tf1 = pytest.mark.tf1 +op_tf1 = pytest.mark.tf1 # will delete later, may be in the last pr. op_all = pytest.mark.all op_onnxrt16 = pytest.mark.onnxrt16 From d22e95c633236e7b7c980f313dfb31c5b1bbe0a5 Mon Sep 17 00:00:00 2001 From: liangs6212 Date: Wed, 31 Aug 2022 09:51:30 +0800 Subject: [PATCH 08/10] rollback tf1 uts --- python/chronos/dev/test/run-pytests.sh | 16 ++++++++-------- python/chronos/test/bigdl/chronos/__init__.py | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/python/chronos/dev/test/run-pytests.sh b/python/chronos/dev/test/run-pytests.sh index 8e522e1df65..e3ea9979b0b 100755 --- a/python/chronos/dev/test/run-pytests.sh +++ b/python/chronos/dev/test/run-pytests.sh @@ -43,10 +43,10 @@ fi if [ $RUN_PART1 = 1 ]; then echo "Running chronos tests Part 1" -python -m pytest -v -m "not onnxrt16 and not tf1" test/bigdl/chronos/model \ - test/bigdl/chronos/forecaster \ - test/bigdl/chronos/metric \ - test/bigdl/chronos/pytorch \ +python -m pytest -v -m "not onnxrt16" test/bigdl/chronos/model \ + test/bigdl/chronos/forecaster \ + test/bigdl/chronos/metric \ + test/bigdl/chronos/pytorch \ -k "not test_forecast_tcmf_distributed" exit_status_0=$? if [ $exit_status_0 -ne 0 ]; @@ -57,10 +57,10 @@ fi if [ $RUN_PART2 = 1 ]; then echo "Running chronos tests Part 2" -python -m pytest -v -m "not onnxrt16 and not tf1" test/bigdl/chronos/autots\ - test/bigdl/chronos/data \ - test/bigdl/chronos/simulator \ - test/bigdl/chronos/detector \ +python -m pytest -v -m "not onnxrt16" test/bigdl/chronos/autots\ + test/bigdl/chronos/data \ + test/bigdl/chronos/simulator \ + test/bigdl/chronos/detector \ -k "not test_ae_fit_score_unrolled" exit_status_0=$? if [ $exit_status_0 -ne 0 ]; diff --git a/python/chronos/test/bigdl/chronos/__init__.py b/python/chronos/test/bigdl/chronos/__init__.py index 6bf488c336a..4e9165b5c71 100644 --- a/python/chronos/test/bigdl/chronos/__init__.py +++ b/python/chronos/test/bigdl/chronos/__init__.py @@ -24,6 +24,5 @@ op_distributed = pytest.mark.distributed # other mark -op_tf1 = pytest.mark.tf1 # will delete later, may be in the last pr. op_all = pytest.mark.all op_onnxrt16 = pytest.mark.onnxrt16 From b3c3ee06abbe39ba7bd785d01491fd1f5376fd54 Mon Sep 17 00:00:00 2001 From: liangs6212 Date: Wed, 31 Aug 2022 10:07:48 +0800 Subject: [PATCH 09/10] rollback tf1 UTs --- python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py b/python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py index bfeaf22a298..4eb04373951 100644 --- a/python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py +++ b/python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py @@ -22,6 +22,7 @@ from bigdl.chronos.data import TSDataset import pandas as pd import numpy as np +import tensorflow as tf from numpy.testing import assert_array_almost_equal import tensorflow as tf @@ -56,7 +57,6 @@ def get_data(num_samples): class TestMTNetKeras(TestCase): def setup_method(self, method): - import tensorflow as tf tf.keras.backend.clear_session() train_data, test_data = create_data() self.x_train, y_train = train_data.to_numpy() From afab0e90b00f5dc40afdb4e5cfc5b353fb8e357f Mon Sep 17 00:00:00 2001 From: liangs6212 Date: Thu, 1 Sep 2022 10:19:34 +0800 Subject: [PATCH 10/10] rollback invalid changes --- python/chronos/dev/test/run-pytests-horovod-pytorch.sh | 2 +- python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/python/chronos/dev/test/run-pytests-horovod-pytorch.sh b/python/chronos/dev/test/run-pytests-horovod-pytorch.sh index 465094598f9..853cdd63030 100755 --- a/python/chronos/dev/test/run-pytests-horovod-pytorch.sh +++ b/python/chronos/dev/test/run-pytests-horovod-pytorch.sh @@ -28,4 +28,4 @@ ray stop -f cd ../../ echo "Running chronos tcmf distributed test" -python -m pytest -v test/bigdl/chronos/forecaster/test_tcmf_forecaster.py::TestChronosModelTCMFForecaster::test_forecast_tcmf_distributed \ No newline at end of file +python -m pytest -v test/bigdl/chronos/forecaster/test_tcmf_forecaster.py::TestChronosModelTCMFForecaster::test_forecast_tcmf_distributed diff --git a/python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py b/python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py index 4eb04373951..dea6ca7cf2b 100644 --- a/python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py +++ b/python/chronos/test/bigdl/chronos/model/tf2/test_mtnet_keras.py @@ -24,7 +24,6 @@ import numpy as np import tensorflow as tf from numpy.testing import assert_array_almost_equal -import tensorflow as tf def create_data():