Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Chronos: Delete existing tags and replace with new Installation options #5165

Merged
merged 10 commits into from
Sep 2, 2022
Merged

Chronos: Delete existing tags and replace with new Installation options #5165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
c7cc0d6
remove skipif tf2
Cyberpunk1210 Aug 5, 2022
4428705
delete skip_onnxrt tags
liangs6212 Aug 11, 2022
bd09b0a
fix synatx error
liangs6212 Aug 11, 2022
75241db
add installation options
liangs6212 Aug 15, 2022
fbbae23
remove invalid options
liangs6212 Aug 15, 2022
797d1b8
fix known issues
liangs6212 Aug 16, 2022
c7e512e
fix known issues
liangs6212 Aug 19, 2022
d22e95c
rollback tf1 uts
liangs6212 Aug 31, 2022
b3c3ee0
rollback tf1 UTs
liangs6212 Aug 31, 2022
afab0e9
rollback invalid changes
liangs6212 Sep 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 3 additions & 37 deletions python/chronos/dev/test/run-pytests-onnxrt16.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,43 +27,9 @@ fi

ray stop -f

echo "Running chronos tests TF1 and Deprecated API"
python -m pytest -v test/bigdl/chronos/autots/test_autotsestimator.py::TestAutoTrainer::test_fit_lstm_feature \
test/bigdl/chronos/autots/test_autotsestimator.py::TestAutoTrainer::test_fit_tcn_feature \
test/bigdl/chronos/autots/test_autotsestimator.py::TestAutoTrainer::test_fit_seq2seq_feature \
test/bigdl/chronos/autots/test_tspipeline.py::TestTSPipeline::test_seq2seq_tsppl_support_dataloader \
test/bigdl/chronos/autots/test_tspipeline.py::TestTSPipeline::test_tsppl_quantize_data_creator \
test/bigdl/chronos/autots/model/test_auto_lstm.py::TestAutoLSTM::test_onnx_methods \
test/bigdl/chronos/autots/model/test_auto_lstm.py::TestAutoLSTM::test_save_load \
test/bigdl/chronos/autots/model/test_auto_tcn.py::TestAutoTCN::test_onnx_methods \
test/bigdl/chronos/autots/model/test_auto_tcn.py::TestAutoTCN::test_save_load \
test/bigdl/chronos/autots/model/test_auto_seq2seq.py::TestAutoSeq2Seq::test_onnx_methods \
test/bigdl/chronos/autots/model/test_auto_seq2seq.py::TestAutoSeq2Seq::test_save_load \
test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_onnx_methods \
test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_quantization_onnx \
test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_quantization_onnx_tuning \
test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_distributed \
test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_lstm_forecaster_fit_loader \
test/bigdl/chronos/forecaster/test_lstm_forecaster.py::TestChronosModelLSTMForecaster::test_forecaster_from_tsdataset_data_loader_onnx \
test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_s2s_forecaster_onnx_methods \
test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_s2s_forecaster_distributed \
test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_s2s_forecaster_fit_loader \
test/bigdl/chronos/forecaster/test_seq2seq_forecaster.py::TestChronosModelSeq2SeqForecaster::test_forecaster_from_tsdataset_data_loader_onnx \
test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_onnx_methods \
test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_quantization_onnx \
test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_quantization_onnx_tuning \
test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_distributed \
test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_fit_loader \
test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_forecaster_from_tsdataset_data_loader_onnx \
test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_multi_objective_tune_acceleration \
test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_mo_tune_acceleration_fit_input \
test/bigdl/chronos/forecaster/test_tcn_forecaster.py::TestChronosModelTCNForecaster::test_tcn_forecaster_mo_tune_acceleration_fit \
test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_onnx_methods \
test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_quantization_onnx \
test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_quantization_onnx_tuning \
test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_fit_loader \
test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_nbeats_forecaster_distributed \
test/bigdl/chronos/forecaster/test_nbeats_forecaster.py::TestChronosNBeatsForecaster::test_forecaster_from_tsdataset_data_loader_onnx
echo "Running chronos tests onnxrt16"
python -m pytest -v -m "onnxrt16" test/bigdl/chronos/forecaster \
test/bigdl/chronos/autots

exit_status_0=$?
if [ $exit_status_0 -ne 0 ];
Expand Down
16 changes: 8 additions & 8 deletions python/chronos/dev/test/run-pytests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,10 @@ fi

if [ $RUN_PART1 = 1 ]; then
echo "Running chronos tests Part 1"
python -m pytest -v test/bigdl/chronos/model \
test/bigdl/chronos/forecaster \
test/bigdl/chronos/metric \
test/bigdl/chronos/pytorch \
python -m pytest -v -m "not onnxrt16" test/bigdl/chronos/model \
test/bigdl/chronos/forecaster \
test/bigdl/chronos/metric \
test/bigdl/chronos/pytorch \
-k "not test_forecast_tcmf_distributed"
exit_status_0=$?
if [ $exit_status_0 -ne 0 ];
Expand All @@ -57,10 +57,10 @@ fi

if [ $RUN_PART2 = 1 ]; then
echo "Running chronos tests Part 2"
python -m pytest -v test/bigdl/chronos/autots\
test/bigdl/chronos/data \
test/bigdl/chronos/simulator \
test/bigdl/chronos/detector \
python -m pytest -v -m "not onnxrt16" test/bigdl/chronos/autots\
test/bigdl/chronos/data \
test/bigdl/chronos/simulator \
test/bigdl/chronos/detector \
-k "not test_ae_fit_score_unrolled"
exit_status_0=$?
if [ $exit_status_0 -ne 0 ];
Expand Down
13 changes: 13 additions & 0 deletions python/chronos/test/bigdl/chronos/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest

# framework
op_torch = pytest.mark.torch
op_tf2 = pytest.mark.tf2

# distribution and automl
op_automl = pytest.mark.automl
op_distributed = pytest.mark.distributed

# other mark
op_all = pytest.mark.all
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe op_all can be replaced by tag1 or tag2 ... or we may need to give all ut a op_all tag?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we give all UTs op_all tags, then we can move the torch to the ut method, sounds good.
Usage:

  • pytest -m "all" <your_path>/file.py
  • pytest -m "torch and distributed" <your_path>/file.py
  • pytest -m "torch and automl" <your_path>/file.py
  • pytest -m "torch and distributed and not onnxrt16" <your_path>/file.py
@op_all
class Test():

    @op_torch
    @op_automl
    def test_method1():
        pass

    @op_torch
    @op_distributed
    def test_method2():
        pass

     @op_torch
     @op_distributed
     @op_onnxrt16
     def test_method3():
        passs

op_onnxrt16 = pytest.mark.onnxrt16
10 changes: 5 additions & 5 deletions python/chronos/test/bigdl/chronos/autots/model/test_auto_lstm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,8 @@
from unittest import TestCase
import pytest
import tempfile
import onnxruntime

_onnxrt_ver = onnxruntime.__version__ != '1.6.0' # Jenkins requires 1.6.0(chronos)
skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0")
from ... import op_all, op_onnxrt16
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not quite sure about this syntex: from ... import


from bigdl.chronos.autots.model.auto_lstm import AutoLSTM
from bigdl.orca.automl import hp
Expand Down Expand Up @@ -144,7 +142,8 @@ def test_predict_evaluation(self):
auto_lstm.predict(test_data_x)
auto_lstm.evaluate((test_data_x, test_data_y))

@skip_onnxrt
@op_all
@op_onnxrt16
def test_onnx_methods(self):
auto_lstm = get_auto_estimator()
auto_lstm.fit(data=train_dataloader_creator(config={"batch_size": 64}),
Expand All @@ -164,7 +163,8 @@ def test_onnx_methods(self):
except ImportError:
pass

@skip_onnxrt
@op_all
@op_onnxrt16
def test_save_load(self):
auto_lstm = get_auto_estimator()
auto_lstm.fit(data=train_dataloader_creator(config={"batch_size": 64}),
Expand Down
10 changes: 5 additions & 5 deletions python/chronos/test/bigdl/chronos/autots/model/test_auto_seq2seq.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,8 @@
from unittest import TestCase
import pytest
import tempfile
import onnxruntime

_onnxrt_ver = onnxruntime.__version__ != '1.6.0' # Jenkins requires 1.6.0(chronos)
skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0")
from ... import op_all, op_onnxrt16

from bigdl.chronos.autots.model.auto_seq2seq import AutoSeq2Seq
from bigdl.orca.automl import hp
Expand Down Expand Up @@ -152,7 +150,8 @@ def test_predict_evaluation(self):
auto_seq2seq.predict(test_data_x)
auto_seq2seq.evaluate((test_data_x, test_data_y))

@skip_onnxrt
@op_all
@op_onnxrt16
def test_onnx_methods(self):
auto_seq2seq = get_auto_estimator()
auto_seq2seq.fit(data=train_dataloader_creator(config={"batch_size": 64}),
Expand All @@ -172,7 +171,8 @@ def test_onnx_methods(self):
except ImportError:
pass

@skip_onnxrt
@op_all
@op_onnxrt16
def test_save_load(self):
auto_seq2seq = get_auto_estimator()
auto_seq2seq.fit(data=train_dataloader_creator(config={"batch_size": 64}),
Expand Down
10 changes: 5 additions & 5 deletions python/chronos/test/bigdl/chronos/autots/model/test_auto_tcn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,8 @@
from unittest import TestCase
import pytest
import tempfile
import onnxruntime

_onnxrt_ver = onnxruntime.__version__ != '1.6.0' # Jenkins requires 1.6.0(chronos)
skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0")
from ... import op_all, op_onnxrt16

from bigdl.chronos.autots.model.auto_tcn import AutoTCN
from bigdl.orca.automl import hp
Expand Down Expand Up @@ -187,7 +185,8 @@ def test_predict_evaluation(self):
auto_tcn.predict(test_data_x)
auto_tcn.evaluate((test_data_x, test_data_y))

@skip_onnxrt
@op_all
@op_onnxrt16
def test_onnx_methods(self):
auto_tcn = get_auto_estimator()
auto_tcn.fit(data=train_dataloader_creator(config={"batch_size": 64}),
Expand All @@ -207,7 +206,8 @@ def test_onnx_methods(self):
except ImportError:
pass

@skip_onnxrt
@op_all
@op_onnxrt16
def test_save_load(self):
auto_tcn = get_auto_estimator()
auto_tcn.fit(data=train_dataloader_creator(config={"batch_size": 64}),
Expand Down
14 changes: 7 additions & 7 deletions python/chronos/test/bigdl/chronos/autots/test_autotsestimator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,8 @@
from bigdl.orca.automl import hp
import pandas as pd
import tensorflow as tf
import onnxruntime

_onnxrt_ver = onnxruntime.__version__ != '1.6.0' # Jenkins requires 1.6.0(chronos)
skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0")

from .. import op_all, op_onnxrt16

def get_ts_df():
sample_num = np.random.randint(100, 200)
Expand Down Expand Up @@ -311,7 +308,8 @@ def customized_metric(y_true, y_pred):
best_model = auto_estimator._get_best_automl_model()
assert 4 <= best_config["past_seq_len"] <= 6

@skip_onnxrt
@op_all
@op_onnxrt16
def test_fit_lstm_feature(self):
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
Expand Down Expand Up @@ -378,7 +376,8 @@ def test_fit_lstm_feature(self):
# use tspipeline to incrementally train
new_ts_pipeline.fit(tsdata_valid)

@skip_onnxrt
@op_all
@op_onnxrt16
def test_fit_tcn_feature(self):
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
Expand Down Expand Up @@ -446,7 +445,8 @@ def test_fit_tcn_feature(self):
# use tspipeline to incrementally train
new_ts_pipeline.fit(tsdata_valid)

@skip_onnxrt
@op_all
@op_onnxrt16
def test_fit_seq2seq_feature(self):
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
Expand Down
12 changes: 5 additions & 7 deletions python/chronos/test/bigdl/chronos/autots/test_tspipeline.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,7 @@
from bigdl.chronos.autots import TSPipeline
from bigdl.chronos.data import TSDataset

import onnxruntime

_onnxrt_ver = onnxruntime.__version__ != '1.6.0' # Jenkins requires 1.6.0(chronos)
skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0")

from .. import op_all, op_onnxrt16

def train_data_creator(config):
return DataLoader(TensorDataset(torch.randn(1000,
Expand Down Expand Up @@ -76,7 +72,8 @@ def setUp(self) -> None:
def tearDown(self) -> None:
pass

@skip_onnxrt
@op_all
@op_onnxrt16
def test_seq2seq_tsppl_support_dataloader(self):
# load
tsppl_seq2seq = TSPipeline.load(
Expand Down Expand Up @@ -180,7 +177,8 @@ def test_tsppl_mixed_data_type_usage(self):
with pytest.raises(RuntimeError):
yhat = tsppl_lstm.predict(data=get_test_tsdataset(), batch_size=16)

@skip_onnxrt
@op_all
@op_onnxrt16
def test_tsppl_quantize_data_creator(self):
# s2s not support quantize
with pytest.raises(RuntimeError):
Expand Down
24 changes: 13 additions & 11 deletions python/chronos/test/bigdl/chronos/forecaster/test_lstm_forecaster.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,7 @@
from bigdl.chronos.forecaster.lstm_forecaster import LSTMForecaster
from unittest import TestCase
import pytest
import onnxruntime

_onnxrt_ver = onnxruntime.__version__ != '1.6.0' # Jenkins requires 1.6.0(chronos)
skip_onnxrt = pytest.mark.skipif(_onnxrt_ver, reason="Only runs when onnxrt is 1.6.0")

from .. import op_all, op_onnxrt16

def create_data(loader=False):
num_train_samples = 1000
Expand Down Expand Up @@ -103,7 +99,8 @@ def test_lstm_forecaster_fit_eva_pred(self):
test_mse = forecaster.evaluate(test_data)
assert test_mse[0].shape == test_data[1].shape[1:]

@skip_onnxrt
@op_all
@op_onnxrt16
def test_lstm_forecaster_fit_loader(self):
train_loader, val_loader, test_loader = create_data(loader=True)
forecaster = LSTMForecaster(past_seq_len=24,
Expand All @@ -127,7 +124,8 @@ def test_lstm_forecaster_fit_loader(self):
forecaster.evaluate_with_onnx(test_loader)
forecaster.evaluate_with_onnx(test_loader, batch_size=32, quantize=True)

@skip_onnxrt
@op_all
@op_onnxrt16
def test_lstm_forecaster_onnx_methods(self):
train_data, val_data, test_data = create_data()
forecaster = LSTMForecaster(past_seq_len=24,
Expand Down Expand Up @@ -204,7 +202,8 @@ def test_lstm_forecaster_quantization(self):
np.testing.assert_almost_equal(test_pred_save, test_pred_load)
np.testing.assert_almost_equal(test_pred_save_q, test_pred_load_q)

@skip_onnxrt
@op_all
@op_onnxrt16
def test_lstm_forecaster_quantization_onnx(self):
train_data, val_data, test_data = create_data()
forecaster = LSTMForecaster(past_seq_len=24,
Expand All @@ -218,7 +217,8 @@ def test_lstm_forecaster_quantization_onnx(self):
pred_q = forecaster.predict_with_onnx(test_data[0], quantize=True)
eval_q = forecaster.evaluate_with_onnx(test_data, quantize=True)

@skip_onnxrt
@op_all
@op_onnxrt16
def test_lstm_forecaster_quantization_onnx_tuning(self):
train_data, val_data, test_data = create_data()
forecaster = LSTMForecaster(past_seq_len=24,
Expand Down Expand Up @@ -307,7 +307,8 @@ def transform_to_dict_x(data):
distributed_eval = forecaster.evaluate(val_data)
stop_orca_context()

@skip_onnxrt
@op_all
@op_onnxrt16
def test_lstm_forecaster_distributed(self):
from bigdl.orca import init_orca_context, stop_orca_context
train_data, val_data, test_data = create_data()
Expand Down Expand Up @@ -447,7 +448,8 @@ def test_forecaster_from_tsdataset(self):
_, y_test = test.to_numpy()
assert yhat.shape == y_test.shape

@skip_onnxrt
@op_all
@op_onnxrt16
def test_forecaster_from_tsdataset_data_loader_onnx(self):
train, test = create_tsdataset(roll=False)
train.gen_dt_feature(one_hot_features=['WEEK'])
Expand Down
Loading