diff --git a/.gitignore b/.gitignore index f23d20246a..e7620d5cf3 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ _local _external_src _reports tmp -serialized_data +testdata simple_mesh*.nc ### GT4Py #### diff --git a/ci/cscs.yml b/ci/cscs.yml index 6bf6bee0ff..9a71720925 100644 --- a/ci/cscs.yml +++ b/ci/cscs.yml @@ -43,7 +43,7 @@ variables: NUM_PROCESSES: auto VIRTUALENV_SYSTEM_SITE_PACKAGES: 1 CSCS_NEEDED_DATA: icon4py - SERIALIZED_DATA_PATH: "/apps/daint/UES/jenkssl/ciext/icon4py" + TEST_DATA_PATH: "/apps/daint/UES/jenkssl/ciext/icon4py" build_job: extends: .build_template @@ -52,14 +52,14 @@ test_model_job_roundtrip_simple_grid: extends: .test_template stage: test script: - - tox -r -c model/ --verbose -- --benchmark-skip -n auto + - tox -r -e run_stencil_tests -c model/ --verbose test_model_job_dace_cpu_simple_grid: extends: .test_template stage: test script: - pip install dace==$DACE_VERSION - - tox -r -e run_stencil_tests -c model/ --verbose -- --benchmark-skip -n auto --backend=dace_cpu + - tox -r -e run_stencil_tests -c model/ --verbose -- --backend=dace_cpu only: - main allow_failure: true @@ -69,7 +69,7 @@ test_model_job_dace_gpu_simple_grid: stage: test script: - pip install dace==$DACE_VERSION - - tox -r -e run_stencil_tests -c model/ --verbose -- --benchmark-skip -n auto --backend=dace_gpu + - tox -r -e run_stencil_tests -c model/ --verbose -- --backend=dace_gpu only: - main allow_failure: true @@ -78,13 +78,13 @@ test_model_job_gtfn_cpu_simple_grid: extends: .test_template stage: test script: - - tox -r -e run_stencil_tests -c model/ --verbose -- --benchmark-skip -n auto --backend=gtfn_cpu + - tox -r -e run_stencil_tests -c model/ --verbose -- --backend=gtfn_cpu test_model_job_gtfn_gpu_simple_grid: extends: .test_template stage: test script: - - tox -r -e run_stencil_tests -c model/ --verbose -- --benchmark-skip -n auto --backend=gtfn_gpu + - tox -r -e run_stencil_tests -c model/ --verbose -- --backend=gtfn_gpu test_tools_job: extends: .test_template @@ -97,7 +97,7 @@ benchmark_model_dace_cpu_icon_grid: stage: benchmark script: - pip install dace==$DACE_VERSION - - tox -r -e run_benchmarks -c model/ -- --benchmark-only --backend=dace_cpu --grid=icon_grid + - tox -r -e run_benchmarks -c model/ -- --backend=dace_cpu --grid=icon_grid only: - main when: manual @@ -107,7 +107,7 @@ benchmark_model_dace_gpu_icon_grid: stage: benchmark script: - pip install dace==$DACE_VERSION - - tox -r -e run_benchmarks -c model/ -- --benchmark-only --backend=dace_gpu --grid=icon_grid + - tox -r -e run_benchmarks -c model/ -- --backend=dace_gpu --grid=icon_grid only: - main when: manual @@ -116,10 +116,10 @@ benchmark_model_gtfn_cpu_icon_grid: extends: .test_template stage: benchmark script: - - tox -r -e run_benchmarks -c model/ -- --benchmark-only --backend=gtfn_cpu --grid=icon_grid + - tox -r -e run_benchmarks -c model/ -- --backend=gtfn_cpu --grid=icon_grid benchmark_model_gtfn_gpu_icon_grid: extends: .test_template stage: benchmark script: - - tox -r -e run_benchmarks -c model/ -- --benchmark-only --backend=gtfn_gpu --grid=icon_grid + - tox -r -e run_benchmarks -c model/ -- --backend=gtfn_gpu --grid=icon_grid diff --git a/model/common/src/icon4py/model/common/test_utils/datatest_utils.py b/model/common/src/icon4py/model/common/test_utils/datatest_utils.py index 400c787145..97dff04209 100644 --- a/model/common/src/icon4py/model/common/test_utils/datatest_utils.py +++ b/model/common/src/icon4py/model/common/test_utils/datatest_utils.py @@ -16,23 +16,23 @@ from icon4py.model.common.decomposition.definitions import get_processor_properties -DEFAULT_SERIALIZED_DATA_FOLDER = "serialized_data" +DEFAULT_TEST_DATA_FOLDER = "testdata" -def get_serialized_data_root_path() -> Path: +def get_test_data_root_path() -> Path: test_utils_path = Path(__file__).parent model_path = test_utils_path.parent.parent common_path = model_path.parent.parent.parent.parent - env_base_path = os.getenv("SERIALIZED_DATA_PATH") + env_base_path = os.getenv("TEST_DATA_PATH") if env_base_path: return Path(env_base_path) else: - return common_path.parent.joinpath(DEFAULT_SERIALIZED_DATA_FOLDER) + return common_path.parent.joinpath(DEFAULT_TEST_DATA_FOLDER) -SERIALIZED_DATA_ROOT = get_serialized_data_root_path() -SERIALIZED_DATA_PATH = SERIALIZED_DATA_ROOT.joinpath("ser_icondata") +TEST_DATA_ROOT = get_test_data_root_path() +SERIALIZED_DATA_PATH = TEST_DATA_ROOT.joinpath("ser_icondata") # TODO: a run that contains all the fields needed for dycore, diffusion, interpolation fields needs to be consolidated DATA_URIS = { diff --git a/model/common/tests/grid_tests/conftest.py b/model/common/tests/grid_tests/conftest.py index 3383ed0b80..639e8e1990 100644 --- a/model/common/tests/grid_tests/conftest.py +++ b/model/common/tests/grid_tests/conftest.py @@ -26,10 +26,10 @@ processor_props, ranked_data_path, ) -from icon4py.model.common.test_utils.datatest_utils import SERIALIZED_DATA_PATH +from icon4py.model.common.test_utils.datatest_utils import TEST_DATA_ROOT -grids_path = SERIALIZED_DATA_PATH.joinpath("grids") +grids_path = TEST_DATA_ROOT.joinpath("grids") r04b09_dsl_grid_path = grids_path.joinpath("mch_ch_r04b09_dsl") r04b09_dsl_data_file = r04b09_dsl_grid_path.joinpath("mch_ch_r04b09_dsl_grids_v1.tar.gz").name r02b04_global_grid_path = grids_path.joinpath("r02b04_global") diff --git a/model/driver/README.md b/model/driver/README.md index 77068bc1c3..8142f9b749 100644 --- a/model/driver/README.md +++ b/model/driver/README.md @@ -18,7 +18,7 @@ See the general instructions in the [README.md](../../README.md) in the base fol ```bash export ICON4PY_ROOT= -dycore_driver $ICON4PY_ROOT/serialized_data/ser_icondata/mpitask1/mch_ch_r04b09_dsl/ser_data --run_path=$ICON4PY_ROOT/output +dycore_driver $ICON4PY_ROOT/testdata/ser_icondata/mpitask1/mch_ch_r04b09_dsl/ser_data --run_path=$ICON4PY_ROOT/output ``` The driver code runs in parallel, in order to do this you need to install the optional communication libraries with: @@ -32,7 +32,7 @@ pip install -r requirements-dev-opt.txt then run ```bash -mpirun -np 2 dycore_driver $ICON4PY_ROOT/serialized_data/ser_icondata/mpitask2/mch_ch_r04b09_dsl/ser_data --mpi=True --run_path=$ICON4PY_ROOT/output +mpirun -np 2 dycore_driver $ICON4PY_ROOT/testdata/ser_icondata/mpitask2/mch_ch_r04b09_dsl/ser_data --mpi=True --run_path=$ICON4PY_ROOT/output ``` #### Remarks diff --git a/model/driver/src/icon4py/model/driver/dycore_driver.py b/model/driver/src/icon4py/model/driver/dycore_driver.py index 284ff5cf7d..975598d6a0 100644 --- a/model/driver/src/icon4py/model/driver/dycore_driver.py +++ b/model/driver/src/icon4py/model/driver/dycore_driver.py @@ -383,7 +383,7 @@ def main(input_path, run_path, mpi): """ Run the driver. - usage: python dycore_driver.py abs_path_to_icon4py/serialized_data/ser_icondata/mpitask1/mch_ch_r04b09_dsl/ser_data + usage: python dycore_driver.py abs_path_to_icon4py/testdata/ser_icondata/mpitask1/mch_ch_r04b09_dsl/ser_data steps: 1. initialize model from serialized data: diff --git a/model/tox.ini b/model/tox.ini index 5b55d882c0..61496307b6 100644 --- a/model/tox.ini +++ b/model/tox.ini @@ -28,13 +28,13 @@ allowlist_externals = [testenv:run_stencil_tests] commands = - pytest -v -s -m "not slow_tests" --cov --cov-append atmosphere/diffusion/tests/diffusion_stencil_tests {posargs} - pytest -v -s -m "not slow_tests" --cov --cov-append atmosphere/dycore/tests/dycore_stencil_tests {posargs} + pytest -v -s -m "not slow_tests" --cov --cov-append atmosphere/diffusion/tests/diffusion_stencil_tests --benchmark-skip -n auto {posargs} + pytest -v -s -m "not slow_tests" --cov --cov-append atmosphere/dycore/tests/dycore_stencil_tests --benchmark-skip -n auto {posargs} [testenv:run_benchmarks] commands = - pytest -s -m "not slow_tests" atmosphere/diffusion/tests/diffusion_stencil_tests {posargs} - pytest -s -m "not slow_tests" atmosphere/dycore/tests/dycore_stencil_tests {posargs} + pytest -s -m "not slow_tests" atmosphere/diffusion/tests/diffusion_stencil_tests --benchmark-only {posargs} + pytest -s -m "not slow_tests" atmosphere/dycore/tests/dycore_stencil_tests --benchmark-only {posargs} [testenv:dev]