From 9bb754d988d8f3038d267d823b12c885d04a935c Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 5 Oct 2023 15:12:18 -0500 Subject: [PATCH] Automate zppy tests --- .../generated/test_complete_run_chrysalis.cfg | 2 +- tests/integration/utils.py | 20 ++-- tests/scripts/test_dev.bash | 109 ++++++++++++++++++ 3 files changed, 122 insertions(+), 9 deletions(-) create mode 100755 tests/scripts/test_dev.bash diff --git a/tests/integration/generated/test_complete_run_chrysalis.cfg b/tests/integration/generated/test_complete_run_chrysalis.cfg index 286faf39..c9d65974 100644 --- a/tests/integration/generated/test_complete_run_chrysalis.cfg +++ b/tests/integration/generated/test_complete_run_chrysalis.cfg @@ -93,7 +93,7 @@ years = "1850:1854:2", "1850:1854:4", walltime = "5:00:00" [[ atm_monthly_180x360_aave_environment_commands ]] - environment_commands = "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20230914" + environment_commands = "source" sets = "qbo", ts_subsection = "atm_monthly_180x360_aave" diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 52f3fa40..4be4f93f 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -2,6 +2,7 @@ import re import shutil import subprocess +import sys from typing import List from mache import MachineInfo @@ -133,8 +134,6 @@ def get_chyrsalis_expansions(config): d = { "bundles_walltime": "07:00:00", "constraint": "", - # To run this test, replace conda environment with your e3sm_diags dev environment - "diags_environment_commands": "source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20230914", "diags_walltime": "5:00:00", "environment_commands_test": "source /lcrc/soft/climate/e3sm-unified/test_e3sm_unified_1.9.0rc16_chrysalis.sh", "expected_dir": "/lcrc/group/e3sm/public_html/zppy_test_resources/", @@ -158,8 +157,6 @@ def get_compy_expansions(config): d = { "bundles_walltime": "02:00:00", "constraint": "", - # To run this test, replace conda environment with your e3sm_diags dev environment - "diags_environment_commands": "source /qfs/people/fors729/mambaforge/etc/profile.d/conda.sh; conda activate e3sm_diags_20230914", "diags_walltime": "03:00:00", "environment_commands_test": "source /share/apps/E3SM/conda_envs/test_e3sm_unified_1.9.0rc16_compy.sh", "expected_dir": "/compyfs/www/zppy_test_resources/", @@ -183,8 +180,6 @@ def get_perlmutter_expansions(config): d = { "bundles_walltime": "6:00:00", "constraint": "cpu", - # To run this test, replace conda environment with your e3sm_diags dev environment - "diags_environment_commands": "source /global/homes/f/forsyth/miniconda3/etc/profile.d/conda.sh; conda activate e3sm_diags_20230914", "diags_walltime": "6:00:00", "environment_commands_test": "source /global/common/software/e3sm/anaconda_envs/test_e3sm_unified_1.9.0rc16_pm-cpu.sh", "expected_dir": "/global/cfs/cdirs/e3sm/www/zppy_test_resources/", @@ -234,19 +229,26 @@ def substitute_expansions(expansions, file_in, file_out): file_write.write(line) -def generate_cfgs(unified_testing=False, dry_run=False): +def generate_cfgs(unified_testing=False, diags_environment_commands="", dry_run=False): git_top_level = ( subprocess.check_output("git rev-parse --show-toplevel".split()) .strip() .decode("utf-8") ) expansions = get_expansions() + if unified_testing: expansions["environment_commands"] = expansions["environment_commands_test"] else: # The cfg doesn't need this line, # but it would be difficult to only write environment_commands in the unified_testing case. expansions["environment_commands"] = "" + + if diags_environment_commands: + expansions["diags_environment_commands"] = diags_environment_commands + else: + expansions["diags_environment_commands"] = "" + machine = expansions["machine"] if dry_run: @@ -276,4 +278,6 @@ def generate_cfgs(unified_testing=False, dry_run=False): if __name__ == "__main__": - generate_cfgs(unified_testing=False) + generate_cfgs( + unified_testing=(sys.argv[1] == "True"), diags_environment_commands=sys.argv[2] + ) diff --git a/tests/scripts/test_dev.bash b/tests/scripts/test_dev.bash new file mode 100755 index 00000000..32a49665 --- /dev/null +++ b/tests/scripts/test_dev.bash @@ -0,0 +1,109 @@ +#!/bin/bash + +# Before running this script ######################################################## + +# Make sure you're on the branch you want to test! + +# If you want to test `main`, do the following: +# git fetch upstream main +# git checkout -b test_pre_zppy_rc<#>_ upstream/main +# git log # check the commits match https://github.com/E3SM-Project/zppy/commits/main + +# Set these parameters +DIAGS_DIR=/home/ac.forsyth2/e3sm_diags/ +DIAGS_DEV=diags_dev_2023_10_05 +ZPPY_DIR=/home/ac.forsyth2/zppy/ +DIAGS_ENV_CMD="source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate ${DIAGS_DEV}" +ZPPY_DEV=zppy_dev_n516 + +# Make sure you do not have important changes on the `main` branch in your +# E3SM_DIAGS_DIRECTORY. This script will reset that branch to match `upstream`! + +##################################################################################### + +echo "Update E3SM Diags" +# `cd` to e3sm_diags directory +cd ${DIAGS_DIR} +git checkout main +git fetch upstream +git reset --hard upstream/main +git log # Should match https://github.com/E3SM-Project/e3sm_diags/commits/main # TODO: Requires user review +mamba clean --all # TODO: Requires user input to advance +conda remove -n ${DIAGS_DEV} --all +mamba env create -f conda-env/dev.yml -n ${DIAGS_DEV} +conda activate ${DIAGS_DEV} # TODO: errors with ./tests/scripts/test_dev.bash: line 34: {DIAGS_DEV}: command not found +pip install . +cd ${ZPPY_DIR} + +echo "Make sure we're using the latest packages" +UNIFIED_TESTING=False +python tests/integration/utils.py ${UNIFIED_TESTING} ${DIAGS_ENV_CMD} + +echo "Set up our environment" +mamba clean --all # TODO: Requires user input to advance +mamba env create -f conda/dev.yml -n ${ZPPY_DEV} +conda activate ${ZPPY_DEV} # TODO: errors with CommandNotFoundError: Your shell has not been properly configured to use 'conda activate'. +pip install . + +echo "Run unit tests" +python -u -m unittest tests/test_*.py +if [ $? != 0 ]; then + echo 'ERROR (1): unit tests failed' + exit 1 +fi + +exit 7 + +echo "Set up integration tests" +# TODO: somehow use Mache (a Python package) to get machine-independent paths in this bash script! +# test_complete_run +rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/v2.LR.historical_0201 +rm -rf /lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/v2.LR.historical_0201/post +# Run jobs: +zppy -c tests/integration/generated/test_complete_run_chrysalis.cfg +# TODO: how can we possibly tell, automatically, when this is finished? +# After they finish, check the results: +cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/v2.LR.historical_0201/post/scripts +grep -v "OK" *status +if [ $? == 0 ]; then + # The above command succeeds only if there are reported failures. + echo 'ERROR (2): zppy complete run failed.' + exit 2 +fi +cd ${ZPPY_DIR} +# test_bundles +rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/v2.LR.historical_0201 +rm -rf /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201/post +# Run first set of jobs: +zppy -c tests/integration/generated/test_bundles_chrysalis.cfg +# TODO: how can we possibly tell, automatically, when this is finished? +# bundle1 and bundle2 should run. After they finish, check the results: +cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201/post/scripts +grep -v "OK" *status +if [ $? == 0 ]; then + # The above command succeeds only if there are reported failures. + echo 'ERROR (3): zppy bundles 1st run failed.' + exit 3 +fi +cd ${ZPPY_DIR} +# Now, invoke zppy again to run jobs that needed to wait for dependencies: +zppy -c tests/integration/generated/test_bundles_chrysalis.cfg +# TODO: how can we possibly tell, automatically, when this is finished? +# bundle3 and ilamb should run. After they finish, check the results: +cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201/post/scripts +grep -v "OK" *status +if [ $? == 0 ]; then + # The above command succeeds only if there are reported failures. + echo 'ERROR (4): zppy bundles 2nd run failed.' + exit 4 +fi +cd ${ZPPY_DIR} + +echo "Run the integration tests" +python -u -m unittest tests/integration/test_*.py +if [ $? != 0 ]; then + echo 'ERROR (5): integration tests failed' + exit 5 +fi + +echo "All tests passed!"