From 6c26e08a50613750e69afd5828d00e3ad2cfb6dc Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Thu, 5 Oct 2023 15:12:18 -0500 Subject: [PATCH] Automate zppy tests --- .../generated/test_complete_run_chrysalis.cfg | 155 ++++++++++++++++++ tests/integration/utils.py | 23 +-- tests/scripts/test_dev.bash | 109 ++++++++++++ 3 files changed, 276 insertions(+), 11 deletions(-) create mode 100644 tests/integration/generated/test_complete_run_chrysalis.cfg create mode 100755 tests/scripts/test_dev.bash diff --git a/tests/integration/generated/test_complete_run_chrysalis.cfg b/tests/integration/generated/test_complete_run_chrysalis.cfg new file mode 100644 index 00000000..c9d65974 --- /dev/null +++ b/tests/integration/generated/test_complete_run_chrysalis.cfg @@ -0,0 +1,155 @@ +[default] +case = v2.LR.historical_0201 +constraint = "" +dry_run = "False" +environment_commands = "" +input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201" +input_subdir = archive/atm/hist +mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" +# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py +output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/v2.LR.historical_0201" +partition = "debug" +qos = "regular" +www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www" + +[climo] +active = True +walltime = "00:30:00" +years = "1850:1854:2", "1850:1854:4", + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + + [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] + frequency = "diurnal_8xdaily" + input_files = "eam.h4" + input_subdir = "archive/atm/hist" + vars = "PRECT" + +[ts] +active = True +walltime = "00:30:00" +years = "1850:1854:2", + + [[ atm_monthly_180x360_aave ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + ts_fmt = "cmip" + + [[ atm_daily_180x360_aave ]] + frequency = "daily" + input_files = "eam.h1" + input_subdir = "archive/atm/hist" + vars = "PRECT" + + [[ atm_monthly_glb ]] + frequency = "monthly" + input_files = "eam.h0" + input_subdir = "archive/atm/hist" + mapping_file = "glb" + years = "1850:1860:5", + + [[ land_monthly ]] + extra_vars = "landfrac" + frequency = "monthly" + input_files = "elm.h0" + input_subdir = "archive/lnd/hist" + vars = "FSH,LAISHA,LAISUN,RH2M" + ts_fmt = "cmip" + + [[ rof_monthly ]] + extra_vars = 'areatotal2' + frequency = "monthly" + input_files = "mosart.h0" + input_subdir = "archive/rof/hist" + mapping_file = "" + vars = "RIVER_DISCHARGE_OVER_LAND_LIQ" + +[tc_analysis] +active = True +scratch = "/lcrc/globalscratch/ac.forsyth2/" +walltime = "00:30:00" +years = "1850:1854:2", + +[e3sm_diags] +active = True +grid = '180x360_aave' +ref_final_yr = 2014 +ref_start_yr = 1985 +# TODO: this directory is missing OMI-MLS +sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere", "tc_analysis", +short_name = 'v2.LR.historical_0201' +ts_num_years = 2 +walltime = "00:30:00" +years = "1850:1854:2", "1850:1854:4", + + [[ atm_monthly_180x360_aave ]] + climo_diurnal_frequency = "diurnal_8xdaily" + climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" + partition = "compute" + qos = "regular" + sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere", + walltime = "5:00:00" + + [[ atm_monthly_180x360_aave_environment_commands ]] + environment_commands = "source" + sets = "qbo", + ts_subsection = "atm_monthly_180x360_aave" + + [[ atm_monthly_180x360_aave_tc_analysis ]] + # Running as its own subtask because tc_analysis requires jobs to run sequentially, which slows down testing + sets = "tc_analysis", + years = "1850:1852:2", + + [[ atm_monthly_180x360_aave_mvm ]] + # Test model-vs-model using the same files as the reference + climo_diurnal_frequency = "diurnal_8xdaily" + climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" + climo_subsection = "atm_monthly_180x360_aave" + diff_title = "Difference" + partition = "compute" + qos = "regular" + ref_final_yr = 1851 + ref_name = "v2.LR.historical_0201" + ref_start_yr = 1850 + ref_years = "1850-1851", + reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/v2.LR.historical_0201/post/atm/180x360_aave/clim" + run_type = "model_vs_model" + short_ref_name = "v2.LR.historical_0201" + swap_test_ref = False + tag = "model_vs_model" + ts_num_years_ref = 2 + ts_subsection = "atm_monthly_180x360_aave" + walltime = "5:00:00" + years = "1852-1853", + +[mpas_analysis] +active = True +anomalyRefYear = 1850 +climo_years ="1850-1854", "1855-1860", +enso_years = "1850-1854", "1855-1860", +mesh = "EC30to60E2r2" +parallelTaskCount = 6 +partition = "compute" +qos = "regular" +ts_years = "1850-1854", "1850-1860", +walltime = "00:30:00" + +[global_time_series] +active = True +climo_years ="1850-1854", "1855-1860", +experiment_name = "v2.LR.historical_0201" +figstr = "v2_historical_0201" +moc_file=mocTimeSeries_1850-1860.nc +ts_num_years = 5 +ts_years = "1850-1854", "1850-1860", +walltime = "00:30:00" +years = "1850-1860", + +[ilamb] +active = True +grid = '180x360_aave' +short_name = 'v2.LR.historical_0201' +ts_num_years = 2 +years = "1850:1854:2", diff --git a/tests/integration/utils.py b/tests/integration/utils.py index d008fd41..e4df740c 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -2,6 +2,7 @@ import re import shutil import subprocess +import sys from typing import List from mache import MachineInfo @@ -144,9 +145,6 @@ def get_chyrsalis_expansions(config): "case_name": "v3.LR.historical_0051", "case_name_v2": "v2.LR.historical_0201", "constraint": "", - # To run this test, replace conda environment with your e3sm_diags dev environment - # To use default environment_commands, set to "" - "diags_environment_commands": "source /conda.sh; conda activate ", "diags_walltime": "5:00:00", "e3sm_to_cmip_environment_commands": "", "environment_commands_test": "", @@ -174,9 +172,6 @@ def get_compy_expansions(config): "case_name": "v3.LR.historical_0051", "case_name_v2": "v2.LR.historical_0201", "constraint": "", - # To run this test, replace conda environment with your e3sm_diags dev environment - # To use default environment_commands, set to "" - "diags_environment_commands": "source /conda.sh; conda activate ", "diags_walltime": "03:00:00", "e3sm_to_cmip_environment_commands": "", "environment_commands_test": "", @@ -204,9 +199,6 @@ def get_perlmutter_expansions(config): "case_name": "v3.LR.historical_0051", "case_name_v2": "v2.LR.historical_0201", "constraint": "cpu", - # To run this test, replace conda environment with your e3sm_diags dev environment - # To use default environment_commands, set to "" - "diags_environment_commands": "source /conda.sh; conda activate ", "diags_walltime": "6:00:00", "e3sm_to_cmip_environment_commands": "", "environment_commands_test": "", @@ -260,19 +252,26 @@ def substitute_expansions(expansions, file_in, file_out): file_write.write(line) -def generate_cfgs(unified_testing=False, dry_run=False): +def generate_cfgs(unified_testing=False, diags_environment_commands="", dry_run=False): git_top_level = ( subprocess.check_output("git rev-parse --show-toplevel".split()) .strip() .decode("utf-8") ) expansions = get_expansions() + if unified_testing: expansions["environment_commands"] = expansions["environment_commands_test"] else: # The cfg doesn't need this line, # but it would be difficult to only write environment_commands in the unified_testing case. expansions["environment_commands"] = "" + + if diags_environment_commands: + expansions["diags_environment_commands"] = diags_environment_commands + else: + expansions["diags_environment_commands"] = "" + machine = expansions["machine"] if dry_run: @@ -369,4 +368,6 @@ def generate_cfgs(unified_testing=False, dry_run=False): if __name__ == "__main__": - generate_cfgs(unified_testing=False, dry_run=False) + generate_cfgs( + unified_testing=(sys.argv[1] == "True"), diags_environment_commands=sys.argv[2] + ) diff --git a/tests/scripts/test_dev.bash b/tests/scripts/test_dev.bash new file mode 100755 index 00000000..32a49665 --- /dev/null +++ b/tests/scripts/test_dev.bash @@ -0,0 +1,109 @@ +#!/bin/bash + +# Before running this script ######################################################## + +# Make sure you're on the branch you want to test! + +# If you want to test `main`, do the following: +# git fetch upstream main +# git checkout -b test_pre_zppy_rc<#>_ upstream/main +# git log # check the commits match https://github.com/E3SM-Project/zppy/commits/main + +# Set these parameters +DIAGS_DIR=/home/ac.forsyth2/e3sm_diags/ +DIAGS_DEV=diags_dev_2023_10_05 +ZPPY_DIR=/home/ac.forsyth2/zppy/ +DIAGS_ENV_CMD="source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate ${DIAGS_DEV}" +ZPPY_DEV=zppy_dev_n516 + +# Make sure you do not have important changes on the `main` branch in your +# E3SM_DIAGS_DIRECTORY. This script will reset that branch to match `upstream`! + +##################################################################################### + +echo "Update E3SM Diags" +# `cd` to e3sm_diags directory +cd ${DIAGS_DIR} +git checkout main +git fetch upstream +git reset --hard upstream/main +git log # Should match https://github.com/E3SM-Project/e3sm_diags/commits/main # TODO: Requires user review +mamba clean --all # TODO: Requires user input to advance +conda remove -n ${DIAGS_DEV} --all +mamba env create -f conda-env/dev.yml -n ${DIAGS_DEV} +conda activate ${DIAGS_DEV} # TODO: errors with ./tests/scripts/test_dev.bash: line 34: {DIAGS_DEV}: command not found +pip install . +cd ${ZPPY_DIR} + +echo "Make sure we're using the latest packages" +UNIFIED_TESTING=False +python tests/integration/utils.py ${UNIFIED_TESTING} ${DIAGS_ENV_CMD} + +echo "Set up our environment" +mamba clean --all # TODO: Requires user input to advance +mamba env create -f conda/dev.yml -n ${ZPPY_DEV} +conda activate ${ZPPY_DEV} # TODO: errors with CommandNotFoundError: Your shell has not been properly configured to use 'conda activate'. +pip install . + +echo "Run unit tests" +python -u -m unittest tests/test_*.py +if [ $? != 0 ]; then + echo 'ERROR (1): unit tests failed' + exit 1 +fi + +exit 7 + +echo "Set up integration tests" +# TODO: somehow use Mache (a Python package) to get machine-independent paths in this bash script! +# test_complete_run +rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/v2.LR.historical_0201 +rm -rf /lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/v2.LR.historical_0201/post +# Run jobs: +zppy -c tests/integration/generated/test_complete_run_chrysalis.cfg +# TODO: how can we possibly tell, automatically, when this is finished? +# After they finish, check the results: +cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/v2.LR.historical_0201/post/scripts +grep -v "OK" *status +if [ $? == 0 ]; then + # The above command succeeds only if there are reported failures. + echo 'ERROR (2): zppy complete run failed.' + exit 2 +fi +cd ${ZPPY_DIR} +# test_bundles +rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/v2.LR.historical_0201 +rm -rf /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201/post +# Run first set of jobs: +zppy -c tests/integration/generated/test_bundles_chrysalis.cfg +# TODO: how can we possibly tell, automatically, when this is finished? +# bundle1 and bundle2 should run. After they finish, check the results: +cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201/post/scripts +grep -v "OK" *status +if [ $? == 0 ]; then + # The above command succeeds only if there are reported failures. + echo 'ERROR (3): zppy bundles 1st run failed.' + exit 3 +fi +cd ${ZPPY_DIR} +# Now, invoke zppy again to run jobs that needed to wait for dependencies: +zppy -c tests/integration/generated/test_bundles_chrysalis.cfg +# TODO: how can we possibly tell, automatically, when this is finished? +# bundle3 and ilamb should run. After they finish, check the results: +cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201/post/scripts +grep -v "OK" *status +if [ $? == 0 ]; then + # The above command succeeds only if there are reported failures. + echo 'ERROR (4): zppy bundles 2nd run failed.' + exit 4 +fi +cd ${ZPPY_DIR} + +echo "Run the integration tests" +python -u -m unittest tests/integration/test_*.py +if [ $? != 0 ]; then + echo 'ERROR (5): integration tests failed' + exit 5 +fi + +echo "All tests passed!"