From 0a7b8cc45cbb2488a137ce1e8d52ec59395a8f1a Mon Sep 17 00:00:00 2001 From: Ryan Forsyth Date: Fri, 13 Dec 2024 19:01:43 -0600 Subject: [PATCH] Post-zppy-zi-split --- .../generated/test_complete_run_chrysalis.cfg | 155 ----------------- tests/integration/utils.py | 35 +++- tests/scripts/test_dev.bash | 157 +++++++++--------- 3 files changed, 111 insertions(+), 236 deletions(-) delete mode 100644 tests/integration/generated/test_complete_run_chrysalis.cfg diff --git a/tests/integration/generated/test_complete_run_chrysalis.cfg b/tests/integration/generated/test_complete_run_chrysalis.cfg deleted file mode 100644 index c9d65974..00000000 --- a/tests/integration/generated/test_complete_run_chrysalis.cfg +++ /dev/null @@ -1,155 +0,0 @@ -[default] -case = v2.LR.historical_0201 -constraint = "" -dry_run = "False" -environment_commands = "" -input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201" -input_subdir = archive/atm/hist -mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc" -# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py -output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/v2.LR.historical_0201" -partition = "debug" -qos = "regular" -www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www" - -[climo] -active = True -walltime = "00:30:00" -years = "1850:1854:2", "1850:1854:4", - - [[ atm_monthly_180x360_aave ]] - frequency = "monthly" - - [[ atm_monthly_diurnal_8xdaily_180x360_aave ]] - frequency = "diurnal_8xdaily" - input_files = "eam.h4" - input_subdir = "archive/atm/hist" - vars = "PRECT" - -[ts] -active = True -walltime = "00:30:00" -years = "1850:1854:2", - - [[ atm_monthly_180x360_aave ]] - frequency = "monthly" - input_files = "eam.h0" - input_subdir = "archive/atm/hist" - ts_fmt = "cmip" - - [[ atm_daily_180x360_aave ]] - frequency = "daily" - input_files = "eam.h1" - input_subdir = "archive/atm/hist" - vars = "PRECT" - - [[ atm_monthly_glb ]] - frequency = "monthly" - input_files = "eam.h0" - input_subdir = "archive/atm/hist" - mapping_file = "glb" - years = "1850:1860:5", - - [[ land_monthly ]] - extra_vars = "landfrac" - frequency = "monthly" - input_files = "elm.h0" - input_subdir = "archive/lnd/hist" - vars = "FSH,LAISHA,LAISUN,RH2M" - ts_fmt = "cmip" - - [[ rof_monthly ]] - extra_vars = 'areatotal2' - frequency = "monthly" - input_files = "mosart.h0" - input_subdir = "archive/rof/hist" - mapping_file = "" - vars = "RIVER_DISCHARGE_OVER_LAND_LIQ" - -[tc_analysis] -active = True -scratch = "/lcrc/globalscratch/ac.forsyth2/" -walltime = "00:30:00" -years = "1850:1854:2", - -[e3sm_diags] -active = True -grid = '180x360_aave' -ref_final_yr = 2014 -ref_start_yr = 1985 -# TODO: this directory is missing OMI-MLS -sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere", "tc_analysis", -short_name = 'v2.LR.historical_0201' -ts_num_years = 2 -walltime = "00:30:00" -years = "1850:1854:2", "1850:1854:4", - - [[ atm_monthly_180x360_aave ]] - climo_diurnal_frequency = "diurnal_8xdaily" - climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" - partition = "compute" - qos = "regular" - sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere", - walltime = "5:00:00" - - [[ atm_monthly_180x360_aave_environment_commands ]] - environment_commands = "source" - sets = "qbo", - ts_subsection = "atm_monthly_180x360_aave" - - [[ atm_monthly_180x360_aave_tc_analysis ]] - # Running as its own subtask because tc_analysis requires jobs to run sequentially, which slows down testing - sets = "tc_analysis", - years = "1850:1852:2", - - [[ atm_monthly_180x360_aave_mvm ]] - # Test model-vs-model using the same files as the reference - climo_diurnal_frequency = "diurnal_8xdaily" - climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave" - climo_subsection = "atm_monthly_180x360_aave" - diff_title = "Difference" - partition = "compute" - qos = "regular" - ref_final_yr = 1851 - ref_name = "v2.LR.historical_0201" - ref_start_yr = 1850 - ref_years = "1850-1851", - reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/v2.LR.historical_0201/post/atm/180x360_aave/clim" - run_type = "model_vs_model" - short_ref_name = "v2.LR.historical_0201" - swap_test_ref = False - tag = "model_vs_model" - ts_num_years_ref = 2 - ts_subsection = "atm_monthly_180x360_aave" - walltime = "5:00:00" - years = "1852-1853", - -[mpas_analysis] -active = True -anomalyRefYear = 1850 -climo_years ="1850-1854", "1855-1860", -enso_years = "1850-1854", "1855-1860", -mesh = "EC30to60E2r2" -parallelTaskCount = 6 -partition = "compute" -qos = "regular" -ts_years = "1850-1854", "1850-1860", -walltime = "00:30:00" - -[global_time_series] -active = True -climo_years ="1850-1854", "1855-1860", -experiment_name = "v2.LR.historical_0201" -figstr = "v2_historical_0201" -moc_file=mocTimeSeries_1850-1860.nc -ts_num_years = 5 -ts_years = "1850-1854", "1850-1860", -walltime = "00:30:00" -years = "1850-1860", - -[ilamb] -active = True -grid = '180x360_aave' -short_name = 'v2.LR.historical_0201' -ts_num_years = 2 -years = "1850:1854:2", diff --git a/tests/integration/utils.py b/tests/integration/utils.py index e4df740c..36899e25 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -145,6 +145,9 @@ def get_chyrsalis_expansions(config): "case_name": "v3.LR.historical_0051", "case_name_v2": "v2.LR.historical_0201", "constraint": "", + # To run this test, replace conda environment with your e3sm_diags dev environment + # To use default environment_commands, set to "" + "diags_environment_commands": "source /conda.sh; conda activate ", "diags_walltime": "5:00:00", "e3sm_to_cmip_environment_commands": "", "environment_commands_test": "", @@ -172,6 +175,9 @@ def get_compy_expansions(config): "case_name": "v3.LR.historical_0051", "case_name_v2": "v2.LR.historical_0201", "constraint": "", + # To run this test, replace conda environment with your e3sm_diags dev environment + # To use default environment_commands, set to "" + "diags_environment_commands": "source /conda.sh; conda activate ", "diags_walltime": "03:00:00", "e3sm_to_cmip_environment_commands": "", "environment_commands_test": "", @@ -199,6 +205,9 @@ def get_perlmutter_expansions(config): "case_name": "v3.LR.historical_0051", "case_name_v2": "v2.LR.historical_0201", "constraint": "cpu", + # To run this test, replace conda environment with your e3sm_diags dev environment + # To use default environment_commands, set to "" + "diags_environment_commands": "source /conda.sh; conda activate ", "diags_walltime": "6:00:00", "e3sm_to_cmip_environment_commands": "", "environment_commands_test": "", @@ -252,14 +261,18 @@ def substitute_expansions(expansions, file_in, file_out): file_write.write(line) -def generate_cfgs(unified_testing=False, diags_environment_commands="", dry_run=False): +def generate_cfgs( + unified_testing=False, + diags_environment_commands=None, + global_time_series_environment_commands=None, + dry_run=False, +): git_top_level = ( subprocess.check_output("git rev-parse --show-toplevel".split()) .strip() .decode("utf-8") ) expansions = get_expansions() - if unified_testing: expansions["environment_commands"] = expansions["environment_commands_test"] else: @@ -269,8 +282,10 @@ def generate_cfgs(unified_testing=False, diags_environment_commands="", dry_run= if diags_environment_commands: expansions["diags_environment_commands"] = diags_environment_commands - else: - expansions["diags_environment_commands"] = "" + if global_time_series_environment_commands: + expansions["global_time_series_environment_commands"] = ( + global_time_series_environment_commands + ) machine = expansions["machine"] @@ -368,6 +383,12 @@ def generate_cfgs(unified_testing=False, diags_environment_commands="", dry_run= if __name__ == "__main__": - generate_cfgs( - unified_testing=(sys.argv[1] == "True"), diags_environment_commands=sys.argv[2] - ) + if len(sys.argv) <= 2: + generate_cfgs(unified_testing=False, dry_run=False) + else: + generate_cfgs( + unified_testing=sys.argv[2] == "True", + diags_environment_commands=sys.argv[3], + global_time_series_environment_commands=sys.argv[4], + dry_run=False, + ) diff --git a/tests/scripts/test_dev.bash b/tests/scripts/test_dev.bash index 32a49665..f5f521c9 100755 --- a/tests/scripts/test_dev.bash +++ b/tests/scripts/test_dev.bash @@ -2,108 +2,117 @@ # Before running this script ######################################################## -# Make sure you're on the branch you want to test! - +# Set up branch # If you want to test `main`, do the following: # git fetch upstream main -# git checkout -b test_pre_zppy_rc<#>_ upstream/main +# git checkout -b test_main_ upstream/main # git log # check the commits match https://github.com/E3SM-Project/zppy/commits/main -# Set these parameters -DIAGS_DIR=/home/ac.forsyth2/e3sm_diags/ -DIAGS_DEV=diags_dev_2023_10_05 +# Set these parameters: + +# Make sure you do not have important changes in any of these directories. This script will reset them! ZPPY_DIR=/home/ac.forsyth2/zppy/ -DIAGS_ENV_CMD="source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate ${DIAGS_DEV}" +DIAGS_DIR=/home/ac.forsyth2/e3sm_diags/ +ZI_DIR=/home/ac.forsyth2/zppy-interfaces/ + ZPPY_DEV=zppy_dev_n516 +DIAGS_DEV=diags_dev_2024_12_13 +ZI_DEV=zi_dev_2024_12_13 -# Make sure you do not have important changes on the `main` branch in your -# E3SM_DIAGS_DIRECTORY. This script will reset that branch to match `upstream`! +DIAGS_ENV_CMD="source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate ${DIAGS_DEV}" +ZI_ENV_CMD="source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate ${ZI_DEV}" + +UNIQUE_ID="unique_id" ##################################################################################### -echo "Update E3SM Diags" -# `cd` to e3sm_diags directory +# Set up zppy-interfaces env +cd ${ZI_DIR} +git fetch upstream main +git checkout main upstream/main +if [ $? != 0 ]; then + echo 'ERROR (1): Could not check out zppy-interfaces main branch' + exit 1 +fi +git reset --hard upstream/main +conda clean --all --y +# TODO: previous iterations of this script had issues with activating conda environments +conda env create -f conda/dev.yml -n ${ZI_DEV} +conda activate ${ZI_DEV} +pip install . +pytest tests/global_time_series/test_*.py +if [ $? != 0 ]; then + echo 'ERROR (2): zppy-interfaces unit tests failed' + exit 2 +fi + +# Set up e3sm_diags env cd ${DIAGS_DIR} -git checkout main git fetch upstream +git checkout main +if [ $? != 0 ]; then + echo 'ERROR (3): Could not check out e3sm_diags main branch' + exit 3 +fi git reset --hard upstream/main -git log # Should match https://github.com/E3SM-Project/e3sm_diags/commits/main # TODO: Requires user review -mamba clean --all # TODO: Requires user input to advance -conda remove -n ${DIAGS_DEV} --all -mamba env create -f conda-env/dev.yml -n ${DIAGS_DEV} -conda activate ${DIAGS_DEV} # TODO: errors with ./tests/scripts/test_dev.bash: line 34: {DIAGS_DEV}: command not found +conda clean --all --y +conda env create -f conda-env/dev.yml -n ${DIAGS_DEV} +conda activate ${DIAGS_DEV} pip install . -cd ${ZPPY_DIR} -echo "Make sure we're using the latest packages" -UNIFIED_TESTING=False -python tests/integration/utils.py ${UNIFIED_TESTING} ${DIAGS_ENV_CMD} - -echo "Set up our environment" -mamba clean --all # TODO: Requires user input to advance -mamba env create -f conda/dev.yml -n ${ZPPY_DEV} -conda activate ${ZPPY_DEV} # TODO: errors with CommandNotFoundError: Your shell has not been properly configured to use 'conda activate'. +# Set up zppy env +cd ${ZPPY_DIR} +# We should already be on the branch we want to test! +conda clean --all --y +conda env create -f conda/dev.yml -n ${ZPPY_DEV} +conda activate ${ZPPY_DEV} pip install . - -echo "Run unit tests" -python -u -m unittest tests/test_*.py +pytest tests/test_*.py if [ $? != 0 ]; then - echo 'ERROR (1): unit tests failed' - exit 1 + echo 'ERROR (4): zppy unit tests failed' + exit 4 fi -exit 7 - -echo "Set up integration tests" -# TODO: somehow use Mache (a Python package) to get machine-independent paths in this bash script! -# test_complete_run -rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/v2.LR.historical_0201 -rm -rf /lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/v2.LR.historical_0201/post -# Run jobs: -zppy -c tests/integration/generated/test_complete_run_chrysalis.cfg -# TODO: how can we possibly tell, automatically, when this is finished? -# After they finish, check the results: -cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/v2.LR.historical_0201/post/scripts +# Integration testing for zppy +python tests/integration/utils.py False ${DIAGS_DEV} ${ZI_DEV} + + +zppy -c tests/integration/generated/test_weekly_comprehensive_v3_chrysalis.cfg +zppy -c tests/integration/generated/test_weekly_comprehensive_v2_chrysalis.cfg +zppy -c tests/integration/generated/test_weekly_bundles_chrysalis.cfg # Runs 1st part of bundles cfg + +# TODO: figure out how to wait for all those jobs to finish + +zppy -c tests/integration/generated/test_weekly_bundles_chrysalis.cfg + +# TODO: figure out how to wait for all those jobs to finish + +# Check output +cd /lcrc/group/e3sm/${USER}/zppy_weekly_comprehensive_v3_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts/ grep -v "OK" *status if [ $? == 0 ]; then - # The above command succeeds only if there are reported failures. - echo 'ERROR (2): zppy complete run failed.' - exit 2 + echo 'ERROR (5): weekly_comprehensive_v3 failed' + exit 5 fi -cd ${ZPPY_DIR} -# test_bundles -rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/v2.LR.historical_0201 -rm -rf /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201/post -# Run first set of jobs: -zppy -c tests/integration/generated/test_bundles_chrysalis.cfg -# TODO: how can we possibly tell, automatically, when this is finished? -# bundle1 and bundle2 should run. After they finish, check the results: -cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201/post/scripts + +cd /lcrc/group/e3sm/${USER}/zppy_weekly_comprehensive_v2_output/${UNIQUE_ID}/v2.LR.historical_0201/post/scripts grep -v "OK" *status if [ $? == 0 ]; then - # The above command succeeds only if there are reported failures. - echo 'ERROR (3): zppy bundles 1st run failed.' - exit 3 + echo 'ERROR (6): weekly_comprehensive_v2 failed' + exit 6 fi -cd ${ZPPY_DIR} -# Now, invoke zppy again to run jobs that needed to wait for dependencies: -zppy -c tests/integration/generated/test_bundles_chrysalis.cfg -# TODO: how can we possibly tell, automatically, when this is finished? -# bundle3 and ilamb should run. After they finish, check the results: -cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201/post/scripts + +cd /lcrc/group/e3sm/${USER}/zppy_weekly_bundles_output/${UNIQUE_ID}/v3.LR.historical_0051/post/scripts grep -v "OK" *status if [ $? == 0 ]; then - # The above command succeeds only if there are reported failures. - echo 'ERROR (4): zppy bundles 2nd run failed.' - exit 4 + echo 'ERROR (7): weekly_bundles failed' + exit 7 fi -cd ${ZPPY_DIR} -echo "Run the integration tests" -python -u -m unittest tests/integration/test_*.py +# Run integration tests +cd ~/ez/zppy +pytest tests/integration/test_*.py if [ $? != 0 ]; then - echo 'ERROR (5): integration tests failed' - exit 5 + echo 'ERROR (8): zppy integration tests failed' + exit 8 fi - -echo "All tests passed!"