Skip to content

Commit

Permalink
Automate zppy tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@forsyth2
forsyth2 committed Dec 14, 2024
1 parent ea0a84d commit 6c26e08
Show file tree
Hide file tree
Showing 3 changed files with 276 additions and 11 deletions.
155 changes: 155 additions & 0 deletions tests/integration/generated/test_complete_run_chrysalis.cfg
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
[default]
case = v2.LR.historical_0201
constraint = ""
dry_run = "False"
environment_commands = ""
input = "/lcrc/group/e3sm/ac.forsyth2//E3SMv2/v2.LR.historical_0201"
input_subdir = archive/atm/hist
mapping_file = "map_ne30pg2_to_cmip6_180x360_aave.20200201.nc"
# To run this test, edit `output` and `www` in this file, along with `actual_images_dir` in test_complete_run.py
output = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/v2.LR.historical_0201"
partition = "debug"
qos = "regular"
www = "/lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www"

[climo]
active = True
walltime = "00:30:00"
years = "1850:1854:2", "1850:1854:4",

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"

[[ atm_monthly_diurnal_8xdaily_180x360_aave ]]
frequency = "diurnal_8xdaily"
input_files = "eam.h4"
input_subdir = "archive/atm/hist"
vars = "PRECT"

[ts]
active = True
walltime = "00:30:00"
years = "1850:1854:2",

[[ atm_monthly_180x360_aave ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"
ts_fmt = "cmip"

[[ atm_daily_180x360_aave ]]
frequency = "daily"
input_files = "eam.h1"
input_subdir = "archive/atm/hist"
vars = "PRECT"

[[ atm_monthly_glb ]]
frequency = "monthly"
input_files = "eam.h0"
input_subdir = "archive/atm/hist"
mapping_file = "glb"
years = "1850:1860:5",

[[ land_monthly ]]
extra_vars = "landfrac"
frequency = "monthly"
input_files = "elm.h0"
input_subdir = "archive/lnd/hist"
vars = "FSH,LAISHA,LAISUN,RH2M"
ts_fmt = "cmip"

[[ rof_monthly ]]
extra_vars = 'areatotal2'
frequency = "monthly"
input_files = "mosart.h0"
input_subdir = "archive/rof/hist"
mapping_file = ""
vars = "RIVER_DISCHARGE_OVER_LAND_LIQ"

[tc_analysis]
active = True
scratch = "/lcrc/globalscratch/ac.forsyth2/"
walltime = "00:30:00"
years = "1850:1854:2",

[e3sm_diags]
active = True
grid = '180x360_aave'
ref_final_yr = 2014
ref_start_yr = 1985
# TODO: this directory is missing OMI-MLS
sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere", "tc_analysis",
short_name = 'v2.LR.historical_0201'
ts_num_years = 2
walltime = "00:30:00"
years = "1850:1854:2", "1850:1854:4",

[[ atm_monthly_180x360_aave ]]
climo_diurnal_frequency = "diurnal_8xdaily"
climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave"
partition = "compute"
qos = "regular"
sets = "lat_lon","zonal_mean_xy","zonal_mean_2d","polar","cosp_histogram","meridional_mean_2d","enso_diags","qbo","diurnal_cycle","annual_cycle_zonal_mean","streamflow", "zonal_mean_2d_stratosphere",
walltime = "5:00:00"

[[ atm_monthly_180x360_aave_environment_commands ]]
environment_commands = "source"
sets = "qbo",
ts_subsection = "atm_monthly_180x360_aave"

[[ atm_monthly_180x360_aave_tc_analysis ]]
# Running as its own subtask because tc_analysis requires jobs to run sequentially, which slows down testing
sets = "tc_analysis",
years = "1850:1852:2",

[[ atm_monthly_180x360_aave_mvm ]]
# Test model-vs-model using the same files as the reference
climo_diurnal_frequency = "diurnal_8xdaily"
climo_diurnal_subsection = "atm_monthly_diurnal_8xdaily_180x360_aave"
climo_subsection = "atm_monthly_180x360_aave"
diff_title = "Difference"
partition = "compute"
qos = "regular"
ref_final_yr = 1851
ref_name = "v2.LR.historical_0201"
ref_start_yr = 1850
ref_years = "1850-1851",
reference_data_path = "/lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/v2.LR.historical_0201/post/atm/180x360_aave/clim"
run_type = "model_vs_model"
short_ref_name = "v2.LR.historical_0201"
swap_test_ref = False
tag = "model_vs_model"
ts_num_years_ref = 2
ts_subsection = "atm_monthly_180x360_aave"
walltime = "5:00:00"
years = "1852-1853",

[mpas_analysis]
active = True
anomalyRefYear = 1850
climo_years ="1850-1854", "1855-1860",
enso_years = "1850-1854", "1855-1860",
mesh = "EC30to60E2r2"
parallelTaskCount = 6
partition = "compute"
qos = "regular"
ts_years = "1850-1854", "1850-1860",
walltime = "00:30:00"

[global_time_series]
active = True
climo_years ="1850-1854", "1855-1860",
experiment_name = "v2.LR.historical_0201"
figstr = "v2_historical_0201"
moc_file=mocTimeSeries_1850-1860.nc
ts_num_years = 5
ts_years = "1850-1854", "1850-1860",
walltime = "00:30:00"
years = "1850-1860",

[ilamb]
active = True
grid = '180x360_aave'
short_name = 'v2.LR.historical_0201'
ts_num_years = 2
years = "1850:1854:2",
23 changes: 12 additions & 11 deletions tests/integration/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import re
import shutil
import subprocess
import sys
from typing import List

from mache import MachineInfo
Expand Down Expand Up @@ -144,9 +145,6 @@ def get_chyrsalis_expansions(config):
"case_name": "v3.LR.historical_0051",
"case_name_v2": "v2.LR.historical_0201",
"constraint": "",
# To run this test, replace conda environment with your e3sm_diags dev environment
# To use default environment_commands, set to ""
"diags_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
"diags_walltime": "5:00:00",
"e3sm_to_cmip_environment_commands": "",
"environment_commands_test": "",
Expand Down Expand Up @@ -174,9 +172,6 @@ def get_compy_expansions(config):
"case_name": "v3.LR.historical_0051",
"case_name_v2": "v2.LR.historical_0201",
"constraint": "",
# To run this test, replace conda environment with your e3sm_diags dev environment
# To use default environment_commands, set to ""
"diags_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
"diags_walltime": "03:00:00",
"e3sm_to_cmip_environment_commands": "",
"environment_commands_test": "",
Expand Down Expand Up @@ -204,9 +199,6 @@ def get_perlmutter_expansions(config):
"case_name": "v3.LR.historical_0051",
"case_name_v2": "v2.LR.historical_0201",
"constraint": "cpu",
# To run this test, replace conda environment with your e3sm_diags dev environment
# To use default environment_commands, set to ""
"diags_environment_commands": "source <INSERT PATH TO CONDA>/conda.sh; conda activate <INSERT ENV NAME>",
"diags_walltime": "6:00:00",
"e3sm_to_cmip_environment_commands": "",
"environment_commands_test": "",
Expand Down Expand Up @@ -260,19 +252,26 @@ def substitute_expansions(expansions, file_in, file_out):
file_write.write(line)


def generate_cfgs(unified_testing=False, dry_run=False):
def generate_cfgs(unified_testing=False, diags_environment_commands="", dry_run=False):
git_top_level = (
subprocess.check_output("git rev-parse --show-toplevel".split())
.strip()
.decode("utf-8")
)
expansions = get_expansions()

if unified_testing:
expansions["environment_commands"] = expansions["environment_commands_test"]
else:
# The cfg doesn't need this line,
# but it would be difficult to only write environment_commands in the unified_testing case.
expansions["environment_commands"] = ""

if diags_environment_commands:
expansions["diags_environment_commands"] = diags_environment_commands
else:
expansions["diags_environment_commands"] = ""

machine = expansions["machine"]

if dry_run:
Expand Down Expand Up @@ -369,4 +368,6 @@ def generate_cfgs(unified_testing=False, dry_run=False):


if __name__ == "__main__":
generate_cfgs(unified_testing=False, dry_run=False)
generate_cfgs(
unified_testing=(sys.argv[1] == "True"), diags_environment_commands=sys.argv[2]
)
109 changes: 109 additions & 0 deletions tests/scripts/test_dev.bash
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
#!/bin/bash

# Before running this script ########################################################

# Make sure you're on the branch you want to test!

# If you want to test `main`, do the following:
# git fetch upstream main
# git checkout -b test_pre_zppy_rc<#>_<machine_name> upstream/main
# git log # check the commits match https://github.com/E3SM-Project/zppy/commits/main

# Set these parameters
DIAGS_DIR=/home/ac.forsyth2/e3sm_diags/
DIAGS_DEV=diags_dev_2023_10_05
ZPPY_DIR=/home/ac.forsyth2/zppy/
DIAGS_ENV_CMD="source /home/ac.forsyth2/miniconda3/etc/profile.d/conda.sh; conda activate ${DIAGS_DEV}"
ZPPY_DEV=zppy_dev_n516

# Make sure you do not have important changes on the `main` branch in your
# E3SM_DIAGS_DIRECTORY. This script will reset that branch to match `upstream`!

#####################################################################################

echo "Update E3SM Diags"
# `cd` to e3sm_diags directory
cd ${DIAGS_DIR}
git checkout main
git fetch upstream
git reset --hard upstream/main
git log # Should match https://github.com/E3SM-Project/e3sm_diags/commits/main # TODO: Requires user review
mamba clean --all # TODO: Requires user input to advance
conda remove -n ${DIAGS_DEV} --all
mamba env create -f conda-env/dev.yml -n ${DIAGS_DEV}
conda activate ${DIAGS_DEV} # TODO: errors with ./tests/scripts/test_dev.bash: line 34: {DIAGS_DEV}: command not found
pip install .
cd ${ZPPY_DIR}

echo "Make sure we're using the latest packages"
UNIFIED_TESTING=False
python tests/integration/utils.py ${UNIFIED_TESTING} ${DIAGS_ENV_CMD}

echo "Set up our environment"
mamba clean --all # TODO: Requires user input to advance
mamba env create -f conda/dev.yml -n ${ZPPY_DEV}
conda activate ${ZPPY_DEV} # TODO: errors with CommandNotFoundError: Your shell has not been properly configured to use 'conda activate'.
pip install .

echo "Run unit tests"
python -u -m unittest tests/test_*.py
if [ $? != 0 ]; then
echo 'ERROR (1): unit tests failed'
exit 1
fi

exit 7

echo "Set up integration tests"
# TODO: somehow use Mache (a Python package) to get machine-independent paths in this bash script!
# test_complete_run
rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_complete_run_www/v2.LR.historical_0201
rm -rf /lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/v2.LR.historical_0201/post
# Run jobs:
zppy -c tests/integration/generated/test_complete_run_chrysalis.cfg
# TODO: how can we possibly tell, automatically, when this is finished?
# After they finish, check the results:
cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_complete_run_output/v2.LR.historical_0201/post/scripts
grep -v "OK" *status
if [ $? == 0 ]; then
# The above command succeeds only if there are reported failures.
echo 'ERROR (2): zppy complete run failed.'
exit 2
fi
cd ${ZPPY_DIR}
# test_bundles
rm -rf /lcrc/group/e3sm/public_html/diagnostic_output/ac.forsyth2/zppy_test_bundles_www/v2.LR.historical_0201
rm -rf /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201/post
# Run first set of jobs:
zppy -c tests/integration/generated/test_bundles_chrysalis.cfg
# TODO: how can we possibly tell, automatically, when this is finished?
# bundle1 and bundle2 should run. After they finish, check the results:
cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201/post/scripts
grep -v "OK" *status
if [ $? == 0 ]; then
# The above command succeeds only if there are reported failures.
echo 'ERROR (3): zppy bundles 1st run failed.'
exit 3
fi
cd ${ZPPY_DIR}
# Now, invoke zppy again to run jobs that needed to wait for dependencies:
zppy -c tests/integration/generated/test_bundles_chrysalis.cfg
# TODO: how can we possibly tell, automatically, when this is finished?
# bundle3 and ilamb should run. After they finish, check the results:
cd /lcrc/group/e3sm/ac.forsyth2/zppy_test_bundles_output/v2.LR.historical_0201/post/scripts
grep -v "OK" *status
if [ $? == 0 ]; then
# The above command succeeds only if there are reported failures.
echo 'ERROR (4): zppy bundles 2nd run failed.'
exit 4
fi
cd ${ZPPY_DIR}

echo "Run the integration tests"
python -u -m unittest tests/integration/test_*.py
if [ $? != 0 ]; then
echo 'ERROR (5): integration tests failed'
exit 5
fi

echo "All tests passed!"

0 comments on commit 6c26e08

Please sign in to comment.