From 20e49ad8c780b42888a8bb51637fd19f8e84714b Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Sat, 20 Apr 2024 09:55:51 -0700 Subject: [PATCH 01/46] Updates mvk test to be configured through testmod --- CIME/SystemTests/mvk.py | 156 ++++++++++++++++++++++++++++++++-------- 1 file changed, 127 insertions(+), 29 deletions(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 2ab2f72cd33..d73d52a221c 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -17,14 +17,71 @@ from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case.case_setup import case_setup from CIME.XML.machines import Machines - +from CIME.config import ConfigBase +from CIME.utils import parse_test_name +from CIME.utils import CIMEError +from CIME.XML.files import Files import evv4esm # pylint: disable=import-error from evv4esm.__main__ import main as evv # pylint: disable=import-error evv_lib_dir = os.path.abspath(os.path.dirname(evv4esm.__file__)) + logger = logging.getLogger(__name__) -NINST = 30 + + +class MVKConfig(ConfigBase): + def __init__(self): + super().__init__() + + if self.loaded: + return + + self._set_attribute("component", None, "Main component") + self._set_attribute("ninst", 30, "Number of instances") + + def write_inst_nml(self, case, write_line, iinst): + """Write per instance namelist. + + This method is called once per instance. + + Args: + case (CIME.case.case.Case): The case instance. + write_line (function): Function takes single `str` argument. + iinst (int): Instance unique number. + """ + write_line("new_random = .true.") + write_line("pertlim = 1.0e-10") + write_line("seed_custom = {}".format(iinst)) + write_line("seed_clock = .true.") + + def test_config(self, case, run_dir, base_dir, evv_lib_dir): + """Configure the evv test. + + This method is used to pass the evv4esm configuration to be written for the test. + + Args: + case (CIME.case.case.Case): The case instance. + run_dir (str): Path the case's run directory. + base_dir (str): Path to the case's baseline directory. + evv_lib_dir (str): Path to the evv4esm package root. + + Returns: + dict: Dictionary with test configuration. + """ + config = { + "module": os.path.join(evv_lib_dir, "extensions", "ks.py"), + "test-case": "Test", + "test-dir": run_dir, + "ref-case": "Baseline", + "ref-dir": base_dir, + "var-set": "default", + "ninst": self.ninst, + "critical": 13, + "component": self.component, + } + + return config class MVK(SystemTestsCommon): @@ -34,10 +91,56 @@ def __init__(self, case, **kwargs): """ SystemTestsCommon.__init__(self, case, **kwargs) - if self._case.get_value("MODEL") == "e3sm": - self.component = "eam" + casebaseid = self._case.get_value("CASEBASEID") + + *_, test_mods = parse_test_name(casebaseid) + + if test_mods: + comp_interface = self._case.get_value("COMP_INTERFACE") + + files = Files(comp_interface=comp_interface) + + for mod in test_mods: + if mod.find("/") == -1: + raise CIMEError( + "Missing testmod component. Testmods are specified as '${component}-${testmod}" + ) + else: + component, modpath = mod.split("/", 1) + + testmods_dir = files.get_value( + "TESTS_MODS_DIR", {"component": component} + ) + + test_mod_file = os.path.join(testmods_dir, component, modpath) + + if not os.path.exists(test_mod_file): + usermods_dir = files.get_value( + "USER_MODS_DIR", {"component": component} + ) + + test_mod_file = os.path.join(usermods_dir, component, modpath) + + if not os.path.exists(test_mod_file): + raise CIMEError( + "Missing testmod file {!r}, checked {} and {}".format( + modpath, testmods_dir, usermods_dir + ) + ) + + self._config = MVKConfig.load(test_mod_file) else: - self.component = "cam" + self._config = MVKConfig() + + # Use old behavior for component + if self._config.component is None: + # TODO remove model specific + if self._case.get_value("MODEL") == "e3sm": + self.component = "eam" + else: + self.component = "cam" + else: + self.component = self._config.component if ( self._case.get_value("RESUBMIT") == 0 @@ -57,22 +160,24 @@ def build_phase(self, sharedlib_only=False, model_only=False): ntasks = self._case.get_value("NTASKS_{}".format(comp)) - self._case.set_value("NTASKS_{}".format(comp), ntasks * NINST) + self._case.set_value( + "NTASKS_{}".format(comp), ntasks * self._config.ninst + ) + if comp != "CPL": - self._case.set_value("NINST_{}".format(comp), NINST) + self._case.set_value("NINST_{}".format(comp), self._config.ninst) self._case.flush() case_setup(self._case, test_mode=False, reset=True) - for iinst in range(1, NINST + 1): + for iinst in range(1, self._config.ninst + 1): with open( "user_nl_{}_{:04d}".format(self.component, iinst), "w" - ) as nl_atm_file: - nl_atm_file.write("new_random = .true.\n") - nl_atm_file.write("pertlim = 1.0e-10\n") - nl_atm_file.write("seed_custom = {}\n".format(iinst)) - nl_atm_file.write("seed_clock = .true.\n") + ) as nml_file: + write_line = lambda x: nml_file.write(f"{x}\n") + + self._config.write_inst_nml(self._case, write_line, iinst) self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) @@ -128,25 +233,18 @@ def _compare_baseline(self): ) test_name = "{}".format(case_name.split(".")[-1]) - evv_config = { - test_name: { - "module": os.path.join(evv_lib_dir, "extensions", "ks.py"), - "test-case": "Test", - "test-dir": run_dir, - "ref-case": "Baseline", - "ref-dir": base_dir, - "var-set": "default", - "ninst": NINST, - "critical": 13, - "component": self.component, - } - } - - json_file = os.path.join(run_dir, ".".join([case_name, "json"])) + + test_config = self._config.test_config( + self._case, run_dir, base_dir, evv_lib_dir + ) + + evv_config = {test_name: test_config} + + json_file = os.path.join(run_dir, f"{case_name}.json") with open(json_file, "w") as config_file: json.dump(evv_config, config_file, indent=4) - evv_out_dir = os.path.join(run_dir, ".".join([case_name, "evv"])) + evv_out_dir = os.path.join(run_dir, f"{case_name}.evv") evv(["-e", json_file, "-o", evv_out_dir]) with open(os.path.join(evv_out_dir, "index.json")) as evv_f: From 4d614ed686b234c68fc3eb3367c50854159ea47e Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 3 May 2024 17:47:24 -0700 Subject: [PATCH 02/46] Fixes default component value --- CIME/SystemTests/mvk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index d73d52a221c..66440389eac 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -37,7 +37,7 @@ def __init__(self): if self.loaded: return - self._set_attribute("component", None, "Main component") + self._set_attribute("component", "", "Main component") self._set_attribute("ninst", 30, "Number of instances") def write_inst_nml(self, case, write_line, iinst): @@ -133,7 +133,7 @@ def __init__(self, case, **kwargs): self._config = MVKConfig() # Use old behavior for component - if self._config.component is None: + if self._config.component == "": # TODO remove model specific if self._case.get_value("MODEL") == "e3sm": self.component = "eam" From 7b59a191b4ef3bd8d640ba2717b207b3f686c603 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 3 May 2024 19:45:22 -0700 Subject: [PATCH 03/46] Refactors print_rst_table and adds method documentation --- CIME/config.py | 105 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 79 insertions(+), 26 deletions(-) diff --git a/CIME/config.py b/CIME/config.py index d2306d354d0..9fa28cd109d 100644 --- a/CIME/config.py +++ b/CIME/config.py @@ -3,12 +3,50 @@ import logging import importlib.machinery import importlib.util +import inspect from CIME import utils logger = logging.getLogger(__name__) +def print_rst_header(header): + n = len(header) + print("-" * n) + print(header) + print("-" * n) + + +def print_rst_table(headers, *rows): + column_widths = [] + + columns = [[rows[y][x] for y in range(len(rows))] for x in range(len(rows[0]))] + + for header, column in zip(headers, columns): + column_widths.append( + max( + [ + len(x) + for x in [ + header, + ] + + column + ] + ) + ) + + divider = " ".join([f"{'=' * x}" for x in column_widths]) + + print(divider) + print(" ".join(f"{y}{' ' * (x - len(y))}" for x, y in zip(column_widths, headers))) + print(divider) + + for row in rows: + print(" ".join([f"{y}{' ' * (x-len(y))}" for x, y in zip(column_widths, row)])) + + print(divider) + + class ConfigBase: def __new__(cls): if not hasattr(cls, "_instance"): @@ -93,39 +131,54 @@ def _set_attribute(self, name, value, desc=None): } def print_rst_table(self): - max_variable = max([len(x) for x in self._attribute_config.keys()]) - max_default = max( - [len(str(x["default"])) for x in self._attribute_config.values()] - ) - max_type = max( - [len(type(x["default"]).__name__) for x in self._attribute_config.values()] - ) - max_desc = max([len(x["desc"]) for x in self._attribute_config.values()]) + self.print_variable_rst() + + print("") + + self.print_method_rst() + + def print_variable_rst(self): + print_rst_header("Variables") + + headers = ("Variable", "Default", "Type", "Description") - divider_row = ( - f"{'='*max_variable} {'='*max_default} {'='*max_type} {'='*max_desc}" + rows = ( + (x, str(y["default"]), type(y["default"]).__name__, y["desc"]) + for x, y in self._attribute_config.items() ) - rows = [ - divider_row, - f"Variable{' '*(max_variable-8)} Default{' '*(max_default-7)} Type{' '*(max_type-4)} Description{' '*(max_desc-11)}", - divider_row, - ] + print_rst_table(headers, *rows) - for variable, value in sorted( - self._attribute_config.items(), key=lambda x: x[0] - ): - variable_fill = max_variable - len(variable) - default_fill = max_default - len(str(value["default"])) - type_fill = max_type - len(type(value["default"]).__name__) + def print_method_rst(self): + print_rst_header("Methods") - rows.append( - f"{variable}{' '*variable_fill} {value['default']}{' '*default_fill} {type(value['default']).__name__}{' '*type_fill} {value['desc']}" - ) + methods = inspect.getmembers(self, lambda x: inspect.ismethod(x)) - rows.append(divider_row) + ignore = ( + "__init__", + "loaded", + "load", + "instance", + "_load_file", + "_set_attribute", + "print_rst_table", + "print_method_rst", + "print_variable_rst", + ) + + child_methods = [ + (x[0], inspect.signature(x[1]), inspect.getdoc(x[1])) + for x in methods + if x[1].__class__ != Config and x[0] not in ignore + ] - print("\n".join(rows)) + for (name, sig, doc) in child_methods: + print(".. code-block::\n") + print(f" def {name}{sig!s}:") + print(' """') + for line in doc.split("\n"): + print(f" {line}") + print(' """') class Config(ConfigBase): From 8fe4f8a64db22162b12459650c5481a7949c46b6 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 3 May 2024 20:24:40 -0700 Subject: [PATCH 04/46] Exposes all default configuration values --- CIME/SystemTests/mvk.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 66440389eac..163a53d3d51 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -25,7 +25,7 @@ import evv4esm # pylint: disable=import-error from evv4esm.__main__ import main as evv # pylint: disable=import-error -evv_lib_dir = os.path.abspath(os.path.dirname(evv4esm.__file__)) +EVV_LIB_DIR = os.path.abspath(os.path.dirname(evv4esm.__file__)) logger = logging.getLogger(__name__) @@ -37,13 +37,21 @@ def __init__(self): if self.loaded: return - self._set_attribute("component", "", "Main component") - self._set_attribute("ninst", 30, "Number of instances") + self._set_attribute("component", "", "Model component name.") + self._set_attribute("ninst", 30, "The number of instances.") + self._set_attribute( + "critical", 13, "The critical value for rejecting the null hypothese." + ) + self._set_attribute( + "var_set", "default", "Name of the variable set to analyze." + ) + self._set_attribute("ref_case", "Baseline", "Name of the reference case.") + self._set_attribute("test_case", "Test", "Name of the test case.") def write_inst_nml(self, case, write_line, iinst): """Write per instance namelist. - This method is called once per instance. + This method is called once per instance to generate the namelist. Args: case (CIME.case.case.Case): The case instance. @@ -71,13 +79,13 @@ def test_config(self, case, run_dir, base_dir, evv_lib_dir): """ config = { "module": os.path.join(evv_lib_dir, "extensions", "ks.py"), - "test-case": "Test", + "test-case": self.test_case, "test-dir": run_dir, - "ref-case": "Baseline", + "ref-case": self.ref_case, "ref-dir": base_dir, - "var-set": "default", + "var-set": self.var_set, "ninst": self.ninst, - "critical": 13, + "critical": self.critical, "component": self.component, } @@ -235,7 +243,7 @@ def _compare_baseline(self): test_name = "{}".format(case_name.split(".")[-1]) test_config = self._config.test_config( - self._case, run_dir, base_dir, evv_lib_dir + self._case, run_dir, base_dir, EVV_LIB_DIR ) evv_config = {test_name: test_config} From 6c23f74dfb433e339617e45786b67a25ef8ca155 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 3 May 2024 20:42:25 -0700 Subject: [PATCH 05/46] Fixes lint errors --- CIME/SystemTests/mvk.py | 16 ++++++++++++---- CIME/config.py | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 163a53d3d51..ece5e185185 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -48,7 +48,9 @@ def __init__(self): self._set_attribute("ref_case", "Baseline", "Name of the reference case.") self._set_attribute("test_case", "Test", "Name of the test case.") - def write_inst_nml(self, case, write_line, iinst): + def write_inst_nml( + self, case, write_line, iinst + ): # pylint: disable=unused-argument """Write per instance namelist. This method is called once per instance to generate the namelist. @@ -63,7 +65,9 @@ def write_inst_nml(self, case, write_line, iinst): write_line("seed_custom = {}".format(iinst)) write_line("seed_clock = .true.") - def test_config(self, case, run_dir, base_dir, evv_lib_dir): + def test_config( + self, case, run_dir, base_dir, evv_lib_dir + ): # pylint: disable=unused-argument """Configure the evv test. This method is used to pass the evv4esm configuration to be written for the test. @@ -108,7 +112,7 @@ def __init__(self, case, **kwargs): files = Files(comp_interface=comp_interface) - for mod in test_mods: + for mod in test_mods: # pylint: disable=not-an-iterable if mod.find("/") == -1: raise CIMEError( "Missing testmod component. Testmods are specified as '${component}-${testmod}" @@ -183,7 +187,11 @@ def build_phase(self, sharedlib_only=False, model_only=False): with open( "user_nl_{}_{:04d}".format(self.component, iinst), "w" ) as nml_file: - write_line = lambda x: nml_file.write(f"{x}\n") + write_line = ( + lambda x: nml_file.write( # pylint: disable=cell-var-from-loop + f"{x}\n" + ) + ) self._config.write_inst_nml(self._case, write_line, iinst) diff --git a/CIME/config.py b/CIME/config.py index 9fa28cd109d..ea6eb62530d 100644 --- a/CIME/config.py +++ b/CIME/config.py @@ -152,7 +152,7 @@ def print_variable_rst(self): def print_method_rst(self): print_rst_header("Methods") - methods = inspect.getmembers(self, lambda x: inspect.ismethod(x)) + methods = inspect.getmembers(self, inspect.ismethod) ignore = ( "__init__", From a7ff10ddc365fe22b0fb67608439e6845d37ec63 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 3 May 2024 21:03:11 -0700 Subject: [PATCH 06/46] Fixes loading only params.py file from testmod --- CIME/SystemTests/mvk.py | 9 +++++++-- CIME/config.py | 16 +++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index ece5e185185..733d10f317f 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -11,6 +11,7 @@ import logging from distutils import dir_util +from pathlib import Path import CIME.test_status import CIME.utils @@ -140,8 +141,12 @@ def __init__(self, case, **kwargs): ) ) - self._config = MVKConfig.load(test_mod_file) - else: + params_file = Path(test_mod_file, "params.py") + + if params_file.exists(): + self._config = MVKConfig.load(test_mod_file) + + if self._config is None: self._config = MVKConfig() # Use old behavior for component diff --git a/CIME/config.py b/CIME/config.py index ea6eb62530d..26318dedb9b 100644 --- a/CIME/config.py +++ b/CIME/config.py @@ -4,6 +4,7 @@ import importlib.machinery import importlib.util import inspect +from pathlib import Path from CIME import utils @@ -75,12 +76,17 @@ def load(cls, customize_path): logger.debug("Searching %r for files to load", customize_path) - customize_files = glob.glob(f"{customize_path}/**/*.py", recursive=True) + customize_path = Path(customize_path) - # filter out any tests - customize_files = [ - x for x in customize_files if "tests" not in x and "conftest" not in x - ] + if customize_path.is_file(): + customize_files = [f"{customize_path}"] + else: + customize_files = glob.glob(f"{customize_path}/**/*.py", recursive=True) + + # filter out any tests + customize_files = [ + x for x in customize_files if "tests" not in x and "conftest" not in x + ] customize_module_spec = importlib.machinery.ModuleSpec("cime_customize", None) From 058c974073ec84c674dd8af5793020be77badbb7 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 3 May 2024 21:08:35 -0700 Subject: [PATCH 07/46] Renames write_line to set_nml_variable --- CIME/SystemTests/mvk.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 733d10f317f..78ec3d7ba3c 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -50,7 +50,7 @@ def __init__(self): self._set_attribute("test_case", "Test", "Name of the test case.") def write_inst_nml( - self, case, write_line, iinst + self, case, set_nml_variable, iinst ): # pylint: disable=unused-argument """Write per instance namelist. @@ -58,13 +58,13 @@ def write_inst_nml( Args: case (CIME.case.case.Case): The case instance. - write_line (function): Function takes single `str` argument. + write_nml_variable (function): Function takes two `str` arguments. iinst (int): Instance unique number. """ - write_line("new_random = .true.") - write_line("pertlim = 1.0e-10") - write_line("seed_custom = {}".format(iinst)) - write_line("seed_clock = .true.") + set_nml_variable("new_random", ".true.") + set_nml_variable("pertlim", "1.0e-10") + set_nml_variable("seed_custom", f"{iinst}") + set_nml_variable("seed_clock", ".true.") def test_config( self, case, run_dir, base_dir, evv_lib_dir @@ -192,13 +192,13 @@ def build_phase(self, sharedlib_only=False, model_only=False): with open( "user_nl_{}_{:04d}".format(self.component, iinst), "w" ) as nml_file: - write_line = ( + set_nml_variable = ( lambda x: nml_file.write( # pylint: disable=cell-var-from-loop f"{x}\n" ) ) - self._config.write_inst_nml(self._case, write_line, iinst) + self._config.write_inst_nml(self._case, set_nml_variable, iinst) self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) From f6b2c723025d464a02b3317bca21300344d53877 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 10 May 2024 12:10:36 -0700 Subject: [PATCH 08/46] Adds ability to generate namelists for multiple components --- CIME/SystemTests/mvk.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 78ec3d7ba3c..bb59d5d9b93 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -38,7 +38,10 @@ def __init__(self): if self.loaded: return - self._set_attribute("component", "", "Model component name.") + self._set_attribute("component", "", "The main component.") + self._set_attribute( + "components", [], "Components that require namelist customization." + ) self._set_attribute("ninst", 30, "The number of instances.") self._set_attribute( "critical", 13, "The critical value for rejecting the null hypothese." @@ -50,7 +53,7 @@ def __init__(self): self._set_attribute("test_case", "Test", "Name of the test case.") def write_inst_nml( - self, case, set_nml_variable, iinst + self, case, set_nml_variable, component, iinst ): # pylint: disable=unused-argument """Write per instance namelist. @@ -59,6 +62,7 @@ def write_inst_nml( Args: case (CIME.case.case.Case): The case instance. write_nml_variable (function): Function takes two `str` arguments. + component (str): Component the namelist belongs to. iinst (int): Instance unique number. """ set_nml_variable("new_random", ".true.") @@ -159,6 +163,11 @@ def __init__(self, case, **kwargs): else: self.component = self._config.component + if self._config.components: + self.components = [self.component] + else: + self.components = self._config.components + if ( self._case.get_value("RESUBMIT") == 0 and self._case.get_value("GENERATE_BASELINE") is False @@ -189,16 +198,19 @@ def build_phase(self, sharedlib_only=False, model_only=False): case_setup(self._case, test_mode=False, reset=True) for iinst in range(1, self._config.ninst + 1): - with open( - "user_nl_{}_{:04d}".format(self.component, iinst), "w" - ) as nml_file: - set_nml_variable = ( - lambda x: nml_file.write( # pylint: disable=cell-var-from-loop - f"{x}\n" + for component in self.components: + with open( + "user_nl_{}_{:04d}".format(component, iinst), "w" + ) as nml_file: + set_nml_variable = ( + lambda x: nml_file.write( # pylint: disable=cell-var-from-loop + f"{x}\n" + ) ) - ) - self._config.write_inst_nml(self._case, set_nml_variable, iinst) + self._config.write_inst_nml( + self._case, set_nml_variable, component, iinst + ) self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) From 6ffc92ac3ac21b9686c8d7333000f66cc151273e Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 10 May 2024 12:17:16 -0700 Subject: [PATCH 09/46] Fixes customize files glob --- CIME/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CIME/config.py b/CIME/config.py index 8d0fb545c2b..f61b1552693 100644 --- a/CIME/config.py +++ b/CIME/config.py @@ -79,11 +79,13 @@ def load(cls, customize_path): customize_path = Path(customize_path) - if customize_path.is_file() + if customize_path.is_file(): customize_files = [f"{customize_path}"] else: ignore_pattern = re.compile(f"{customize_path}/(?:tests|conftest|test_)") + customize_files = customize_path.glob("**/*.py") + # filter out any tests customize_files = [ x for x in customize_files if ignore_pattern.search(x) is None From 9366309094f83be152af2f0528d4f3a466b96b91 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 10 May 2024 12:18:25 -0700 Subject: [PATCH 10/46] Removes import --- CIME/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/CIME/config.py b/CIME/config.py index f61b1552693..b93d208e5d6 100644 --- a/CIME/config.py +++ b/CIME/config.py @@ -1,6 +1,5 @@ import re import sys -import glob import logging import importlib.machinery import importlib.util From 25c56bc8b1a01d4eec6d8c95db60a445e2479aa9 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 10 May 2024 12:27:54 -0700 Subject: [PATCH 11/46] Fixes glob --- CIME/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CIME/config.py b/CIME/config.py index b93d208e5d6..41ce40594a2 100644 --- a/CIME/config.py +++ b/CIME/config.py @@ -83,11 +83,11 @@ def load(cls, customize_path): else: ignore_pattern = re.compile(f"{customize_path}/(?:tests|conftest|test_)") - customize_files = customize_path.glob("**/*.py") - # filter out any tests customize_files = [ - x for x in customize_files if ignore_pattern.search(x) is None + f"{x}" + for x in customize_path.glob("**/*.py") + if ignore_pattern.search(x) is None ] customize_module_spec = importlib.machinery.ModuleSpec("cime_customize", None) From 10af6ea01dfc851352ebb67a4b87f22ff5eccfd6 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Tue, 14 May 2024 19:06:11 -0700 Subject: [PATCH 12/46] Fixes converting Path to str --- CIME/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/config.py b/CIME/config.py index 41ce40594a2..b8440493884 100644 --- a/CIME/config.py +++ b/CIME/config.py @@ -87,7 +87,7 @@ def load(cls, customize_path): customize_files = [ f"{x}" for x in customize_path.glob("**/*.py") - if ignore_pattern.search(x) is None + if ignore_pattern.search(f"{x}") is None ] customize_module_spec = importlib.machinery.ModuleSpec("cime_customize", None) From 15b314f43c4b0517877a5aefe80514541e46c5b1 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 15 May 2024 14:33:05 -0700 Subject: [PATCH 13/46] Adds initial mvk unit test --- CIME/SystemTests/mvk.py | 4 +++- CIME/tests/test_unit_system_tests_mvk.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 CIME/tests/test_unit_system_tests_mvk.py diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index bb59d5d9b93..e3f76e170a7 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -108,6 +108,8 @@ def __init__(self, case, **kwargs): """ SystemTestsCommon.__init__(self, case, **kwargs) + self._config = None + casebaseid = self._case.get_value("CASEBASEID") *_, test_mods = parse_test_name(casebaseid) @@ -163,7 +165,7 @@ def __init__(self, case, **kwargs): else: self.component = self._config.component - if self._config.components: + if len(self._config.components) == 0: self.components = [self.component] else: self.components = self._config.components diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py new file mode 100644 index 00000000000..803478c74ee --- /dev/null +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +import unittest +from unittest import mock + +from CIME.SystemTests.mvk import MVK + +class TestSystemTestsMVK(unittest.TestCase): + def test_mvk(self): + case = mock.MagicMock() + case.get_value.side_effect = ( + "/tmp/case", # CASEROOT + "MVK.f19_g16.S.docker_gnu", # CASEBASEID + "mct", # COMP_INTERFACE + "MVK.f19_g16.S.docker_gnu", # CASEBASEID + "e3sm", # MODEL + 0, # RESUBMIT + False, # GENERATE_BASELINE + ) + + test = MVK(case) + + assert test.component == "eam" + assert test.components == ["eam"] From bf19cab32f984a5f2256b53af05f0ab63232bd95 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 15 May 2024 17:14:30 -0700 Subject: [PATCH 14/46] Refactors testmod search to find_test_mods --- CIME/SystemTests/mvk.py | 50 ++---------- CIME/SystemTests/test_mods.py | 81 ++++++++++++++++++++ CIME/test_scheduler.py | 38 +++------ CIME/tests/test_sys_test_scheduler.py | 98 ++++++++++++++++++++---- CIME/tests/test_unit_system_tests_mvk.py | 15 ++-- 5 files changed, 191 insertions(+), 91 deletions(-) create mode 100644 CIME/SystemTests/test_mods.py diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index e3f76e170a7..e5060eb9263 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -9,9 +9,7 @@ import os import json import logging - from distutils import dir_util -from pathlib import Path import CIME.test_status import CIME.utils @@ -20,8 +18,7 @@ from CIME.XML.machines import Machines from CIME.config import ConfigBase from CIME.utils import parse_test_name -from CIME.utils import CIMEError -from CIME.XML.files import Files +from CIME.SystemTests.test_mods import find_test_mods import evv4esm # pylint: disable=import-error from evv4esm.__main__ import main as evv # pylint: disable=import-error @@ -106,51 +103,16 @@ def __init__(self, case, **kwargs): """ initialize an object interface to the MVK test """ - SystemTestsCommon.__init__(self, case, **kwargs) - self._config = None - casebaseid = self._case.get_value("CASEBASEID") - - *_, test_mods = parse_test_name(casebaseid) - - if test_mods: - comp_interface = self._case.get_value("COMP_INTERFACE") - - files = Files(comp_interface=comp_interface) - - for mod in test_mods: # pylint: disable=not-an-iterable - if mod.find("/") == -1: - raise CIMEError( - "Missing testmod component. Testmods are specified as '${component}-${testmod}" - ) - else: - component, modpath = mod.split("/", 1) - - testmods_dir = files.get_value( - "TESTS_MODS_DIR", {"component": component} - ) - - test_mod_file = os.path.join(testmods_dir, component, modpath) - - if not os.path.exists(test_mod_file): - usermods_dir = files.get_value( - "USER_MODS_DIR", {"component": component} - ) - - test_mod_file = os.path.join(usermods_dir, component, modpath) + SystemTestsCommon.__init__(self, case, **kwargs) - if not os.path.exists(test_mod_file): - raise CIMEError( - "Missing testmod file {!r}, checked {} and {}".format( - modpath, testmods_dir, usermods_dir - ) - ) + *_, test_mods = parse_test_name(self._casebaseid) - params_file = Path(test_mod_file, "params.py") + test_mods_paths = find_test_mods(case.get_value("COMP_INTERFACE"), test_mods) - if params_file.exists(): - self._config = MVKConfig.load(test_mod_file) + for test_mods_path in test_mods_paths: + self._config = MVKConfig.load(test_mods_path) if self._config is None: self._config = MVKConfig() diff --git a/CIME/SystemTests/test_mods.py b/CIME/SystemTests/test_mods.py new file mode 100644 index 00000000000..0f163280099 --- /dev/null +++ b/CIME/SystemTests/test_mods.py @@ -0,0 +1,81 @@ +import logging +import os + +from CIME.utils import CIMEError +from CIME.XML.files import Files + +logger = logging.getLogger(__name__) + +MODS_DIR_VARS = ("TESTS_MODS_DIR", "USER_MODS_DIR") + + +def find_test_mods(comp_interface, test_mods): + """Finds paths from names of testmods. + + Testmod format is `${component}-${testmod}`. Each testmod is search for + it it's component respective `TESTS_MODS_DIR` and `USER_MODS_DIR`. + + Args: + comp_interface (str): Name of the component interface. + test_mods (list): List of testmods names. + + Returns: + List of paths for each testmod. + + Raises: + CIMEError: If a testmod is not in correct format. + CIMEError: If testmod could not be found. + """ + if test_mods is None: + return [] + + files = Files(comp_interface=comp_interface) + + test_mods_paths = [] + + logger.debug("Checking for testmods {}".format(test_mods)) + + for test_mod in test_mods: + if test_mod.find("/") != -1: + component, mod_path = test_mod.split("/", 1) + else: + raise CIMEError( + f"Invalid testmod, format should be `${{component}}-${{testmod}}`, got {test_mod!r}" + ) + + logger.info( + "Searching for testmod {!r} for component {!r}".format(mod_path, component) + ) + + test_mod_path = None + + for var in MODS_DIR_VARS: + mods_dir = files.get_value(var, {"component": component}) + + try: + candidate_path = os.path.join(mods_dir, component, mod_path) + except TypeError: + # mods_dir is None + continue + + logger.debug( + "Checking for testmod {!r} in {!r}".format(test_mod, candidate_path) + ) + + if os.path.exists(candidate_path): + test_mod_path = candidate_path + + logger.info( + "Found testmod {!r} for component {!r} in {!r}".format( + mod_path, component, test_mod_path + ) + ) + + break + + if test_mod_path is None: + raise CIMEError(f"Could not locate testmod {mod_path!r}") + + test_mods_paths.append(test_mod_path) + + return test_mods_paths diff --git a/CIME/test_scheduler.py b/CIME/test_scheduler.py index b8b12ae08d2..1eee4de3f76 100644 --- a/CIME/test_scheduler.py +++ b/CIME/test_scheduler.py @@ -30,6 +30,7 @@ get_timestamp, get_cime_default_driver, clear_folder, + CIMEError, ) from CIME.config import Config from CIME.test_status import * @@ -47,6 +48,7 @@ from CIME.cs_status_creator import create_cs_status from CIME.hist_utils import generate_teststatus from CIME.build import post_build +from CIME.SystemTests.test_mods import find_test_mods logger = logging.getLogger(__name__) @@ -697,34 +699,18 @@ def _create_newcase_phase(self, test): if test_mods is not None: create_newcase_cmd += " --user-mods-dir " - for one_test_mod in test_mods: # pylint: disable=not-an-iterable - if one_test_mod.find("/") != -1: - (component, modspath) = one_test_mod.split("/", 1) - else: - error = "Missing testmod component. Testmods are specified as '${component}-${testmod}'" - self._log_output(test, error) - return False, error + try: + test_mods_paths = find_test_mods(self._cime_driver, test_mods) + except CIMEError as e: + error = f"{e}" - files = Files(comp_interface=self._cime_driver) - testmods_dir = files.get_value( - "TESTS_MODS_DIR", {"component": component} - ) - test_mod_file = os.path.join(testmods_dir, component, modspath) - # if no testmod is found check if a usermod of the same name exists and - # use it if it does. - if not os.path.exists(test_mod_file): - usermods_dir = files.get_value( - "USER_MODS_DIR", {"component": component} - ) - test_mod_file = os.path.join(usermods_dir, modspath) - if not os.path.exists(test_mod_file): - error = "Missing testmod file '{}', checked {} and {}".format( - modspath, testmods_dir, usermods_dir - ) - self._log_output(test, error) - return False, error + self._log_output(test, error) + + return False, error + else: + test_mods_paths = " ".join(test_mods_paths) - create_newcase_cmd += "{} ".format(test_mod_file) + create_newcase_cmd += f"{test_mods_paths}" # create_test mpilib option overrides default but not explicitly set case_opt mpilib if mpilib is None and self._mpilib is not None: diff --git a/CIME/tests/test_sys_test_scheduler.py b/CIME/tests/test_sys_test_scheduler.py index 3dfd4b62124..65858a411ee 100755 --- a/CIME/tests/test_sys_test_scheduler.py +++ b/CIME/tests/test_sys_test_scheduler.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import re import glob import logging import os @@ -14,6 +15,22 @@ class TestTestScheduler(base.BaseTestCase): + def get_default_tests(self): + # exclude the MEMLEAK tests here. + return get_tests.get_full_test_names( + [ + "cime_test_only", + "^TESTMEMLEAKFAIL_P1.f09_g16.X", + "^TESTMEMLEAKPASS_P1.f09_g16.X", + "^TESTRUNSTARCFAIL_P1.f19_g16_rx1.A", + "^TESTTESTDIFF_P1.f19_g16_rx1.A", + "^TESTBUILDFAILEXC_P1.f19_g16_rx1.A", + "^TESTRUNFAILEXC_P1.f19_g16_rx1.A", + ], + self._machine, + self._compiler, + ) + @mock.patch("time.strftime", return_value="00:00:00") def test_chksum(self, strftime): # pylint: disable=unused-argument if self._config.test_mode == "e3sm": @@ -38,21 +55,74 @@ def test_chksum(self, strftime): # pylint: disable=unused-argument from_dir="/tests/SEQ_Ln9.f19_g16_rx1.A.perlmutter_gnu.00:00:00", ) - def test_a_phases(self): - # exclude the MEMLEAK tests here. - tests = get_tests.get_full_test_names( - [ - "cime_test_only", - "^TESTMEMLEAKFAIL_P1.f09_g16.X", - "^TESTMEMLEAKPASS_P1.f09_g16.X", - "^TESTRUNSTARCFAIL_P1.f19_g16_rx1.A", - "^TESTTESTDIFF_P1.f19_g16_rx1.A", - "^TESTBUILDFAILEXC_P1.f19_g16_rx1.A", - "^TESTRUNFAILEXC_P1.f19_g16_rx1.A", - ], - self._machine, - self._compiler, + def test_testmods(self): + tests = self.get_default_tests() + ct = test_scheduler.TestScheduler( + tests, + test_root=self._testroot, + output_root=self._testroot, + compiler=self._compiler, + mpilib=self.TEST_MPILIB, + machine_name=self.MACHINE.get_machine_name(), + ) + + with mock.patch.object(ct, "_shell_cmd_for_phase"): + ct._create_newcase_phase( + "TESTRUNPASS_P1.f19_g16_rx1.A.docker_gnu.eam-rrtmgp" + ) + + create_newcase_cmd = ct._shell_cmd_for_phase.call_args.args[1] + + assert ( + re.search(r"--user-mods-dir .*eam/rrtmgp", create_newcase_cmd) + is not None + ), create_newcase_cmd + + def test_testmods_malformed(self): + tests = self.get_default_tests() + ct = test_scheduler.TestScheduler( + tests, + test_root=self._testroot, + output_root=self._testroot, + compiler=self._compiler, + mpilib=self.TEST_MPILIB, + machine_name=self.MACHINE.get_machine_name(), + ) + + with mock.patch.object(ct, "_shell_cmd_for_phase"): + success, message = ct._create_newcase_phase( + "TESTRUNPASS_P1.f19_g16_rx1.A.docker_gnu.notacomponent?fun" + ) + + assert not success + assert ( + message + == "Invalid testmod, format should be `${component}-${testmod}`, got 'notacomponent?fun'" + ), message + + def test_testmods_missing(self): + tests = self.get_default_tests() + ct = test_scheduler.TestScheduler( + tests, + test_root=self._testroot, + output_root=self._testroot, + compiler=self._compiler, + mpilib=self.TEST_MPILIB, + machine_name=self.MACHINE.get_machine_name(), ) + + with mock.patch.object(ct, "_shell_cmd_for_phase"): + success, message = ct._create_newcase_phase( + "TESTRUNPASS_P1.f19_g16_rx1.A.docker_gnu.notacomponent-fun" + ) + + assert not success + assert ( + re.search("Could not locate testmod 'fun'", message) is not None + ), message + + def test_a_phases(self): + tests = self.get_default_tests() self.assertEqual(len(tests), 3) ct = test_scheduler.TestScheduler( tests, diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py index 803478c74ee..090a1d420c1 100644 --- a/CIME/tests/test_unit_system_tests_mvk.py +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -5,17 +5,18 @@ from CIME.SystemTests.mvk import MVK + class TestSystemTestsMVK(unittest.TestCase): def test_mvk(self): case = mock.MagicMock() case.get_value.side_effect = ( - "/tmp/case", # CASEROOT - "MVK.f19_g16.S.docker_gnu", # CASEBASEID - "mct", # COMP_INTERFACE - "MVK.f19_g16.S.docker_gnu", # CASEBASEID - "e3sm", # MODEL - 0, # RESUBMIT - False, # GENERATE_BASELINE + "/tmp/case", # CASEROOT + "MVK.f19_g16.S.docker_gnu", # CASEBASEID + "mct", # COMP_INTERFACE + "MVK.f19_g16.S.docker_gnu", # CASEBASEID + "e3sm", # MODEL + 0, # RESUBMIT + False, # GENERATE_BASELINE ) test = MVK(case) From 9098540791d7b947e4dccd9d026db4ab2973cc88 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 15 May 2024 19:56:49 -0700 Subject: [PATCH 15/46] Adds unit testing for mvk test --- CIME/SystemTests/mvk.py | 20 +- CIME/tests/test_unit_case_setup.py | 12 +- CIME/tests/test_unit_system_tests_mvk.py | 346 ++++++++++++++++++++++- CIME/tests/utils.py | 12 + 4 files changed, 354 insertions(+), 36 deletions(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index e5060eb9263..fe208d9fc99 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -112,7 +112,7 @@ def __init__(self, case, **kwargs): test_mods_paths = find_test_mods(case.get_value("COMP_INTERFACE"), test_mods) for test_mods_path in test_mods_paths: - self._config = MVKConfig.load(test_mods_path) + self._config = MVKConfig.load(os.path.join(test_mods_path, "params.py")) if self._config is None: self._config = MVKConfig() @@ -121,16 +121,12 @@ def __init__(self, case, **kwargs): if self._config.component == "": # TODO remove model specific if self._case.get_value("MODEL") == "e3sm": - self.component = "eam" + self._config.component = "eam" else: - self.component = "cam" - else: - self.component = self._config.component + self._config.component = "cam" if len(self._config.components) == 0: - self.components = [self.component] - else: - self.components = self._config.components + self._config.components = [self._config.component] if ( self._case.get_value("RESUBMIT") == 0 @@ -162,14 +158,12 @@ def build_phase(self, sharedlib_only=False, model_only=False): case_setup(self._case, test_mode=False, reset=True) for iinst in range(1, self._config.ninst + 1): - for component in self.components: + for component in self._config.components: with open( "user_nl_{}_{:04d}".format(component, iinst), "w" ) as nml_file: - set_nml_variable = ( - lambda x: nml_file.write( # pylint: disable=cell-var-from-loop - f"{x}\n" - ) + set_nml_variable = lambda key, value: nml_file.write( # pylint: disable=cell-var-from-loop + f"{key} = {value}\n" ) self._config.write_inst_nml( diff --git a/CIME/tests/test_unit_case_setup.py b/CIME/tests/test_unit_case_setup.py index fe5fa7308c1..a00dcf1b413 100644 --- a/CIME/tests/test_unit_case_setup.py +++ b/CIME/tests/test_unit_case_setup.py @@ -8,6 +8,7 @@ from unittest import mock from CIME.case import case_setup +from CIME.tests.utils import chdir @contextlib.contextmanager @@ -23,17 +24,6 @@ def create_machines_dir(): yield temp_path -@contextlib.contextmanager -def chdir(path): - old_path = os.getcwd() - os.chdir(path) - - try: - yield - finally: - os.chdir(old_path) - - # pylint: disable=protected-access class TestCaseSetup(unittest.TestCase): @mock.patch("CIME.case.case_setup.copy_depends_files") diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py index 090a1d420c1..20a7368bb39 100644 --- a/CIME/tests/test_unit_system_tests_mvk.py +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -1,25 +1,347 @@ #!/usr/bin/env python3 +import os +import json import unittest +import tempfile +import contextlib +from pathlib import Path from unittest import mock from CIME.SystemTests.mvk import MVK +from CIME.SystemTests.mvk import MVKConfig +from CIME.tests.utils import chdir -class TestSystemTestsMVK(unittest.TestCase): - def test_mvk(self): - case = mock.MagicMock() - case.get_value.side_effect = ( - "/tmp/case", # CASEROOT - "MVK.f19_g16.S.docker_gnu", # CASEBASEID - "mct", # COMP_INTERFACE - "MVK.f19_g16.S.docker_gnu", # CASEBASEID - "e3sm", # MODEL +def create_complex_case( + case_name, temp_dir, run_dir, baseline_dir, compare_baseline=False +): + case = mock.MagicMock() + + side_effect = [ + str(temp_dir), # CASEROOT + "MVK.f19_g16.S.docker_gnu", # CASEBASEID + "mct", # COMP_INTERFACE + "mct", # COMP_INTERFACE + ] + + # single extra call for _compare_baseline + if compare_baseline: + side_effect.append("e3sm") # MODEL + + side_effect.extend( + [ 0, # RESUBMIT False, # GENERATE_BASELINE - ) + 0, # RESUBMIT + str(run_dir), # RUNDIR + case_name, # CASE + str(baseline_dir), # BASELINE_ROOT + "", # BASECMP_CASE + "docker", # MACH + ] + ) + + case.get_value.side_effect = side_effect + + run_dir.mkdir(parents=True) + + evv_output = run_dir / f"{case_name}.evv" / "index.json" + + evv_output.parent.mkdir(parents=True) + + with open(evv_output, "w") as fd: + fd.write(json.dumps({"Page": {"elements": []}})) + + return case + + +def create_simple_case(): + case = mock.MagicMock() + + case.get_value.side_effect = ( + "/tmp/case", # CASEROOT + "MVK.f19_g16.S.docker_gnu", # CASEBASEID + "mct", # COMP_INTERFACE + "MVK.f19_g16.S.docker_gnu", # CASEBASEID + "e3sm", # MODEL + 0, # RESUBMIT + False, # GENERATE_BASELINE + ) + + return case + + +class TestSystemTestsMVK(unittest.TestCase): + def tearDown(self): + # reset singleton + delattr(MVKConfig, "_instance") + + @mock.patch("CIME.SystemTests.mvk.find_test_mods") + @mock.patch("CIME.SystemTests.mvk.evv") + def test_testmod_complex(self, evv, find_test_mods): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + print(temp_dir) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + testmods_dir = temp_dir / "testmods" / "eam" + + testmods_dir.mkdir(parents=True) + + find_test_mods.return_value = [str(testmods_dir)] + + with open(testmods_dir / "params.py", "w") as fd: + fd.write( + """ +import os + +component = "new-comp" +components = ["new-comp", "secondary-comp"] +ninst = 8 + +def write_inst_nml(case, set_nml_variable, component, iinst): + if component == "new-comp": + set_nml_variable("var1", "value1") + elif component == "secondary-comp": + set_nml_variable("var2", "value2") + +def test_config(case, run_dir, base_dir, evv_lib_dir): + return { + "module": os.path.join(evv_lib_dir, "extensions", "kso.py"), + "component": "someother-comp" + } + """ + ) + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + case = create_complex_case(case_name, temp_dir, run_dir, baseline_dir) + test = MVK(case) + + stack.enter_context(mock.patch.object(test, "build_indv")) + + test.build_phase(False, True) + test._compare_baseline() + + with open(run_dir / f"{case_name}.json", "r") as fd: + config = json.load(fd) + + expected_config = { + "20240515_212034_41b5u2": { + "module": "/opt/conda/lib/python3.10/site-packages/evv4esm/extensions/kso.py", + "component": "someother-comp", + } + } + + assert config == expected_config + + nml_files = [x for x in os.listdir(temp_dir) if x.startswith("user_nl")] + + assert len(nml_files) == 16 + + with open(sorted(nml_files)[0], "r") as fd: + lines = fd.readlines() + + assert lines == ["var1 = value1\n"] + + with open(sorted(nml_files)[-1], "r") as fd: + lines = fd.readlines() + + assert lines == ["var2 = value2\n"] + + @mock.patch("CIME.SystemTests.mvk.find_test_mods") + @mock.patch("CIME.SystemTests.mvk.evv") + def test_testmod_simple(self, evv, find_test_mods): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + testmods_dir = temp_dir / "testmods" / "eam" + + testmods_dir.mkdir(parents=True) + + find_test_mods.return_value = [str(testmods_dir)] + + with open(testmods_dir / "params.py", "w") as fd: + fd.write( + """ +component = "new-comp" +components = ["new-comp", "second-comp"] +ninst = 8 +critical = 32 +var_set = "special" +ref_case = "Reference" +test_case = "Default" + """ + ) + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + case = create_complex_case(case_name, temp_dir, run_dir, baseline_dir) + test = MVK(case) + + stack.enter_context(mock.patch.object(test, "build_indv")) + + test.build_phase(False, True) + test._compare_baseline() + + with open(run_dir / f"{case_name}.json", "r") as fd: + config = json.load(fd) + + expected_config = { + "20240515_212034_41b5u2": { + "module": "/opt/conda/lib/python3.10/site-packages/evv4esm/extensions/ks.py", + "test-case": "Default", + "test-dir": f"{run_dir}", + "ref-case": "Reference", + "ref-dir": f"{baseline_dir}/", + "var-set": "special", + "ninst": 8, + "critical": 32, + "component": "new-comp", + } + } + + assert config == expected_config + + nml_files = [x for x in os.listdir(temp_dir) if x.startswith("user_nl")] + + assert len(nml_files) == 16 + + with open(sorted(nml_files)[0], "r") as fd: + lines = fd.readlines() + + assert lines == [ + "new_random = .true.\n", + "pertlim = 1.0e-10\n", + "seed_custom = 1\n", + "seed_clock = .true.\n", + ] + + with open(sorted(nml_files)[-1], "r") as fd: + lines = fd.readlines() + + assert lines == [ + "new_random = .true.\n", + "pertlim = 1.0e-10\n", + "seed_custom = 8\n", + "seed_clock = .true.\n", + ] + + @mock.patch("CIME.SystemTests.mvk.evv") + def test__compare_baseline(self, evv): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + case = create_complex_case(case_name, temp_dir, run_dir, baseline_dir, True) + + test = MVK(case) + + test._compare_baseline() + + with open(run_dir / f"{case_name}.json", "r") as fd: + config = json.load(fd) + + expected_config = { + "20240515_212034_41b5u2": { + "module": "/opt/conda/lib/python3.10/site-packages/evv4esm/extensions/ks.py", + "test-case": "Test", + "test-dir": f"{run_dir}", + "ref-case": "Baseline", + "ref-dir": f"{baseline_dir}/", + "var-set": "default", + "ninst": 30, + "critical": 13, + "component": "eam", + } + } + + assert config == expected_config + + def test_write_inst_nml_multiple_components(self): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + print(temp_dir) + + stack.enter_context(chdir(temp_dir)) + + case = create_simple_case() + + test = MVK(case) + + stack.enter_context(mock.patch.object(test, "build_indv")) + + test._config.components = ["eam", "elm"] + + test.build_phase(False, True) + + nml_files = os.listdir(temp_dir) + + assert len(nml_files) == 60 + + with open(sorted(nml_files)[0], "r") as fd: + lines = fd.readlines() + + assert lines == [ + "new_random = .true.\n", + "pertlim = 1.0e-10\n", + "seed_custom = 1\n", + "seed_clock = .true.\n", + ] + + def test_write_inst_nml(self): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + case = create_simple_case() + + test = MVK(case) + + stack.enter_context(mock.patch.object(test, "build_indv")) + + test.build_phase(False, True) + + nml_files = os.listdir(temp_dir) + + assert len(nml_files) == 30 + + with open(sorted(nml_files)[0], "r") as fd: + lines = fd.readlines() + + assert lines == [ + "new_random = .true.\n", + "pertlim = 1.0e-10\n", + "seed_custom = 1\n", + "seed_clock = .true.\n", + ] + + def test_mvk(self): + case = create_simple_case() test = MVK(case) - assert test.component == "eam" - assert test.components == ["eam"] + assert test._config.component == "eam" + assert test._config.components == ["eam"] diff --git a/CIME/tests/utils.py b/CIME/tests/utils.py index 719aaaff290..0f75fa5ad24 100644 --- a/CIME/tests/utils.py +++ b/CIME/tests/utils.py @@ -5,6 +5,7 @@ import shutil import sys import time +import contextlib from collections.abc import Iterable from CIME import utils @@ -50,6 +51,17 @@ ] +@contextlib.contextmanager +def chdir(path): + old_path = os.getcwd() + os.chdir(path) + + try: + yield + finally: + os.chdir(old_path) + + def parse_test_status(line): status, test = line.split()[0:2] return test, status From c300abe46de3f32b31e99659b64b1fbbc4c021dd Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 16 May 2024 12:08:06 -0700 Subject: [PATCH 16/46] Adds System Test Mods section --- doc/source/users_guide/testing.rst | 122 ++++++++++++++++++++++++++++- 1 file changed, 119 insertions(+), 3 deletions(-) diff --git a/doc/source/users_guide/testing.rst b/doc/source/users_guide/testing.rst index ed15a849e00..8a35a51560c 100644 --- a/doc/source/users_guide/testing.rst +++ b/doc/source/users_guide/testing.rst @@ -17,10 +17,11 @@ An individual test can be run as:: Everything the test will do is controlled by parsing the test name. +.. _`Test naming`: + ================= Testname syntax ================= -.. _`Test naming`: Tests must be named with the following forms, [ ]=optional:: @@ -209,12 +210,12 @@ You can combine testtype modifiers:: CIMEROOT/scripts/create_test ERP_D_Ld3.ne4pg2_oQU480.F2010 +.. _GROUP-TESTMODS: + ------------------- Test Case Modifiers ------------------- -.. _GROUP-TESTMODS: - create_test runs with out-of-the-box compsets and grid sets. Sometimes you may want to run a test with modification to a namelist or other setting without creating an entire compset. CCS provides the testmods capability for this situation. @@ -266,6 +267,121 @@ in an F-case test. The "rrtmpg" directory contains the actual testmods to apply. Note; do not use '-' in the testmods directory name because it has a special meaning to create_test. +.. _System Test Mods: + +--------------------- +System Test Mods +--------------------- +`Test Case Modifiers`_ are not only a way to modify a test case but they can also be used to configure certain :ref:`Test types `. + +Supported :ref:`test types ` can be configured by creating a ``params.py`` file in a :ref:`test case modifier `. + +^^^^^^^^^^^^ +MVK +^^^^^^^^^^^^ +The `MVK` system test can be configured by defining :ref:`variables ` and :ref:`methods ` in ``params.py``. + +See :ref:`examples ` for a simple and complex use case. + +.. _MVK Variables: + +""""""""" +Variables +""""""""" +Available settings for the MVK test type. + +========== ======== ==== ==================================================== +Variable Default Type Description +========== ======== ==== ==================================================== +component str The main component. +components [] list Components that require namelist customization. +ninst 30 int The number of instances. +critical 13 int The critical value for rejecting the null hypothese. +var_set default str Name of the variable set to analyze. +ref_case Baseline str Name of the reference case. +test_case Test str Name of the test case. +========== ======== ==== ==================================================== + +.. _MVK Methods: + +""""""" +Methods +""""""" +Available methods for the MVK test type. + +.. code-block:: + + def test_config(case, run_dir, base_dir, evv_lib_dir): + """ + Configure the evv test. + + This method is used to pass the evv4esm configuration to be written for the test. + + Args: + case (CIME.case.case.Case): The case instance. + run_dir (str): Path the case's run directory. + base_dir (str): Path to the case's baseline directory. + evv_lib_dir (str): Path to the evv4esm package root. + + Returns: + dict: Dictionary with test configuration. + """ + +.. code-block:: + + def write_inst_nml(case, set_nml_variable, component, iinst): + """ + Write per instance namelist. + + This method is called once per instance to generate the namelist. + + Args: + case (CIME.case.case.Case): The case instance. + write_nml_variable (function): Function takes two `str` arguments. + component (str): Component the namelist belongs to. + iinst (int): Instance unique number. + """ + +.. _MVK Examples: + +"""""""""" +Examples +"""""""""" +.. _MVK Simple: +A simple customization of the `MVK` :ref:`test type ` would be just defining some :ref:`variables ` in ``params.py``. + +.. code-block:: + + component = "eam" + # components = [] can be omitted when modifying a single component + ninst = 10 + critical = 21 + +.. _MVK Complex: +A complex customization of the `MVK` :ref:`test type ` would be defining only the required :ref:`variables ` and defining some :ref:`methods ` in ``params.py``. + +.. code-block:: + + import os + + component = "eam" + components = ["eam", "clm"] + + def test_config(case, run_dir, base_dir, evv_lib_dir): + return { + "module": os.path.join(evv_lib_dir, "extensions", "kso.py"), + "component": component, + } + + def write_inst_nml(case, set_nml_variable, component, iinst): + if component == "eam": + set_nml_variable("eam_specific", f"perturb-{iinst}") + elif component == "clm": + if iinst % 2 == 0: + set_nml_variable("clm_specific", "even") + else: + set_nml_variable("clm_specific", "odd") + ======================== Test progress and output ======================== From 809cee70a6ec6876946ed4f893e18607a1840a04 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 16 May 2024 12:41:07 -0700 Subject: [PATCH 17/46] Adds copybutton to docs --- doc/requirements.txt | 1 + doc/source/conf.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/requirements.txt b/doc/requirements.txt index 956df97689b..e6edcf66b06 100644 --- a/doc/requirements.txt +++ b/doc/requirements.txt @@ -1,4 +1,5 @@ sphinx sphinxcontrib-programoutput sphinx-rtd-theme +sphinx-copybutton evv4esm diff --git a/doc/source/conf.py b/doc/source/conf.py index 0239006c530..c6a1de2db7e 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -* coding: utf-8 -*- # # on documentation build configuration file, created by # sphinx-quickstart on Tue Jan 31 19:46:36 2017. @@ -46,6 +46,7 @@ "sphinx.ext.todo", "sphinxcontrib.programoutput", "sphinx_rtd_theme", + "sphinx_copybutton", ] todo_include_todos = True From ddb480b767f4ea2219c3830afc038b31479ecaba Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 16 May 2024 12:50:59 -0700 Subject: [PATCH 18/46] Fixes missing testing requirements --- .github/workflows/testing.yml | 2 ++ test-requirements.txt | 1 + 2 files changed, 3 insertions(+) create mode 100644 test-requirements.txt diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 81494fb57b6..adf083e7140 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -103,6 +103,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 + - name: Install test-requirements.txt + run: pip install -r test-requirements.txt - name: Run tests shell: bash env: diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 00000000000..4420847ab60 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1 @@ +evv4e3sm From 26868306466097f6911ad1c5bb0e16c1a67d4699 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 16 May 2024 13:12:49 -0700 Subject: [PATCH 19/46] Fixes package name and fixes system testing workflow --- .github/workflows/testing.yml | 2 ++ test-requirements.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index adf083e7140..f8522be3ab3 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -157,6 +157,8 @@ jobs: with: path: /storage/inputdata key: inputdata-2 + - name: Install test-requirements.txt + run: pip install -r test-requirements.txt - name: Run tests shell: bash env: diff --git a/test-requirements.txt b/test-requirements.txt index 4420847ab60..d4abdec4d0f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1 +1 @@ -evv4e3sm +evv4esm From 4d2113e5ddb1d2ef46d712d70327a40594aea297 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 16 May 2024 15:36:40 -0700 Subject: [PATCH 20/46] Fixes imports and adds test for append_testlog --- CIME/SystemTests/mvk.py | 44 +++++++++--------- CIME/tests/test_unit_system_tests_mvk.py | 57 ++++++++++++++++++++---- 2 files changed, 72 insertions(+), 29 deletions(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index fe208d9fc99..6e543ceebcf 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -11,14 +11,13 @@ import logging from distutils import dir_util -import CIME.test_status -import CIME.utils +from CIME import test_status +from CIME import utils from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case.case_setup import case_setup from CIME.XML.machines import Machines from CIME.config import ConfigBase -from CIME.utils import parse_test_name -from CIME.SystemTests.test_mods import find_test_mods +from CIME.SystemTests import test_mods import evv4esm # pylint: disable=import-error from evv4esm.__main__ import main as evv # pylint: disable=import-error @@ -107,9 +106,11 @@ def __init__(self, case, **kwargs): SystemTestsCommon.__init__(self, case, **kwargs) - *_, test_mods = parse_test_name(self._casebaseid) + *_, case_test_mods = utils.parse_test_name(self._casebaseid) - test_mods_paths = find_test_mods(case.get_value("COMP_INTERFACE"), test_mods) + test_mods_paths = test_mods.find_test_mods( + case.get_value("COMP_INTERFACE"), case_test_mods + ) for test_mods_path in test_mods_paths: self._config = MVKConfig.load(os.path.join(test_mods_path, "params.py")) @@ -178,7 +179,7 @@ def _generate_baseline(self): """ super(MVK, self)._generate_baseline() - with CIME.utils.SharedArea(): + with utils.SharedArea(): basegen_dir = os.path.join( self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE"), @@ -189,17 +190,20 @@ def _generate_baseline(self): env_archive = self._case.get_env("archive") hists = env_archive.get_all_hist_files( - self._case.get_value("CASE"), self.component, rundir, ref_case=ref_case + self._case.get_value("CASE"), + self._config.component, + rundir, + ref_case=ref_case, ) logger.debug("MVK additional baseline files: {}".format(hists)) hists = [os.path.join(rundir, hist) for hist in hists] for hist in hists: - basename = hist[hist.rfind(self.component) :] + basename = hist[hist.rfind(self._config.component) :] baseline = os.path.join(basegen_dir, basename) if os.path.exists(baseline): os.remove(baseline) - CIME.utils.safe_copy(hist, baseline, preserve_meta=False) + utils.safe_copy(hist, baseline, preserve_meta=False) def _compare_baseline(self): with self._test_status: @@ -208,12 +212,12 @@ def _compare_baseline(self): # and we only want to compare once the whole run is finished. We # need to return a pass here to continue the submission process. self._test_status.set_status( - CIME.test_status.BASELINE_PHASE, CIME.test_status.TEST_PASS_STATUS + test_status.BASELINE_PHASE, test_status.TEST_PASS_STATUS ) return self._test_status.set_status( - CIME.test_status.BASELINE_PHASE, CIME.test_status.TEST_FAIL_STATUS + test_status.BASELINE_PHASE, test_status.TEST_FAIL_STATUS ) run_dir = self._case.get_value("RUNDIR") @@ -250,18 +254,18 @@ def _compare_baseline(self): ) if evv_ele["Table"]["data"]["Test status"][0].lower() == "pass": self._test_status.set_status( - CIME.test_status.BASELINE_PHASE, - CIME.test_status.TEST_PASS_STATUS, + test_status.BASELINE_PHASE, + test_status.TEST_PASS_STATUS, ) break - status = self._test_status.get_status(CIME.test_status.BASELINE_PHASE) + status = self._test_status.get_status(test_status.BASELINE_PHASE) mach_name = self._case.get_value("MACH") mach_obj = Machines(machine=mach_name) - htmlroot = CIME.utils.get_htmlroot(mach_obj) - urlroot = CIME.utils.get_urlroot(mach_obj) + htmlroot = utils.get_htmlroot(mach_obj) + urlroot = utils.get_urlroot(mach_obj) if htmlroot is not None: - with CIME.utils.SharedArea(): + with utils.SharedArea(): dir_util.copy_tree( evv_out_dir, os.path.join(htmlroot, "evv", case_name), @@ -284,7 +288,7 @@ def _compare_baseline(self): " {}\n" " EVV results can be viewed at:\n" " {}".format( - CIME.test_status.BASELINE_PHASE, + test_status.BASELINE_PHASE, status, test_name, comments, @@ -292,4 +296,4 @@ def _compare_baseline(self): ) ) - CIME.utils.append_testlog(comments, self._orig_caseroot) + utils.append_testlog(comments, self._orig_caseroot) diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py index 20a7368bb39..d16993c0510 100644 --- a/CIME/tests/test_unit_system_tests_mvk.py +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -14,7 +14,12 @@ def create_complex_case( - case_name, temp_dir, run_dir, baseline_dir, compare_baseline=False + case_name, + temp_dir, + run_dir, + baseline_dir, + compare_baseline=False, + mock_evv_output=False, ): case = mock.MagicMock() @@ -50,8 +55,28 @@ def create_complex_case( evv_output.parent.mkdir(parents=True) + if mock_evv_output: + evv_output_data = { + "Page": { + "elements": [ + { + "Table": { + "data": { + "Test status": ["pass"], + "Variables analyzed": ["v1", "v2"], + "Rejecting": [2], + "Critical value": [12], + } + } + } + ] + } + } + else: + evv_output_data = {"Page": {"elements": []}} + with open(evv_output, "w") as fd: - fd.write(json.dumps({"Page": {"elements": []}})) + fd.write(json.dumps(evv_output_data)) return case @@ -75,9 +100,12 @@ def create_simple_case(): class TestSystemTestsMVK(unittest.TestCase): def tearDown(self): # reset singleton - delattr(MVKConfig, "_instance") + try: + delattr(MVKConfig, "_instance") + except: + pass - @mock.patch("CIME.SystemTests.mvk.find_test_mods") + @mock.patch("CIME.SystemTests.mvk.test_mods.find_test_mods") @mock.patch("CIME.SystemTests.mvk.evv") def test_testmod_complex(self, evv, find_test_mods): with contextlib.ExitStack() as stack: @@ -155,7 +183,7 @@ def test_config(case, run_dir, base_dir, evv_lib_dir): assert lines == ["var2 = value2\n"] - @mock.patch("CIME.SystemTests.mvk.find_test_mods") + @mock.patch("CIME.SystemTests.mvk.test_mods.find_test_mods") @mock.patch("CIME.SystemTests.mvk.evv") def test_testmod_simple(self, evv, find_test_mods): with contextlib.ExitStack() as stack: @@ -239,8 +267,9 @@ def test_testmod_simple(self, evv, find_test_mods): "seed_clock = .true.\n", ] + @mock.patch("CIME.SystemTests.mvk.utils.append_testlog") @mock.patch("CIME.SystemTests.mvk.evv") - def test__compare_baseline(self, evv): + def test__compare_baseline(self, evv, append_testlog): with contextlib.ExitStack() as stack: temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) @@ -253,7 +282,9 @@ def test__compare_baseline(self, evv): case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE - case = create_complex_case(case_name, temp_dir, run_dir, baseline_dir, True) + case = create_complex_case( + case_name, temp_dir, run_dir, baseline_dir, True, mock_evv_output=True + ) test = MVK(case) @@ -278,12 +309,20 @@ def test__compare_baseline(self, evv): assert config == expected_config + expected_comments = f"""BASELINE PASS for test '20240515_212034_41b5u2'. + Test status: pass; Variables analyzed: v1; Rejecting: 2; Critical value: 12 + EVV results can be viewed at: + {run_dir}/MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2.evv + EVV viewing instructions can be found at: https://github.com/E3SM-Project/E3SM/blob/master/cime/scripts/climate_reproducibility/README.md#test-passfail-and-extended-output""" + + append_testlog.assert_any_call( + expected_comments, str(temp_dir) + ), append_testlog.call_args.args + def test_write_inst_nml_multiple_components(self): with contextlib.ExitStack() as stack: temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) - print(temp_dir) - stack.enter_context(chdir(temp_dir)) case = create_simple_case() From 1eed09f1642172c642bdcf431257ceec5b6fb01f Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 16 May 2024 17:02:13 -0700 Subject: [PATCH 21/46] Refactors _compare_baseline --- CIME/SystemTests/mvk.py | 118 +++++++++++++++++++++++----------------- 1 file changed, 68 insertions(+), 50 deletions(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 6e543ceebcf..db4788113cd 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -242,58 +242,76 @@ def _compare_baseline(self): evv_out_dir = os.path.join(run_dir, f"{case_name}.evv") evv(["-e", json_file, "-o", evv_out_dir]) - with open(os.path.join(evv_out_dir, "index.json")) as evv_f: - evv_status = json.load(evv_f) - - comments = "" - for evv_ele in evv_status["Page"]["elements"]: - if "Table" in evv_ele: - comments = "; ".join( - "{}: {}".format(key, val[0]) - for key, val in evv_ele["Table"]["data"].items() - ) - if evv_ele["Table"]["data"]["Test status"][0].lower() == "pass": - self._test_status.set_status( - test_status.BASELINE_PHASE, - test_status.TEST_PASS_STATUS, - ) - break - - status = self._test_status.get_status(test_status.BASELINE_PHASE) - mach_name = self._case.get_value("MACH") - mach_obj = Machines(machine=mach_name) - htmlroot = utils.get_htmlroot(mach_obj) + self.update_testlog(test_name, case_name, evv_out_dir) + + def update_testlog(self, test_name, case_name, evv_out_dir): + comments = self.process_evv_output(evv_out_dir) + + status = self._test_status.get_status(test_status.BASELINE_PHASE) + + mach_name = self._case.get_value("MACH") + + mach_obj = Machines(machine=mach_name) + + htmlroot = utils.get_htmlroot(mach_obj) + + if htmlroot is not None: urlroot = utils.get_urlroot(mach_obj) - if htmlroot is not None: - with utils.SharedArea(): - dir_util.copy_tree( - evv_out_dir, - os.path.join(htmlroot, "evv", case_name), - preserve_mode=False, - ) - if urlroot is None: - urlroot = "[{}_URL]".format(mach_name.capitalize()) - viewing = "{}/evv/{}/index.html".format(urlroot, case_name) - else: - viewing = ( - "{}\n" - " EVV viewing instructions can be found at: " - " https://github.com/E3SM-Project/E3SM/blob/master/cime/scripts/" - "climate_reproducibility/README.md#test-passfail-and-extended-output" - "".format(evv_out_dir) - ) - comments = ( - "{} {} for test '{}'.\n" - " {}\n" - " EVV results can be viewed at:\n" - " {}".format( - test_status.BASELINE_PHASE, - status, - test_name, - comments, - viewing, + with utils.SharedArea(): + dir_util.copy_tree( + evv_out_dir, + os.path.join(htmlroot, "evv", case_name), + preserve_mode=False, ) + + if urlroot is None: + urlroot = "[{}_URL]".format(mach_name.capitalize()) + + viewing = "{}/evv/{}/index.html".format(urlroot, case_name) + else: + viewing = ( + "{}\n" + " EVV viewing instructions can be found at: " + " https://github.com/E3SM-Project/E3SM/blob/master/cime/scripts/" + "climate_reproducibility/README.md#test-passfail-and-extended-output" + "".format(evv_out_dir) + ) + + comments = ( + "{} {} for test '{}'.\n" + " {}\n" + " EVV results can be viewed at:\n" + " {}".format( + test_status.BASELINE_PHASE, + status, + test_name, + comments, + viewing, ) + ) + + utils.append_testlog(comments, self._orig_caseroot) + + def process_evv_output(self, evv_out_dir): + with open(os.path.join(evv_out_dir, "index.json")) as evv_f: + evv_status = json.load(evv_f) + + comments = "" + + for evv_ele in evv_status["Page"]["elements"]: + if "Table" in evv_ele: + comments = "; ".join( + "{}: {}".format(key, val[0]) + for key, val in evv_ele["Table"]["data"].items() + ) + + if evv_ele["Table"]["data"]["Test status"][0].lower() == "pass": + self._test_status.set_status( + test_status.BASELINE_PHASE, + test_status.TEST_PASS_STATUS, + ) + + break - utils.append_testlog(comments, self._orig_caseroot) + return comments From 516aa29f6bb745f13e8d88f20eeffcec3484a4f2 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 16 May 2024 17:39:04 -0700 Subject: [PATCH 22/46] Fixes installing deps --- .github/workflows/testing.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index f8522be3ab3..563fc5ca757 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -103,8 +103,6 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Install test-requirements.txt - run: pip install -r test-requirements.txt - name: Run tests shell: bash env: @@ -112,6 +110,8 @@ jobs: CIME_DRIVER: "nuopc" CIME_TEST_PLATFORM: ubuntu-latest run: | + pip install -r test-requirements.txt + export SRC_PATH="${GITHUB_WORKSPACE}" mamba install -y python=${{ matrix.python-version }} @@ -157,8 +157,6 @@ jobs: with: path: /storage/inputdata key: inputdata-2 - - name: Install test-requirements.txt - run: pip install -r test-requirements.txt - name: Run tests shell: bash env: @@ -166,6 +164,8 @@ jobs: CIME_DRIVER: ${{ matrix.driver }} CIME_TEST_PLATFORM: ubuntu-latest run: | + pip install -r test-requirements.txt + export SRC_PATH="${GITHUB_WORKSPACE}" source /entrypoint.sh From 8bf1e19b8921e4ebb2f368e34b42d50fce45390b Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 16 May 2024 17:46:30 -0700 Subject: [PATCH 23/46] Fixes workflow --- .github/workflows/testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 563fc5ca757..da136938dfb 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -110,12 +110,12 @@ jobs: CIME_DRIVER: "nuopc" CIME_TEST_PLATFORM: ubuntu-latest run: | - pip install -r test-requirements.txt - export SRC_PATH="${GITHUB_WORKSPACE}" mamba install -y python=${{ matrix.python-version }} + pip install -r test-requirements.txt + source /entrypoint.sh # GitHub runner home is different than container From 26e6f9f571d308199f79139c7b79dc41f65f45ef Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 16 May 2024 18:03:27 -0700 Subject: [PATCH 24/46] Fixes asserts --- CIME/tests/test_unit_system_tests_mvk.py | 31 +++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py index d16993c0510..37ebf2826bb 100644 --- a/CIME/tests/test_unit_system_tests_mvk.py +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import re import os import json import unittest @@ -162,11 +163,19 @@ def test_config(case, run_dir, base_dir, evv_lib_dir): expected_config = { "20240515_212034_41b5u2": { - "module": "/opt/conda/lib/python3.10/site-packages/evv4esm/extensions/kso.py", "component": "someother-comp", } } + module = config["20240515_212034_41b5u2"].pop("module") + + assert ( + re.search( + r"/opt/conda/lib/python.*/site-packages/evv4esm/extensions/kso.py", + module, + ) + is not None + ) assert config == expected_config nml_files = [x for x in os.listdir(temp_dir) if x.startswith("user_nl")] @@ -229,7 +238,6 @@ def test_testmod_simple(self, evv, find_test_mods): expected_config = { "20240515_212034_41b5u2": { - "module": "/opt/conda/lib/python3.10/site-packages/evv4esm/extensions/ks.py", "test-case": "Default", "test-dir": f"{run_dir}", "ref-case": "Reference", @@ -241,6 +249,15 @@ def test_testmod_simple(self, evv, find_test_mods): } } + module = config["20240515_212034_41b5u2"].pop("module") + + assert ( + re.search( + r"/opt/conda/lib/python.*/site-packages/evv4esm/extensions/ks.py", + module, + ) + is not None + ) assert config == expected_config nml_files = [x for x in os.listdir(temp_dir) if x.startswith("user_nl")] @@ -295,7 +312,6 @@ def test__compare_baseline(self, evv, append_testlog): expected_config = { "20240515_212034_41b5u2": { - "module": "/opt/conda/lib/python3.10/site-packages/evv4esm/extensions/ks.py", "test-case": "Test", "test-dir": f"{run_dir}", "ref-case": "Baseline", @@ -307,6 +323,15 @@ def test__compare_baseline(self, evv, append_testlog): } } + module = config["20240515_212034_41b5u2"].pop("module") + + assert ( + re.search( + r"/opt/conda/lib/python.*/site-packages/evv4esm/extensions/ks.py", + module, + ) + is not None + ) assert config == expected_config expected_comments = f"""BASELINE PASS for test '20240515_212034_41b5u2'. From 4a92293def40a59dc56c7db5d506a119ac2e2110 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Sat, 18 May 2024 10:48:47 -0700 Subject: [PATCH 25/46] Disables sys test for cesm --- CIME/tests/test_sys_test_scheduler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CIME/tests/test_sys_test_scheduler.py b/CIME/tests/test_sys_test_scheduler.py index 65858a411ee..d4822f93e32 100755 --- a/CIME/tests/test_sys_test_scheduler.py +++ b/CIME/tests/test_sys_test_scheduler.py @@ -56,6 +56,9 @@ def test_chksum(self, strftime): # pylint: disable=unused-argument ) def test_testmods(self): + if self._config.test_mode == "cesm": + self.skipTest("Skipping testmods test. Depends on E3SM settings") + tests = self.get_default_tests() ct = test_scheduler.TestScheduler( tests, From fb3a7c476aee679d39042d27889da1a6a9852612 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 13 Jun 2024 10:22:32 -0700 Subject: [PATCH 26/46] Adds assert to constrain version --- CIME/SystemTests/mvk.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index db4788113cd..011bc9704b4 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -22,6 +22,10 @@ import evv4esm # pylint: disable=import-error from evv4esm.__main__ import main as evv # pylint: disable=import-error +version = evv4esm.__version_info__ + +assert version[0] <= 0 and version[1] <= 5, "Please install evv4esm less than 0.5.x" + EVV_LIB_DIR = os.path.abspath(os.path.dirname(evv4esm.__file__)) logger = logging.getLogger(__name__) From 6aaf58908318e3ac753ee003dd80de8d4ac1396a Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 13 Jun 2024 15:17:33 -0700 Subject: [PATCH 27/46] Updates testing and fixes bug --- CIME/SystemTests/mvk.py | 10 +- CIME/tests/test_unit_system_tests_mvk.py | 312 ++++++++++++++++++++++- 2 files changed, 309 insertions(+), 13 deletions(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 011bc9704b4..fdbedc6f7dd 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -146,6 +146,7 @@ def build_phase(self, sharedlib_only=False, model_only=False): # so it has to happen there. if not model_only: logging.warning("Starting to build multi-instance exe") + for comp in self._case.get_values("COMP_CLASSES"): self._case.set_value("NTHRDS_{}".format(comp), 1) @@ -311,10 +312,11 @@ def process_evv_output(self, evv_out_dir): ) if evv_ele["Table"]["data"]["Test status"][0].lower() == "pass": - self._test_status.set_status( - test_status.BASELINE_PHASE, - test_status.TEST_PASS_STATUS, - ) + with self._test_status: + self._test_status.set_status( + test_status.BASELINE_PHASE, + test_status.TEST_PASS_STATUS, + ) break diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py index 37ebf2826bb..a559d8bffb4 100644 --- a/CIME/tests/test_unit_system_tests_mvk.py +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -50,12 +50,18 @@ def create_complex_case( case.get_value.side_effect = side_effect - run_dir.mkdir(parents=True) + run_dir.mkdir(parents=True, exist_ok=True) evv_output = run_dir / f"{case_name}.evv" / "index.json" - evv_output.parent.mkdir(parents=True) + evv_output.parent.mkdir(parents=True, exist_ok=True) + write_evv_output(evv_output, mock_evv_output=mock_evv_output) + + return case + + +def write_evv_output(evv_output_path, mock_evv_output): if mock_evv_output: evv_output_data = { "Page": { @@ -76,13 +82,11 @@ def create_complex_case( else: evv_output_data = {"Page": {"elements": []}} - with open(evv_output, "w") as fd: + with open(evv_output_path, "w") as fd: fd.write(json.dumps(evv_output_data)) - return case - -def create_simple_case(): +def create_simple_case(model="e3sm", resubmit=0, generate_baseline=False): case = mock.MagicMock() case.get_value.side_effect = ( @@ -90,9 +94,9 @@ def create_simple_case(): "MVK.f19_g16.S.docker_gnu", # CASEBASEID "mct", # COMP_INTERFACE "MVK.f19_g16.S.docker_gnu", # CASEBASEID - "e3sm", # MODEL - 0, # RESUBMIT - False, # GENERATE_BASELINE + model, + resubmit, + generate_baseline, ) return case @@ -192,6 +196,125 @@ def test_config(case, run_dir, base_dir, evv_lib_dir): assert lines == ["var2 = value2\n"] + @mock.patch("CIME.SystemTests.mvk.utils.append_testlog") + @mock.patch("CIME.SystemTests.mvk.Machines") + def test_update_testlog(self, machines, append_testlog): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + + run_dir.mkdir(parents=True) + + evv_output_path = run_dir / "index.json" + + write_evv_output(evv_output_path, True) + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + machines.return_value.get_value.return_value = "docker" + + case = create_complex_case(case_name, temp_dir, run_dir, baseline_dir) + + test = MVK(case) + + test.update_testlog("test1", case_name, str(run_dir)) + + append_testlog.assert_any_call( + """BASELINE PASS for test 'test1'. + Test status: pass; Variables analyzed: v1; Rejecting: 2; Critical value: 12 + EVV results can be viewed at: + docker/evv/MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2/index.html""", + str(temp_dir), + ) + + @mock.patch("CIME.SystemTests.mvk.utils.get_urlroot") + @mock.patch("CIME.SystemTests.mvk.utils.append_testlog") + @mock.patch("CIME.SystemTests.mvk.Machines") + def test_update_testlog_urlroot_None(self, machines, append_testlog, get_urlroot): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + + run_dir.mkdir(parents=True) + + evv_output_path = run_dir / "index.json" + + write_evv_output(evv_output_path, True) + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + machines.return_value.get_value.return_value = "docker" + + get_urlroot.return_value = None + + case = create_complex_case(case_name, temp_dir, run_dir, baseline_dir) + + test = MVK(case) + + test.update_testlog("test1", case_name, str(run_dir)) + + print(append_testlog.call_args_list) + append_testlog.assert_any_call( + f"""BASELINE PASS for test 'test1'. + Test status: pass; Variables analyzed: v1; Rejecting: 2; Critical value: 12 + EVV results can be viewed at: + [{run_dir!s}_URL]/evv/MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2/index.html""", + str(temp_dir), + ) + + @mock.patch("CIME.SystemTests.mvk.utils.get_htmlroot") + @mock.patch("CIME.SystemTests.mvk.utils.append_testlog") + @mock.patch("CIME.SystemTests.mvk.Machines") + def test_update_testlog_htmlroot(self, machines, append_testlog, get_htmlroot): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + + run_dir.mkdir(parents=True) + + evv_output_path = run_dir / "index.json" + + write_evv_output(evv_output_path, True) + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + machines.return_value.get_value.return_value = "docker" + + get_htmlroot.return_value = None + + case = create_complex_case(case_name, temp_dir, run_dir, baseline_dir) + + test = MVK(case) + + test.update_testlog("test1", case_name, str(run_dir)) + + append_testlog.assert_any_call( + f"""BASELINE PASS for test 'test1'. + Test status: pass; Variables analyzed: v1; Rejecting: 2; Critical value: 12 + EVV results can be viewed at: + {run_dir!s} + EVV viewing instructions can be found at: https://github.com/E3SM-Project/E3SM/blob/master/cime/scripts/climate_reproducibility/README.md#test-passfail-and-extended-output""", + str(temp_dir), + ) + @mock.patch("CIME.SystemTests.mvk.test_mods.find_test_mods") @mock.patch("CIME.SystemTests.mvk.evv") def test_testmod_simple(self, evv, find_test_mods): @@ -226,6 +349,7 @@ def test_testmod_simple(self, evv, find_test_mods): case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE case = create_complex_case(case_name, temp_dir, run_dir, baseline_dir) + test = MVK(case) stack.enter_context(mock.patch.object(test, "build_indv")) @@ -284,6 +408,150 @@ def test_testmod_simple(self, evv, find_test_mods): "seed_clock = .true.\n", ] + @mock.patch("CIME.SystemTests.mvk.case_setup") + @mock.patch("CIME.SystemTests.mvk.MVK.build_indv") + def test_build_phase(self, build_indv, case_setup): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + case = create_complex_case( + case_name, temp_dir, run_dir, baseline_dir, True, mock_evv_output=True + ) + + case.get_values.side_effect = (("CPL", "LND"),) + + side_effect = [x for x in case.get_value.side_effect] + + n = 7 + side_effect.insert(n, 8) + side_effect.insert(n, 16) + + case.get_value.side_effect = side_effect + + test = MVK(case) + + test.build_phase(sharedlib_only=True) + + case.set_value.assert_any_call("NTHRDS_CPL", 1) + case.set_value.assert_any_call("NTASKS_CPL", 480) + case.set_value.assert_any_call("NTHRDS_LND", 1) + case.set_value.assert_any_call("NTASKS_LND", 240) + case.set_value.assert_any_call("NINST_LND", 30) + + case.flush.assert_called() + + case_setup.assert_any_call(case, test_mode=False, reset=True) + + @mock.patch("CIME.SystemTests.mvk.SystemTestsCommon._generate_baseline") + @mock.patch("CIME.SystemTests.mvk.utils.append_testlog") + @mock.patch("CIME.SystemTests.mvk.evv") + def test__generate_baseline(self, evv, append_testlog, _generate_baseline): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + case = create_complex_case( + case_name, temp_dir, run_dir, baseline_dir, True, mock_evv_output=True + ) + + # use original 5 args + side_effect = [x for x in case.get_value.side_effect][:7] + + side_effect.extend( + [ + str(baseline_dir), + "MVK.f19_g16.S", + str(run_dir), + "MVK.f19_g16.S", + case_name, + ] + ) + + case.get_value.side_effect = side_effect + + case_baseline_dir = baseline_dir / "MVK.f19_g16.S" / "eam" + + case_baseline_dir.mkdir(parents=True, exist_ok=True) + + (run_dir / "eam").mkdir(parents=True, exist_ok=True) + + (run_dir / "eam" / "test1.nc").touch() + (run_dir / "eam" / "test2.nc").touch() + + case.get_env.return_value.get_all_hist_files.return_value = ( + "eam/test1.nc", + "eam/test2.nc", + ) + + test = MVK(case) + + test._generate_baseline() + + files = os.listdir(case_baseline_dir) + + assert files == ["test1.nc", "test2.nc"] + + # reset side_effect + case.get_value.side_effect = side_effect + + test = MVK(case) + + # test baseline_dir already exists + test._generate_baseline() + + files = os.listdir(case_baseline_dir) + + assert files == ["test1.nc", "test2.nc"] + + @mock.patch("CIME.SystemTests.mvk.utils.append_testlog") + @mock.patch("CIME.SystemTests.mvk.evv") + def test__compare_baseline_resubmit(self, evv, append_testlog): + with contextlib.ExitStack() as stack: + temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) + + stack.enter_context(chdir(temp_dir)) + + # convert to Path + temp_dir = Path(temp_dir) + run_dir = temp_dir / "run" + baseline_dir = temp_dir / "baselines" + + case_name = "MVK.f19_g16.S.docker_gnu.20240515_212034_41b5u2" # CASE + + case = create_complex_case( + case_name, temp_dir, run_dir, baseline_dir, True, mock_evv_output=True + ) + + side_effect = [x for x in case.get_value.side_effect][:-8] + + side_effect.extend([1, 1]) + + case.get_value.side_effect = side_effect + + test = MVK(case) + + with mock.patch.object(test, "_test_status") as _test_status: + test._compare_baseline() + + _test_status.set_status.assert_any_call("BASELINE", "PASS") + @mock.patch("CIME.SystemTests.mvk.utils.append_testlog") @mock.patch("CIME.SystemTests.mvk.evv") def test__compare_baseline(self, evv, append_testlog): @@ -402,6 +670,25 @@ def test_write_inst_nml(self): "seed_clock = .true.\n", ] + def test_compare_baseline(self): + case = create_simple_case() + + MVK(case) + + case.set_value.assert_any_call("COMPARE_BASELINE", True) + + case = create_simple_case(generate_baseline=True) + + MVK(case) + + case.set_value.assert_any_call("COMPARE_BASELINE", False) + + case = create_simple_case(resubmit=1, generate_baseline=True) + + MVK(case) + + case.set_value.assert_any_call("COMPARE_BASELINE", False) + def test_mvk(self): case = create_simple_case() @@ -409,3 +696,10 @@ def test_mvk(self): assert test._config.component == "eam" assert test._config.components == ["eam"] + + case = create_simple_case("cesm") + + test = MVK(case) + + assert test._config.component == "cam" + assert test._config.components == ["cam"] From 00befab23fe0f9e8aaf2e287ab10ed7f1389b03e Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 22 Jul 2024 20:05:35 -0700 Subject: [PATCH 28/46] Removes critical value option --- CIME/SystemTests/mvk.py | 4 ---- CIME/tests/test_unit_system_tests_mvk.py | 3 --- 2 files changed, 7 deletions(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index fdbedc6f7dd..473906f350e 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -43,9 +43,6 @@ def __init__(self): "components", [], "Components that require namelist customization." ) self._set_attribute("ninst", 30, "The number of instances.") - self._set_attribute( - "critical", 13, "The critical value for rejecting the null hypothese." - ) self._set_attribute( "var_set", "default", "Name of the variable set to analyze." ) @@ -94,7 +91,6 @@ def test_config( "ref-dir": base_dir, "var-set": self.var_set, "ninst": self.ninst, - "critical": self.critical, "component": self.component, } diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py index a559d8bffb4..fbf3c18ec10 100644 --- a/CIME/tests/test_unit_system_tests_mvk.py +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -339,7 +339,6 @@ def test_testmod_simple(self, evv, find_test_mods): component = "new-comp" components = ["new-comp", "second-comp"] ninst = 8 -critical = 32 var_set = "special" ref_case = "Reference" test_case = "Default" @@ -368,7 +367,6 @@ def test_testmod_simple(self, evv, find_test_mods): "ref-dir": f"{baseline_dir}/", "var-set": "special", "ninst": 8, - "critical": 32, "component": "new-comp", } } @@ -586,7 +584,6 @@ def test__compare_baseline(self, evv, append_testlog): "ref-dir": f"{baseline_dir}/", "var-set": "default", "ninst": 30, - "critical": 13, "component": "eam", } } From 9af8a4d781609be21027108ba6786c93f65bbc32 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 22 Jul 2024 22:00:49 -0700 Subject: [PATCH 29/46] Changes version check to lower bounds --- CIME/SystemTests/mvk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 473906f350e..8ae0055cd5f 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -24,7 +24,7 @@ version = evv4esm.__version_info__ -assert version[0] <= 0 and version[1] <= 5, "Please install evv4esm less than 0.5.x" +assert version >= (0, 5, 0), "Please install evv4esm greater or equal to 0.5.0" EVV_LIB_DIR = os.path.abspath(os.path.dirname(evv4esm.__file__)) From e3514c6fd72c1463c34f5b7a316f35c7381059b9 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 22 Jul 2024 22:01:33 -0700 Subject: [PATCH 30/46] Removes critical value from docs --- doc/source/users_guide/testing.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/users_guide/testing.rst b/doc/source/users_guide/testing.rst index 8a35a51560c..1a5d947ccb9 100644 --- a/doc/source/users_guide/testing.rst +++ b/doc/source/users_guide/testing.rst @@ -296,7 +296,6 @@ Variable Default Type Description component str The main component. components [] list Components that require namelist customization. ninst 30 int The number of instances. -critical 13 int The critical value for rejecting the null hypothese. var_set default str Name of the variable set to analyze. ref_case Baseline str Name of the reference case. test_case Test str Name of the test case. From 74393700cdc1fb9261fcbc2ee89114fb14031985 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 22 Jul 2024 22:54:19 -0700 Subject: [PATCH 31/46] Renames test_config to evv_test_config --- CIME/SystemTests/mvk.py | 4 ++-- CIME/tests/test_unit_system_tests_mvk.py | 2 +- doc/source/users_guide/testing.rst | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 8ae0055cd5f..927638610a2 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -67,7 +67,7 @@ def write_inst_nml( set_nml_variable("seed_custom", f"{iinst}") set_nml_variable("seed_clock", ".true.") - def test_config( + def evv_test_config( self, case, run_dir, base_dir, evv_lib_dir ): # pylint: disable=unused-argument """Configure the evv test. @@ -230,7 +230,7 @@ def _compare_baseline(self): test_name = "{}".format(case_name.split(".")[-1]) - test_config = self._config.test_config( + test_config = self._config.evv_test_config( self._case, run_dir, base_dir, EVV_LIB_DIR ) diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py index fbf3c18ec10..07e1cc9c43b 100644 --- a/CIME/tests/test_unit_system_tests_mvk.py +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -144,7 +144,7 @@ def write_inst_nml(case, set_nml_variable, component, iinst): elif component == "secondary-comp": set_nml_variable("var2", "value2") -def test_config(case, run_dir, base_dir, evv_lib_dir): +def evv_test_config(case, run_dir, base_dir, evv_lib_dir): return { "module": os.path.join(evv_lib_dir, "extensions", "kso.py"), "component": "someother-comp" diff --git a/doc/source/users_guide/testing.rst b/doc/source/users_guide/testing.rst index 1a5d947ccb9..ba4425c8e45 100644 --- a/doc/source/users_guide/testing.rst +++ b/doc/source/users_guide/testing.rst @@ -310,7 +310,7 @@ Available methods for the MVK test type. .. code-block:: - def test_config(case, run_dir, base_dir, evv_lib_dir): + def evv_test_config(case, run_dir, base_dir, evv_lib_dir): """ Configure the evv test. @@ -366,7 +366,7 @@ A complex customization of the `MVK` :ref:`test type ` would be defini component = "eam" components = ["eam", "clm"] - def test_config(case, run_dir, base_dir, evv_lib_dir): + def evv_test_config(case, run_dir, base_dir, evv_lib_dir): return { "module": os.path.join(evv_lib_dir, "extensions", "kso.py"), "component": component, From 4c9c3689b8c605b4f5f9b1dbbdb5eb7c96703d3e Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Mon, 22 Jul 2024 23:19:59 -0700 Subject: [PATCH 32/46] Enable debugging --- .github/workflows/testing.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 9c84f4f8ca3..cfc62606ff0 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -117,6 +117,8 @@ jobs: pip install -r test-requirements.txt + export DEBUG=true + source /entrypoint.sh # GitHub runner home is different than container From 95e6bea2e069fcf6a9b4987da926db752ef0ff62 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 25 Jul 2024 17:04:22 -0700 Subject: [PATCH 33/46] Updates namelist to support groupless namelist --- CIME/namelist.py | 47 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/CIME/namelist.py b/CIME/namelist.py index 43907a9fbd1..c20c478f1ab 100644 --- a/CIME/namelist.py +++ b/CIME/namelist.py @@ -156,6 +156,17 @@ FORTRAN_REPEAT_PREFIX_REGEX = re.compile(r"^[0-9]*[1-9]+[0-9]*\*") +def convert_bool(value): + if isinstance(value, bool): + value = f".{str(value).lower()}." + elif isinstance(value, str): + value = f".{value.lower()}." + else: + raise ValueError("Unable to convert {}".format(value)) + + return value + + def is_valid_fortran_name(string): """Check that a variable name is allowed in Fortran. @@ -1044,6 +1055,11 @@ def set_variable_value(self, group_name, variable_name, value, var_size=1): >>> x.get_variable_value('foo', 'red') ['', '2', '', '4', '', '6'] """ + if not isinstance(value, (set, list)): + value = [ + value, + ] + minindex, maxindex, step = get_fortran_variable_indices(variable_name, var_size) variable_name = get_fortran_name_only(variable_name) @@ -1211,7 +1227,7 @@ def _write(self, out_file, groups, format_, sorted_groups): else: group_names = groups for group_name in group_names: - if format_ == "nml": + if group_name != "" and format_ == "nml": out_file.write("&{}\n".format(group_name)) # allow empty group if group_name in self._groups: @@ -1227,18 +1243,23 @@ def _write(self, out_file, groups, format_, sorted_groups): # To prettify things for long lists of values, build strings # line-by-line. - if values[0] == "True" or values[0] == "False": - values[0] = ( - values[0] - .replace("True", ".true.") - .replace("False", ".false.") - ) - lines = [" {}{} {}".format(name, equals, values[0])] + if isinstance(values[0], bool) or values[0].lower() in ( + "true", + "false", + ): + values[0] = convert_bool(values[0]) + + if group_name == "": + lines = ["{}{} {}".format(name, equals, values[0])] + else: + lines = [" {}{} {}".format(name, equals, values[0])] for value in values[1:]: - if value == "True" or value == "False": - value = value.replace("True", ".true.").replace( - "False", ".false." - ) + if isinstance(value, bool) or value.lower() in ( + "true", + "false", + ): + value = convert_bool(value) + if len(lines[-1]) + len(value) <= 77: lines[-1] += ", " + value else: @@ -1247,7 +1268,7 @@ def _write(self, out_file, groups, format_, sorted_groups): lines[-1] += "\n" for line in lines: out_file.write(line) - if format_ == "nml": + if group_name != "" and format_ == "nml": out_file.write("/\n") if format_ == "nmlcontents": out_file.write("\n") From 325340bab8d137e335774854f0384f2ed668b469 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 25 Jul 2024 18:56:44 -0700 Subject: [PATCH 34/46] Fixes overwriting existing group --- CIME/namelist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/namelist.py b/CIME/namelist.py index c20c478f1ab..f8db910fe4a 100644 --- a/CIME/namelist.py +++ b/CIME/namelist.py @@ -1070,7 +1070,7 @@ def set_variable_value(self, group_name, variable_name, value, var_size=1): ), ) gn = string_in_list(group_name, self._groups) - if not gn: + if gn is None: gn = group_name self._groups[gn] = {} From 26328ced22615a41d42cf4d48b7a5bfb9ae1d215 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 26 Jul 2024 11:18:26 -0700 Subject: [PATCH 35/46] Adds __call__ to handle writing the output. --- CIME/namelist.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CIME/namelist.py b/CIME/namelist.py index f8db910fe4a..562f0ca55ac 100644 --- a/CIME/namelist.py +++ b/CIME/namelist.py @@ -102,6 +102,7 @@ import re import collections +from contextlib import contextmanager # Disable these because this is our standard setup # pylint: disable=wildcard-import,unused-wildcard-import @@ -937,6 +938,13 @@ def __init__(self, groups=None): variable_name ] + @contextmanager + def __call__(self, filename): + try: + yield self + finally: + self.write(filename) + def clean_groups(self): self._groups = collections.OrderedDict() From df2ddd9d334a694805d17d7c8d0c69a6ab0ed330 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 26 Jul 2024 17:10:55 -0700 Subject: [PATCH 36/46] Updates functions and docs --- CIME/SystemTests/mvk.py | 53 +++-- CIME/config.py | 12 +- CIME/tests/test_unit_system_tests_mvk.py | 23 ++- CIME/utils.py | 2 +- doc/source/users_guide/testing.rst | 253 ++++++++++++++--------- 5 files changed, 208 insertions(+), 135 deletions(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 927638610a2..621c6986915 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -18,6 +18,7 @@ from CIME.XML.machines import Machines from CIME.config import ConfigBase from CIME.SystemTests import test_mods +from CIME.namelist import Namelist import evv4esm # pylint: disable=import-error from evv4esm.__main__ import main as evv # pylint: disable=import-error @@ -49,30 +50,35 @@ def __init__(self): self._set_attribute("ref_case", "Baseline", "Name of the reference case.") self._set_attribute("test_case", "Test", "Name of the test case.") - def write_inst_nml( - self, case, set_nml_variable, component, iinst + def generate_namelist( + self, case, component, i, filename ): # pylint: disable=unused-argument - """Write per instance namelist. + """Generate per instance namelist. - This method is called once per instance to generate the namelist. + This method is called for each instance to generate the desired + modifications. Args: case (CIME.case.case.Case): The case instance. - write_nml_variable (function): Function takes two `str` arguments. component (str): Component the namelist belongs to. - iinst (int): Instance unique number. + i (int): Instance unique number. + filename (str): Name of the namelist that needs to be created. """ - set_nml_variable("new_random", ".true.") - set_nml_variable("pertlim", "1.0e-10") - set_nml_variable("seed_custom", f"{iinst}") - set_nml_variable("seed_clock", ".true.") + namelist = Namelist() + + with namelist(filename) as nml: + nml.set_variable_value("", "new_random", True) + nml.set_variable_value("", "pertlim", "1.0e-10") + nml.set_variable_value("", "seed_custom", f"{i}") + nml.set_variable_value("", "seed_clock", True) def evv_test_config( self, case, run_dir, base_dir, evv_lib_dir ): # pylint: disable=unused-argument - """Configure the evv test. + """Generate the evv4esm configuration file. - This method is used to pass the evv4esm configuration to be written for the test. + This method is used to generate the evv4esm configuration that will + be written to `$RUNDIR/$CASE.json`. Args: case (CIME.case.case.Case): The case instance. @@ -128,6 +134,11 @@ def __init__(self, case, **kwargs): if len(self._config.components) == 0: self._config.components = [self._config.component] + elif ( + self._config.component != "" + and self._config.component not in self._config.components + ): + self._config.components.extend([self._config.component]) if ( self._case.get_value("RESUBMIT") == 0 @@ -159,18 +170,11 @@ def build_phase(self, sharedlib_only=False, model_only=False): case_setup(self._case, test_mode=False, reset=True) - for iinst in range(1, self._config.ninst + 1): + for i in range(1, self._config.ninst + 1): for component in self._config.components: - with open( - "user_nl_{}_{:04d}".format(component, iinst), "w" - ) as nml_file: - set_nml_variable = lambda key, value: nml_file.write( # pylint: disable=cell-var-from-loop - f"{key} = {value}\n" - ) + filename = "user_nl_{}_{:04d}".format(component, i) - self._config.write_inst_nml( - self._case, set_nml_variable, component, iinst - ) + self._config.generate_namelist(self._case, component, i, filename) self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) @@ -317,3 +321,8 @@ def process_evv_output(self, evv_out_dir): break return comments + + +if __name__ == "__main__": + _config = MVKConfig() + _config.print_rst_table() diff --git a/CIME/config.py b/CIME/config.py index e289b1309ce..2b17f17c562 100644 --- a/CIME/config.py +++ b/CIME/config.py @@ -14,11 +14,13 @@ DEFAULT_CUSTOMIZE_PATH = os.path.join(utils.get_src_root(), "cime_config", "customize") -def print_rst_header(header): +def print_rst_header(header, anchor=None, separator='"'): n = len(header) - print("-" * n) + if anchor is not None: + print(f".. _{anchor}\n") + print(separator * n) print(header) - print("-" * n) + print(separator * n) def print_rst_table(headers, *rows): @@ -149,7 +151,7 @@ def print_rst_table(self): self.print_method_rst() def print_variable_rst(self): - print_rst_header("Variables") + print_rst_header("Variables", anchor=f"{self.__class__.__name__} Variables:") headers = ("Variable", "Default", "Type", "Description") @@ -161,7 +163,7 @@ def print_variable_rst(self): print_rst_table(headers, *rows) def print_method_rst(self): - print_rst_header("Methods") + print_rst_header("Methods", anchor=f"{self.__class__.__name__} Methods:") methods = inspect.getmembers(self, inspect.ismethod) diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py index 07e1cc9c43b..6033ac83c99 100644 --- a/CIME/tests/test_unit_system_tests_mvk.py +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -133,16 +133,21 @@ def test_testmod_complex(self, evv, find_test_mods): fd.write( """ import os +from CIME.namelist import Namelist component = "new-comp" components = ["new-comp", "secondary-comp"] ninst = 8 -def write_inst_nml(case, set_nml_variable, component, iinst): +def generate_namelist(case, component, i, filename): + nml = Namelist() + if component == "new-comp": - set_nml_variable("var1", "value1") + nml.set_variable_value("", "var1", "value1") elif component == "secondary-comp": - set_nml_variable("var2", "value2") + nml.set_variable_value("", "var2", "value2") + + nml.write(filename) def evv_test_config(case, run_dir, base_dir, evv_lib_dir): return { @@ -392,8 +397,8 @@ def test_testmod_simple(self, evv, find_test_mods): assert lines == [ "new_random = .true.\n", "pertlim = 1.0e-10\n", - "seed_custom = 1\n", "seed_clock = .true.\n", + "seed_custom = 1\n", ] with open(sorted(nml_files)[-1], "r") as fd: @@ -402,8 +407,8 @@ def test_testmod_simple(self, evv, find_test_mods): assert lines == [ "new_random = .true.\n", "pertlim = 1.0e-10\n", - "seed_custom = 8\n", "seed_clock = .true.\n", + "seed_custom = 8\n", ] @mock.patch("CIME.SystemTests.mvk.case_setup") @@ -609,7 +614,7 @@ def test__compare_baseline(self, evv, append_testlog): expected_comments, str(temp_dir) ), append_testlog.call_args.args - def test_write_inst_nml_multiple_components(self): + def test_generate_namelist_multiple_components(self): with contextlib.ExitStack() as stack: temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) @@ -635,11 +640,11 @@ def test_write_inst_nml_multiple_components(self): assert lines == [ "new_random = .true.\n", "pertlim = 1.0e-10\n", - "seed_custom = 1\n", "seed_clock = .true.\n", + "seed_custom = 1\n", ] - def test_write_inst_nml(self): + def test_generate_namelist(self): with contextlib.ExitStack() as stack: temp_dir = stack.enter_context(tempfile.TemporaryDirectory()) @@ -663,8 +668,8 @@ def test_write_inst_nml(self): assert lines == [ "new_random = .true.\n", "pertlim = 1.0e-10\n", - "seed_custom = 1\n", "seed_clock = .true.\n", + "seed_custom = 1\n", ] def test_compare_baseline(self): diff --git a/CIME/utils.py b/CIME/utils.py index 864f0299b27..4f42094318e 100644 --- a/CIME/utils.py +++ b/CIME/utils.py @@ -1583,7 +1583,7 @@ def get_charge_account(machobj=None, project=None): >>> import CIME >>> import CIME.XML.machines - >>> machobj = CIME.XML.machines.Machines(machine="theta") + >>> machobj = CIME.XML.machines.Machines(machine="docker") >>> project = get_project(machobj) >>> charge_account = get_charge_account(machobj, project) >>> project == charge_account diff --git a/doc/source/users_guide/testing.rst b/doc/source/users_guide/testing.rst index ba4425c8e45..51f54292e1b 100644 --- a/doc/source/users_guide/testing.rst +++ b/doc/source/users_guide/testing.rst @@ -4,33 +4,34 @@ Testing Cases ************** -`create_test <../Tools_user/create_test.html>`_ -is a powerful system testing capability provided by the CIME Case Control System. -create_test can, in one command, create a case, setup, build and run the case -according to the test type and return a PASS or FAIL for the test result. +The `create_test <../Tools_user/create_test.html>`_ command provides +a powerful tool capable of testing a Case. The command can create, +setup, build and run a case according to the :ref:`testname ` syntax, returning +a PASS or FAIL result. .. _individual: An individual test can be run as:: - $CIMEROOT/scripts/create_test $test_name + $CIMEROOT/scripts/create_test -Everything the test will do is controlled by parsing the test name. +Everything the test will do is controlled by the :ref:`testname `. -.. _`Test naming`: +.. _`testname syntax`: -================= +================ Testname syntax -================= +================ -Tests must be named with the following forms, [ ]=optional:: +Tests are defined by the following format, where anything enclosed in ``[]`` is optional:: TESTTYPE[_MODIFIERS].GRID.COMPSET[.MACHINE_COMPILER][.GROUP-TESTMODS] -For example using the minimum required elements of a testname:: +For example using the minimum TESTTYPE_, `GRID <../users_guide/grids.html>`_, and `COMPSET <../users_guide/compsets.html>`_:: - $CIMEROOT/scripts/create_test ERP.ne4pg2_oQU480.F2010 + ERP.ne4pg2_oQU480.F2010 +Below is a break-down of the different parts of the ``testname`` syntax. ================= ===================================================================================== NAME PART @@ -166,7 +167,7 @@ elements of the test through a test type modifier. .. _MODIFIERS: ------------------- -Testtype Modifiers +MODIFIERS ------------------- ============ ===================================================================================== @@ -200,121 +201,159 @@ MODIFIERS Description For example, this will run the ERP test with debugging turned on during compilation:: - CIMEROOT/scripts/create_test ERP_D.ne4pg2_oQU480.F2010 + $CIMEROOT/scripts/create_test ERP_D.ne4pg2_oQU480.F2010 This will run the ERP test for 3 days instead of the default 11 days:: - CIMEROOT/scripts/create_test ERP_Ld3.ne4pg2_oQU480.F2010 + $CIMEROOT/scripts/create_test ERP_Ld3.ne4pg2_oQU480.F2010 You can combine testtype modifiers:: - CIMEROOT/scripts/create_test ERP_D_Ld3.ne4pg2_oQU480.F2010 + $CIMEROOT/scripts/create_test ERP_D_Ld3.ne4pg2_oQU480.F2010 .. _GROUP-TESTMODS: ------------------- -Test Case Modifiers +GROUP-TESTMODS ------------------- -create_test runs with out-of-the-box compsets and grid sets. Sometimes you may want to run a test with -modification to a namelist or other setting without creating an entire compset. CCS provides the testmods -capability for this situation. - -A testmod is a string at the end of the full testname (including machine and compiler) -with the form GROUP-TESTMODS which are parsed by create_test as follows: +The `create_test <../Tools_user/create_test.html>`_ command runs with out-of-the-box compsets and grid sets. +Sometimes you may want to run a test with modification to a namelist or other setting without creating an +entire compset. Case Control System (CCS) provides the testmods capability for this situation. +The ``GROUP-TESTMODS`` string is at the end of the full :ref:`testname ` (including machine and compiler). +The form ``GROUP-TESTMODS`` are parsed as follows. ============ ===================================================================================== -TESTMOD Description +PART Description ============ ===================================================================================== -GROUP Define the subdirectory of testmods_dirs and the parent directory of various testmods. - -TESTMODS A subdirectory of GROUP containing files which set non-default values - of the set-up and run-time variables via namelists or xml_change commands. - Example: - - | GROUP-TESTMODS = cam-outfrq9s points to - | $cesm/components/cam/cime_config/testdefs/testmods_dirs/cam/outfrq9s - | while allactive-defaultio points to - | $cesm/cime_config/testmods_dirs/allactive/defaultio +GROUP Name of the directory under ``TESTS_MODS_DIR`` that contains ``TESTMODS``. +TESTMODS Any combination of `user_nl_* `_, `shell_commands `_, + `user_mods `_, or `params.py `_ in a directory under the + ``GROUP`` directory. ============ ===================================================================================== -For example, the ERP test for an E3SM F-case can be modified to use a different radiation scheme:: +For example, the *ERP* test for an E3SM *F-case* can be modified to use a different radiation scheme by using ``eam-rrtmgp``:: - CIMEROOT/scripts/create_test ERP_D_Ld3.ne4pg2_oQU480.F2010.pm-cpu_intel.eam-rrtmgp + ERP_D_Ld3.ne4pg2_oQU480.F2010.pm-cpu_intel.eam-rrtmgp -This tells create_test to look in $e3sm/components/eam/cime_config/testdefs/testmods_dirs/eam/rrtmpg -where it finds the following lines in the shell_commands file:: +If ``TESTS_MODS_DIR`` was set to ``$E3SM/components/eam/cime_config/testdefs/testmods_dirs`` then the +directory containg the testmods woulc be ``$E3SM/components/eam/cime_config/testdefs/testmods_dirs/eam/rrtmpg``. - #!/bin/bash - ./xmlchange --append CAM_CONFIG_OPTS='-rad rrtmgp' +In this directory you'd find a `shell_commands`` file containing the following:: + + #!/bin/bash + ./xmlchange --append CAM_CONFIG_OPTS='-rad rrtmgp' These commands are applied after the testcase is created and case.setup is called. -The contents of each testmods directory can include +Note; do not use '-' in the testmods directory name because it has a special meaning to create_test. + +.. _USER_NL: + +```````` +Example *user_nl_* +```````` + +A components namelist can be modified by providing a ``user_nl_*`` file in a GROUP-TESTMODS_ directory. +For example, to change the namelist for the *eam* component a file name ``user_nl_eam`` could be used. + :: - user_nl_$components namelist variable=value pairs - shell_commands xmlchange commands - user_mods a list of other GROUP-TESTMODS which should be imported - but at a lower precedence than the local testmods. + # user_nl_eam + deep_scheme = 'off', + zmconv_microp = .false. + shallow_scheme = 'CLUBB_SGS', + l_tracer_aero = .false. + l_rayleigh = .false. + l_gw_drag = .false. + l_ac_energy_chk = .true. + l_bc_energy_fix = .true. + l_dry_adj = .false. + l_st_mac = .true. + l_st_mic = .false. + l_rad = .false. -eam/cime_config/testdefs/testmods_dirs/eam contains modifications for eam in an F-case test. You -might make a directory called eam/cime_config/testdefs/testmods_dirs/elm to modify the land model -in an F-case test. +.. _SHELL_COMMANDS: -The "rrtmpg" directory contains the actual testmods to apply. -Note; do not use '-' in the testmods directory name because it has a special meaning to create_test. +`````````````` +Example *shell_commands* +`````````````` -.. _System Test Mods: +A test can be modified by providing a ``shell_commands`` file in a GROUP-TESTMODS_ directory. +This shell file can contain any arbitrary commands, for example:: ---------------------- -System Test Mods ---------------------- -`Test Case Modifiers`_ are not only a way to modify a test case but they can also be used to configure certain :ref:`Test types `. + # shell_commands + #!/bin/bash + + # Remove exe if chem pp exe (campp) already exists (it ensures that exe is always built) + /bin/rm -f $CIMEROOT/../components/eam/chem_proc/campp + + # Invoke campp (using v3 mechanism file) + ./xmlchange --append CAM_CONFIG_OPTS='-usr_mech_infile $CIMEROOT/../components/eam/chem_proc/inputs/pp_chemUCI_linozv3_mam5_vbs.in' + + # Assuming atmchange is available via $PATH + atmchange initial_conditions::perturbation_random_seed = 32 + +.. _USER_MODS: + +````````` +Example *user_mods* +````````` -Supported :ref:`test types ` can be configured by creating a ``params.py`` file in a :ref:`test case modifier `. +Additional GROUP_TESTMODS_ can be applied by providing a list in a ``user_mods`` file in a GROUP-TESTMODS_ directory. + +:: + + # user_mods + eam/cosp + eam/hommexx + +.. _TESTYPE_MOD: + +`````````````````````` +Example *params.py* +`````````````````````` + +Supported TESTYPES_ can further be modified by providing a ``params.py`` file in the GROUP-TESTMODS_ directory. ^^^^^^^^^^^^ MVK ^^^^^^^^^^^^ -The `MVK` system test can be configured by defining :ref:`variables ` and :ref:`methods ` in ``params.py``. +The `MVK` system test can be configured by defining :ref:`variables ` and :ref:`methods ` in ``params.py``. See :ref:`examples ` for a simple and complex use case. -.. _MVK Variables: +.. _MVKConfig Variables: """"""""" Variables """"""""" -Available settings for the MVK test type. - -========== ======== ==== ==================================================== -Variable Default Type Description -========== ======== ==== ==================================================== -component str The main component. -components [] list Components that require namelist customization. -ninst 30 int The number of instances. -var_set default str Name of the variable set to analyze. -ref_case Baseline str Name of the reference case. -test_case Test str Name of the test case. -========== ======== ==== ==================================================== - -.. _MVK Methods: +========== ======== ==== =============================================== +Variable Default Type Description +========== ======== ==== =============================================== +component str The main component. +components [] list Components that require namelist customization. +ninst 30 int The number of instances. +var_set default str Name of the variable set to analyze. +ref_case Baseline str Name of the reference case. +test_case Test str Name of the test case. +========== ======== ==== =============================================== + +.. _MVKConfig Methods: """"""" Methods """"""" -Available methods for the MVK test type. - .. code-block:: def evv_test_config(case, run_dir, base_dir, evv_lib_dir): """ - Configure the evv test. + Generate the evv4esm configuration file. - This method is used to pass the evv4esm configuration to be written for the test. + This method is used to generate the evv4esm configuration that will + be written to `$RUNDIR/$CASE.json`. Args: case (CIME.case.case.Case): The case instance. @@ -325,20 +364,20 @@ Available methods for the MVK test type. Returns: dict: Dictionary with test configuration. """ - .. code-block:: - def write_inst_nml(case, set_nml_variable, component, iinst): + def generate_namelist(case, component, i, filename): """ - Write per instance namelist. + Generate per instance namelist. - This method is called once per instance to generate the namelist. + This method is called for each instance to generate the desired + modifications. Args: case (CIME.case.case.Case): The case instance. - write_nml_variable (function): Function takes two `str` arguments. component (str): Component the namelist belongs to. - iinst (int): Instance unique number. + i (int): Instance unique number. + filename (str): Name of the namelist that needs to be created. """ .. _MVK Examples: @@ -347,7 +386,9 @@ Available methods for the MVK test type. Examples """""""""" .. _MVK Simple: -A simple customization of the `MVK` :ref:`test type ` would be just defining some :ref:`variables ` in ``params.py``. +In the simplest form just :ref:`variables ` need to be defined in ``params.py``. + +For this case the default ``evv_test_config`` and ``generate_namelist`` functions will be called. .. code-block:: @@ -357,29 +398,45 @@ A simple customization of the `MVK` :ref:`test type ` would be just de critical = 21 .. _MVK Complex: -A complex customization of the `MVK` :ref:`test type ` would be defining only the required :ref:`variables ` and defining some :ref:`methods ` in ``params.py``. + +If more control over the evv4esm configuration file or the per instance configuration is desired then +the ``evv_test_config`` and ``generate_namelist`` functions can be overridden in the ``params.py`` file. + +The only required variables is ``components``. + +In the following example, customizes the default evv4esm test config and writes out different +namelist and customized files per instance. .. code-block:: import os + from CIME.namelist import Namelist + from CIME.utils import run_cmd - component = "eam" - components = ["eam", "clm"] + components = ["eam", "clm", "eamxx"] def evv_test_config(case, run_dir, base_dir, evv_lib_dir): - return { - "module": os.path.join(evv_lib_dir, "extensions", "kso.py"), - "component": component, - } - - def write_inst_nml(case, set_nml_variable, component, iinst): - if component == "eam": - set_nml_variable("eam_specific", f"perturb-{iinst}") - elif component == "clm": - if iinst % 2 == 0: - set_nml_variable("clm_specific", "even") + return { + "module": os.path.join(evv_lib_dir, "extensions", "kso.py"), + "component": "eam" + "ninst": 20, + } + + def generate_namelist(case, component, i, filename): + namelist = Namelist() + + if component in ["eam", "clm"]: + with namelist(filename) as nml: + if component == "eam": + nml.set_variable_value("eam_specific", f"perturn-{i}") + elif component == "clm": + if i % 2 == 0: + nml.set_variable_value("clm_specific", "even") else: - set_nml_variable("clm_specific", "odd") + nml.set_variable_value("clm_specific", "odd") + else: + stat, output, err = run_cmd(f"atmchange initial_conditions::perturbation_random_seed = {i*32}") + ======================== Test progress and output From a695b6b0d880d2ff405429aad12b727c834ba991 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Fri, 26 Jul 2024 17:35:14 -0700 Subject: [PATCH 37/46] Changes how evv_test_config works --- CIME/SystemTests/mvk.py | 28 +++++++++------ CIME/config.py | 2 ++ CIME/tests/test_unit_system_tests_mvk.py | 17 ++++++--- doc/source/users_guide/testing.rst | 44 +++++++++++++++--------- 4 files changed, 59 insertions(+), 32 deletions(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 621c6986915..5a780096356 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -72,23 +72,24 @@ def generate_namelist( nml.set_variable_value("", "seed_custom", f"{i}") nml.set_variable_value("", "seed_clock", True) - def evv_test_config( - self, case, run_dir, base_dir, evv_lib_dir - ): # pylint: disable=unused-argument - """Generate the evv4esm configuration file. + def evv_test_config(self, case, config): # pylint: disable=unused-argument + """Customize the evv4esm configuration. + + This method is used to customize the default evv4esm configuration + or generate a completely new one. - This method is used to generate the evv4esm configuration that will - be written to `$RUNDIR/$CASE.json`. + The return configuration will be written to `$RUNDIR/$CASE.json`. Args: case (CIME.case.case.Case): The case instance. - run_dir (str): Path the case's run directory. - base_dir (str): Path to the case's baseline directory. - evv_lib_dir (str): Path to the evv4esm package root. + config (dict): Default evv4esm configuration. Returns: dict: Dictionary with test configuration. """ + return config + + def _default_evv_test_config(self, run_dir, base_dir, evv_lib_dir): config = { "module": os.path.join(evv_lib_dir, "extensions", "ks.py"), "test-case": self.test_case, @@ -234,8 +235,15 @@ def _compare_baseline(self): test_name = "{}".format(case_name.split(".")[-1]) + default_config = self._config._default_evv_test_config( + run_dir, + base_dir, + EVV_LIB_DIR, + ) + test_config = self._config.evv_test_config( - self._case, run_dir, base_dir, EVV_LIB_DIR + self._case, + default_config, ) evv_config = {test_name: test_config} diff --git a/CIME/config.py b/CIME/config.py index 2b17f17c562..f63a4bea78d 100644 --- a/CIME/config.py +++ b/CIME/config.py @@ -186,6 +186,8 @@ def print_method_rst(self): ] for (name, sig, doc) in child_methods: + if doc is None: + continue print(".. code-block::\n") print(f" def {name}{sig!s}:") print(' """') diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py index 6033ac83c99..002dabd75f8 100644 --- a/CIME/tests/test_unit_system_tests_mvk.py +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -134,6 +134,7 @@ def test_testmod_complex(self, evv, find_test_mods): """ import os from CIME.namelist import Namelist +from CIME.SystemTests.mvk import EVV_LIB_DIR component = "new-comp" components = ["new-comp", "secondary-comp"] @@ -149,11 +150,11 @@ def generate_namelist(case, component, i, filename): nml.write(filename) -def evv_test_config(case, run_dir, base_dir, evv_lib_dir): - return { - "module": os.path.join(evv_lib_dir, "extensions", "kso.py"), - "component": "someother-comp" - } +def evv_test_config(case, config): + config["module"] = os.path.join(EVV_LIB_DIR, "extensions", "kso.py") + config["component"] = "someother-comp" + + return config """ ) @@ -173,6 +174,12 @@ def evv_test_config(case, run_dir, base_dir, evv_lib_dir): expected_config = { "20240515_212034_41b5u2": { "component": "someother-comp", + "ninst": 8, + "ref-case": "Baseline", + "ref-dir": f"{temp_dir}/baselines/", + "test-case": "Test", + "test-dir": f"{temp_dir}/run", + "var-set": "default", } } diff --git a/doc/source/users_guide/testing.rst b/doc/source/users_guide/testing.rst index 51f54292e1b..21e9f88f2be 100644 --- a/doc/source/users_guide/testing.rst +++ b/doc/source/users_guide/testing.rst @@ -348,18 +348,18 @@ Methods """"""" .. code-block:: - def evv_test_config(case, run_dir, base_dir, evv_lib_dir): + def evv_test_config(case, config): """ - Generate the evv4esm configuration file. + Customize the evv4esm configuration. - This method is used to generate the evv4esm configuration that will - be written to `$RUNDIR/$CASE.json`. + This method is used to customize the default evv4esm configuration + or generate a completely new one. + + The return configuration will be written to `$RUNDIR/$CASE.json`. Args: case (CIME.case.case.Case): The case instance. - run_dir (str): Path the case's run directory. - base_dir (str): Path to the case's baseline directory. - evv_lib_dir (str): Path to the evv4esm package root. + config (dict): Default evv4esm configuration. Returns: dict: Dictionary with test configuration. @@ -370,7 +370,7 @@ Methods """ Generate per instance namelist. - This method is called for each instance to generate the desired + This method is called for each instance to generate the desired modifications. Args: @@ -402,25 +402,35 @@ For this case the default ``evv_test_config`` and ``generate_namelist`` function If more control over the evv4esm configuration file or the per instance configuration is desired then the ``evv_test_config`` and ``generate_namelist`` functions can be overridden in the ``params.py`` file. -The only required variables is ``components``. +The :ref:`variables ` will still need to be defined to generate the default +evv4esm config or ``config`` in the ``evv_test_config`` function can be ignored and a completely new +dictionary can be returned. + +In the following example, the default ``module`` is changed as well as ``component`` and ``ninst``. +The ``generate_namelist`` function creates namelists for certain components while running a shell +command to customize others. -In the following example, customizes the default evv4esm test config and writes out different -namelist and customized files per instance. +Note; this is a toy example, no scientific usage. .. code-block:: import os + from CIME.SystemTests.mvk import EVV_LIB_DIR from CIME.namelist import Namelist from CIME.utils import run_cmd + component "eam" + # The generate_namelist function will be called `ninst` times per component components = ["eam", "clm", "eamxx"] + ninst = 30 + + # This can be omitted if the default evv4esm configuration is sufficient + def evv_test_config(case, config): + config["module"] = os.path.join(EVV_LIB_DIR, "extensions", "kso.py") + config["component"] = "clm" + config["ninst"] = 20 - def evv_test_config(case, run_dir, base_dir, evv_lib_dir): - return { - "module": os.path.join(evv_lib_dir, "extensions", "kso.py"), - "component": "eam" - "ninst": 20, - } + return config def generate_namelist(case, component, i, filename): namelist = Namelist() From f7f0bcdae10784f33f206113b627323299835eb3 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 22 Aug 2024 11:34:55 -0700 Subject: [PATCH 38/46] Fixes black formatting --- CIME/code_checker.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/CIME/code_checker.py b/CIME/code_checker.py index 4c05f22e477..fdc1eedefd8 100644 --- a/CIME/code_checker.py +++ b/CIME/code_checker.py @@ -45,11 +45,14 @@ def _run_pylint(all_files, interactive): # cmd_options +=",relative-import" # add init-hook option - cmd_options += ' --init-hook=\'import sys; sys.path.extend(("%s","%s","%s","%s"))\'' % ( - os.path.join(cimeroot, "CIME"), - os.path.join(cimeroot, "CIME", "Tools"), - os.path.join(cimeroot, "scripts", "fortran_unit_testing", "python"), - os.path.join(srcroot, "components", "cmeps", "cime_config", "runseq"), + cmd_options += ( + ' --init-hook=\'import sys; sys.path.extend(("%s","%s","%s","%s"))\'' + % ( + os.path.join(cimeroot, "CIME"), + os.path.join(cimeroot, "CIME", "Tools"), + os.path.join(cimeroot, "scripts", "fortran_unit_testing", "python"), + os.path.join(srcroot, "components", "cmeps", "cime_config", "runseq"), + ) ) files = " ".join(all_files) From 75fb95ea722ca2511a418f45877d593cf9ca7834 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 22 Aug 2024 11:43:58 -0700 Subject: [PATCH 39/46] Fixes module name --- CIME/SystemTests/mvk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 9e076cc4c51..4f8a5c12ea8 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -9,7 +9,7 @@ import os import json import logging -from shutils import copytree +from shutil import copytree from CIME import test_status from CIME import utils From b98fae8c1ee148667871759a4ce5e8db7e12f6e1 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 22 Aug 2024 11:55:51 -0700 Subject: [PATCH 40/46] Fixes copytree call --- CIME/SystemTests/mvk.py | 1 - 1 file changed, 1 deletion(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index 4f8a5c12ea8..de2cd6e81fa 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -275,7 +275,6 @@ def update_testlog(self, test_name, case_name, evv_out_dir): copytree( evv_out_dir, os.path.join(htmlroot, "evv", case_name), - preserve_mode=False, ) if urlroot is None: From 1ff7be4fa95b2134d144f26e19d2677bfaaf8246 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 22 Aug 2024 13:50:38 -0700 Subject: [PATCH 41/46] Fixes when test-requirements are installed --- .github/workflows/testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 3b330be45ac..742abb189f4 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -113,13 +113,13 @@ jobs: export CIME_REMOTE=https://github.com/${{ github.event.pull_request.head.repo.full_name || github.repository }} export CIME_BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF##*/}} - pip install -r test-requirements.txt - source /entrypoint.sh # from 'entrypoint.sh', create and activate new environment create_environment ${{ matrix.python-version }} + pip install -r test-requirements.txt + # GitHub runner home is different than container cp -rf /root/.cime /github/home/ From 6e0a12c4117ae7886086176f2ce87fb640e973d5 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 22 Aug 2024 14:55:05 -0700 Subject: [PATCH 42/46] Fixes append_testlog --- CIME/SystemTests/mvk.py | 3 ++- CIME/tests/test_unit_system_tests_mvk.py | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CIME/SystemTests/mvk.py b/CIME/SystemTests/mvk.py index de2cd6e81fa..60f863a2670 100644 --- a/CIME/SystemTests/mvk.py +++ b/CIME/SystemTests/mvk.py @@ -13,6 +13,7 @@ from CIME import test_status from CIME import utils +from CIME.status import append_testlog from CIME.SystemTests.system_tests_common import SystemTestsCommon from CIME.case.case_setup import case_setup from CIME.XML.machines import Machines @@ -303,7 +304,7 @@ def update_testlog(self, test_name, case_name, evv_out_dir): ) ) - utils.append_testlog(comments, self._orig_caseroot) + append_testlog(comments, self._orig_caseroot) def process_evv_output(self, evv_out_dir): with open(os.path.join(evv_out_dir, "index.json")) as evv_f: diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py index 002dabd75f8..f18284c0109 100644 --- a/CIME/tests/test_unit_system_tests_mvk.py +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -208,7 +208,7 @@ def evv_test_config(case, config): assert lines == ["var2 = value2\n"] - @mock.patch("CIME.SystemTests.mvk.utils.append_testlog") + @mock.patch("CIME.SystemTests.mvk.status.append_testlog") @mock.patch("CIME.SystemTests.mvk.Machines") def test_update_testlog(self, machines, append_testlog): with contextlib.ExitStack() as stack: @@ -246,7 +246,7 @@ def test_update_testlog(self, machines, append_testlog): ) @mock.patch("CIME.SystemTests.mvk.utils.get_urlroot") - @mock.patch("CIME.SystemTests.mvk.utils.append_testlog") + @mock.patch("CIME.SystemTests.mvk.status.append_testlog") @mock.patch("CIME.SystemTests.mvk.Machines") def test_update_testlog_urlroot_None(self, machines, append_testlog, get_urlroot): with contextlib.ExitStack() as stack: @@ -287,7 +287,7 @@ def test_update_testlog_urlroot_None(self, machines, append_testlog, get_urlroot ) @mock.patch("CIME.SystemTests.mvk.utils.get_htmlroot") - @mock.patch("CIME.SystemTests.mvk.utils.append_testlog") + @mock.patch("CIME.SystemTests.mvk.status.append_testlog") @mock.patch("CIME.SystemTests.mvk.Machines") def test_update_testlog_htmlroot(self, machines, append_testlog, get_htmlroot): with contextlib.ExitStack() as stack: @@ -462,7 +462,7 @@ def test_build_phase(self, build_indv, case_setup): case_setup.assert_any_call(case, test_mode=False, reset=True) @mock.patch("CIME.SystemTests.mvk.SystemTestsCommon._generate_baseline") - @mock.patch("CIME.SystemTests.mvk.utils.append_testlog") + @mock.patch("CIME.SystemTests.mvk.status.append_testlog") @mock.patch("CIME.SystemTests.mvk.evv") def test__generate_baseline(self, evv, append_testlog, _generate_baseline): with contextlib.ExitStack() as stack: @@ -530,7 +530,7 @@ def test__generate_baseline(self, evv, append_testlog, _generate_baseline): assert files == ["test1.nc", "test2.nc"] - @mock.patch("CIME.SystemTests.mvk.utils.append_testlog") + @mock.patch("CIME.SystemTests.mvk.status.append_testlog") @mock.patch("CIME.SystemTests.mvk.evv") def test__compare_baseline_resubmit(self, evv, append_testlog): with contextlib.ExitStack() as stack: @@ -562,7 +562,7 @@ def test__compare_baseline_resubmit(self, evv, append_testlog): _test_status.set_status.assert_any_call("BASELINE", "PASS") - @mock.patch("CIME.SystemTests.mvk.utils.append_testlog") + @mock.patch("CIME.SystemTests.mvk.status.append_testlog") @mock.patch("CIME.SystemTests.mvk.evv") def test__compare_baseline(self, evv, append_testlog): with contextlib.ExitStack() as stack: From d1d02fc8ff1214df170eaeb7c66b0c0202b46c20 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 22 Aug 2024 15:57:20 -0700 Subject: [PATCH 43/46] Fixes append_testlog import --- CIME/tests/test_unit_system_tests_mvk.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py index f18284c0109..e18768c987c 100644 --- a/CIME/tests/test_unit_system_tests_mvk.py +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -208,7 +208,7 @@ def evv_test_config(case, config): assert lines == ["var2 = value2\n"] - @mock.patch("CIME.SystemTests.mvk.status.append_testlog") + @mock.patch("CIME.SystemTests.mvk.append_testlog") @mock.patch("CIME.SystemTests.mvk.Machines") def test_update_testlog(self, machines, append_testlog): with contextlib.ExitStack() as stack: @@ -246,7 +246,7 @@ def test_update_testlog(self, machines, append_testlog): ) @mock.patch("CIME.SystemTests.mvk.utils.get_urlroot") - @mock.patch("CIME.SystemTests.mvk.status.append_testlog") + @mock.patch("CIME.SystemTests.mvk.append_testlog") @mock.patch("CIME.SystemTests.mvk.Machines") def test_update_testlog_urlroot_None(self, machines, append_testlog, get_urlroot): with contextlib.ExitStack() as stack: @@ -287,7 +287,7 @@ def test_update_testlog_urlroot_None(self, machines, append_testlog, get_urlroot ) @mock.patch("CIME.SystemTests.mvk.utils.get_htmlroot") - @mock.patch("CIME.SystemTests.mvk.status.append_testlog") + @mock.patch("CIME.SystemTests.mvk.append_testlog") @mock.patch("CIME.SystemTests.mvk.Machines") def test_update_testlog_htmlroot(self, machines, append_testlog, get_htmlroot): with contextlib.ExitStack() as stack: @@ -462,7 +462,7 @@ def test_build_phase(self, build_indv, case_setup): case_setup.assert_any_call(case, test_mode=False, reset=True) @mock.patch("CIME.SystemTests.mvk.SystemTestsCommon._generate_baseline") - @mock.patch("CIME.SystemTests.mvk.status.append_testlog") + @mock.patch("CIME.SystemTests.mvk.append_testlog") @mock.patch("CIME.SystemTests.mvk.evv") def test__generate_baseline(self, evv, append_testlog, _generate_baseline): with contextlib.ExitStack() as stack: @@ -530,7 +530,7 @@ def test__generate_baseline(self, evv, append_testlog, _generate_baseline): assert files == ["test1.nc", "test2.nc"] - @mock.patch("CIME.SystemTests.mvk.status.append_testlog") + @mock.patch("CIME.SystemTests.mvk.append_testlog") @mock.patch("CIME.SystemTests.mvk.evv") def test__compare_baseline_resubmit(self, evv, append_testlog): with contextlib.ExitStack() as stack: @@ -562,7 +562,7 @@ def test__compare_baseline_resubmit(self, evv, append_testlog): _test_status.set_status.assert_any_call("BASELINE", "PASS") - @mock.patch("CIME.SystemTests.mvk.status.append_testlog") + @mock.patch("CIME.SystemTests.mvk.append_testlog") @mock.patch("CIME.SystemTests.mvk.evv") def test__compare_baseline(self, evv, append_testlog): with contextlib.ExitStack() as stack: From b54b175f1573cab46fd364571f1f5918e6d28c56 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Thu, 10 Oct 2024 15:57:55 -0700 Subject: [PATCH 44/46] Fixes asserting module path --- CIME/tests/test_unit_system_tests_mvk.py | 28 +++++++++--------------- 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/CIME/tests/test_unit_system_tests_mvk.py b/CIME/tests/test_unit_system_tests_mvk.py index e18768c987c..56eaa39f67e 100644 --- a/CIME/tests/test_unit_system_tests_mvk.py +++ b/CIME/tests/test_unit_system_tests_mvk.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -import re import os import json import unittest import tempfile import contextlib +import sysconfig from pathlib import Path from unittest import mock @@ -186,12 +186,10 @@ def evv_test_config(case, config): module = config["20240515_212034_41b5u2"].pop("module") assert ( - re.search( - r"/opt/conda/lib/python.*/site-packages/evv4esm/extensions/kso.py", - module, - ) - is not None + f'{sysconfig.get_paths()["purelib"]}/evv4esm/extensions/kso.py' + == module ) + assert config == expected_config nml_files = [x for x in os.listdir(temp_dir) if x.startswith("user_nl")] @@ -386,12 +384,9 @@ def test_testmod_simple(self, evv, find_test_mods): module = config["20240515_212034_41b5u2"].pop("module") assert ( - re.search( - r"/opt/conda/lib/python.*/site-packages/evv4esm/extensions/ks.py", - module, - ) - is not None + f'{sysconfig.get_paths()["purelib"]}/evv4esm/extensions/ks.py' == module ) + assert config == expected_config nml_files = [x for x in os.listdir(temp_dir) if x.startswith("user_nl")] @@ -516,7 +511,7 @@ def test__generate_baseline(self, evv, append_testlog, _generate_baseline): files = os.listdir(case_baseline_dir) - assert files == ["test1.nc", "test2.nc"] + assert sorted(files) == sorted(["test1.nc", "test2.nc"]) # reset side_effect case.get_value.side_effect = side_effect @@ -528,7 +523,7 @@ def test__generate_baseline(self, evv, append_testlog, _generate_baseline): files = os.listdir(case_baseline_dir) - assert files == ["test1.nc", "test2.nc"] + assert sorted(files) == sorted(["test1.nc", "test2.nc"]) @mock.patch("CIME.SystemTests.mvk.append_testlog") @mock.patch("CIME.SystemTests.mvk.evv") @@ -603,12 +598,9 @@ def test__compare_baseline(self, evv, append_testlog): module = config["20240515_212034_41b5u2"].pop("module") assert ( - re.search( - r"/opt/conda/lib/python.*/site-packages/evv4esm/extensions/ks.py", - module, - ) - is not None + f'{sysconfig.get_paths()["purelib"]}/evv4esm/extensions/ks.py' == module ) + assert config == expected_config expected_comments = f"""BASELINE PASS for test '20240515_212034_41b5u2'. From a4d7f2a8c6d0092c3f0d57f5f02fc357fc08c11c Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 16 Oct 2024 16:44:42 -0700 Subject: [PATCH 45/46] Removes old critical reference --- doc/source/users_guide/testing.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/users_guide/testing.rst b/doc/source/users_guide/testing.rst index 21e9f88f2be..f0340abc6ca 100644 --- a/doc/source/users_guide/testing.rst +++ b/doc/source/users_guide/testing.rst @@ -395,7 +395,6 @@ For this case the default ``evv_test_config`` and ``generate_namelist`` function component = "eam" # components = [] can be omitted when modifying a single component ninst = 10 - critical = 21 .. _MVK Complex: From f494afdda2bb6403ec383ad4fc0b6889433a8c14 Mon Sep 17 00:00:00 2001 From: Jason Boutte Date: Wed, 16 Oct 2024 18:43:36 -0700 Subject: [PATCH 46/46] Updates example --- doc/source/users_guide/testing.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/source/users_guide/testing.rst b/doc/source/users_guide/testing.rst index f0340abc6ca..f604e93f7d8 100644 --- a/doc/source/users_guide/testing.rst +++ b/doc/source/users_guide/testing.rst @@ -416,6 +416,7 @@ Note; this is a toy example, no scientific usage. import os from CIME.SystemTests.mvk import EVV_LIB_DIR from CIME.namelist import Namelist + from CIME.utils import safe_copy from CIME.utils import run_cmd component "eam" @@ -437,15 +438,18 @@ Note; this is a toy example, no scientific usage. if component in ["eam", "clm"]: with namelist(filename) as nml: if component == "eam": - nml.set_variable_value("eam_specific", f"perturn-{i}") + # arguments group, key, value + nml.set_variable_value("", "eam_specific", f"perturn-{i}") elif component == "clm": if i % 2 == 0: - nml.set_variable_value("clm_specific", "even") + nml.set_variable_value("", "clm_specific", "even") else: - nml.set_variable_value("clm_specific", "odd") + nml.set_variable_value("", "clm_specific", "odd") else: stat, output, err = run_cmd(f"atmchange initial_conditions::perturbation_random_seed = {i*32}") + safe_copy("namelist_scream.xml", f"namelist_scream_{i:04}.xml") + ======================== Test progress and output