From 5ba0c670c965a6a62f56a481717e32723d17d09f Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Wed, 27 Sep 2017 11:25:47 -0600 Subject: [PATCH 1/2] Add option to define test types in query_testlists --- config/config_tests.xml | 25 ++++++------ scripts/lib/CIME/XML/tests.py | 19 +++++++++ scripts/query_testlists | 47 ++++++++++++++++++++--- scripts/tests/scripts_regression_tests.py | 8 +++- 4 files changed, 81 insertions(+), 18 deletions(-) diff --git a/config/config_tests.xml b/config/config_tests.xml index da15cc10dd9..45f0fe95ffb 100644 --- a/config/config_tests.xml +++ b/config/config_tests.xml @@ -16,6 +16,9 @@ The following are the test functionality categories: 11) other component-specific tests NOTES: +- specifying the attribute INFRASTRUCTURE_TEST="TRUE" declares the given + test type to be an infrastructure-only test that is only meant to be + run by scripts_regression_tests. - unless otherwise noted everything is run in one executable directory - suffix: denotes the component history file suffixes that are added as part of the test - IOP test is done along with regular tests - not as a separate test @@ -354,7 +357,7 @@ NODEFAIL Tests restart upon detected node failure. Generates fake failu FALSE - + For testing infra only. Insta-fail build step. 1 ndays @@ -363,7 +366,7 @@ NODEFAIL Tests restart upon detected node failure. Generates fake failu FALSE - + For testing infra only. Insta-fail build step by failing to init. 1 ndays @@ -372,7 +375,7 @@ NODEFAIL Tests restart upon detected node failure. Generates fake failu FALSE - + For testing infra only. Insta-fail run step. 1 ndays @@ -381,7 +384,7 @@ NODEFAIL Tests restart upon detected node failure. Generates fake failu FALSE - + For testing infra only. Insta-fail run step via exception. 1 ndays @@ -390,7 +393,7 @@ NODEFAIL Tests restart upon detected node failure. Generates fake failu FALSE - + For testing infra only. Insta-pass run step. 1 ndays @@ -399,7 +402,7 @@ NODEFAIL Tests restart upon detected node failure. Generates fake failu FALSE - + For testing infra only. Insta-fail memleak step. 1 ndays @@ -408,7 +411,7 @@ NODEFAIL Tests restart upon detected node failure. Generates fake failu FALSE - + For testing infra only. Insta-pass memleak step. 1 ndays @@ -417,7 +420,7 @@ NODEFAIL Tests restart upon detected node failure. Generates fake failu FALSE - + For testing infra only. Produces a canned hist file. 1 ndays @@ -426,7 +429,7 @@ NODEFAIL Tests restart upon detected node failure. Generates fake failu FALSE - + For testing infra only. Simulates internal test diff (non baseline) 1 ndays @@ -435,7 +438,7 @@ NODEFAIL Tests restart upon detected node failure. Generates fake failu FALSE - + For testing infra only. After 5 minutes of sleep, pass run step. 1 ndays @@ -444,7 +447,7 @@ NODEFAIL Tests restart upon detected node failure. Generates fake failu FALSE - + For testing infra only. Tests restart upon detected node failure 1 nsteps diff --git a/scripts/lib/CIME/XML/tests.py b/scripts/lib/CIME/XML/tests.py index 9411e01e609..944f6934687 100644 --- a/scripts/lib/CIME/XML/tests.py +++ b/scripts/lib/CIME/XML/tests.py @@ -32,3 +32,22 @@ def get_test_node(self, testname): node = self.get_node("test",{"NAME":testname}) logger.debug("Found {}".format(node.text)) return node + + def print_values(self, skip_infrastructure_tests=True): + """ + Print each test type and its description. + + If skip_infrastructure_tests is True, then this does not write + information for tests with the attribute + INFRASTRUCTURE_TEST="TRUE". + """ + all_tests = self.get_nodes(nodename="test") + for one_test in all_tests: + if skip_infrastructure_tests: + infrastructure_test = one_test.get("INFRASTRUCTURE_TEST") + if (infrastructure_test is not None and + infrastructure_test.upper() == "TRUE"): + continue + name = one_test.get("NAME") + desc = one_test.find("DESC").text + print("{}: {}".format(name, desc)) diff --git a/scripts/query_testlists b/scripts/query_testlists index 8a962c9573a..d363df3f564 100755 --- a/scripts/query_testlists +++ b/scripts/query_testlists @@ -4,7 +4,7 @@ Script to query xml test lists, displaying all tests in human-readable form. Usage: - ./query_testlists [--show-options] + ./query_testlists [--show-options] [--define-testtypes] Display a list of tests ./query_testlists --count Count tests by category/machine/compiler @@ -16,6 +16,7 @@ Usage: from __future__ import print_function from Tools.standard_script_setup import * from CIME.test_utils import get_tests_from_xml, test_to_string +from CIME.XML.tests import Tests from CIME.utils import expect logger = logging.getLogger(__name__) @@ -46,6 +47,11 @@ def parse_command_line(args, description): "(wallclock time, memory leak tolerance, etc.). " "(Has no effect with --list or --count options.)") + parser.add_argument("--define-testtypes", action="store_true", + help="At the top of the list of tests, define " + "all of the possible test types. " + "(Has no effect with --list or --count options.)") + parser.add_argument("--xml-category", help="Only include tests in this category; default is all categories") @@ -61,14 +67,37 @@ def parse_command_line(args, description): args = CIME.utils.parse_args_and_handle_standard_logging_options(args, parser) - expect(not(args.count and args.list_type), - "Cannot specify both --count and --list arguments.") + _check_argument_compatibility(args) if args.list_type: _process_list_type(args) return args +############################################################################### +def _check_argument_compatibility(args): +############################################################################### + """Ensures there are no incompatible arguments + + If incompatible arguments are found, aborts with a helpful error + message. + """ + + expect(not(args.count and args.list_type), + "Cannot specify both --count and --list arguments.") + + if args.count: + expect(not args.show_options, + "--show-options is incompatible with --count") + expect(not args.define_testtypes, + "--define-testtypes is incompatible with --count") + + if args.list_type: + expect(not args.show_options, + "--show-options is incompatible with --list") + expect(not args.define_testtypes, + "--define-testtypes is incompatible with --list") + ############################################################################### def _process_list_type(args): ############################################################################### @@ -87,7 +116,7 @@ def _process_list_type(args): args.list_type = 'compiler' ############################################################################### -def print_test_data(test_data, show_options): +def print_test_data(test_data, show_options, define_testtypes): ############################################################################### """ Args: @@ -96,6 +125,14 @@ def print_test_data(test_data, show_options): - category: test category """ + if define_testtypes: + print("#"*72) + print("Test types") + print("----------") + test_definitions = Tests() + test_definitions.print_values(skip_infrastructure_tests=True) + print("#"*72) + categories = sorted(set([item['category'] for item in test_data])) max_category_len = max([len(category) for category in categories]) max_test_len = max([len(item['name']) for item in test_data]) @@ -178,7 +215,7 @@ def _main_func(description): elif args.list_type: list_test_data(test_data, args.list_type) else: - print_test_data(test_data, args.show_options) + print_test_data(test_data, args.show_options, args.define_testtypes) if __name__ == "__main__": _main_func(__doc__) diff --git a/scripts/tests/scripts_regression_tests.py b/scripts/tests/scripts_regression_tests.py index 04d15fa2604..a1eadb5a993 100755 --- a/scripts/tests/scripts_regression_tests.py +++ b/scripts/tests/scripts_regression_tests.py @@ -2279,8 +2279,8 @@ def _run_and_assert_query_testlist(self, extra_args=""): files = Files() testlist_drv = files.get_value("TESTS_SPEC_FILE", {"component":"drv"}) - run_cmd_assert_result(self, "%s/query_testlists --xml-testlist %s %s"% - (SCRIPT_DIR, testlist_drv, extra_args)) + run_cmd_assert_result(self, "{}/query_testlists --xml-testlist {} {}".format( + SCRIPT_DIR, testlist_drv, extra_args)) def test_query_testlists_runs(self): """Make sure that query_testlists runs successfully @@ -2291,6 +2291,10 @@ def test_query_testlists_runs(self): """ self._run_and_assert_query_testlist(extra_args="--show-options") + def test_query_testlists_define_testtypes_runs(self): + """Make sure that query_testlists runs successfully with the --define-testtypes argument""" + self._run_and_assert_query_testlist(extra_args="--define-testtypes") + def test_query_testlists_count_runs(self): """Make sure that query_testlists runs successfully with the --count argument""" self._run_and_assert_query_testlist(extra_args="--count") From 948a290d4f7b4b050e10f4601b59e6bb0b502703 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Wed, 27 Sep 2017 16:17:22 -0600 Subject: [PATCH 2/2] Clarify that HOMME tests only work with the ACME version of the atmosphere component --- config/config_tests.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config_tests.xml b/config/config_tests.xml index 45f0fe95ffb..1b4b78b476b 100644 --- a/config/config_tests.xml +++ b/config/config_tests.xml @@ -339,7 +339,7 @@ NODEFAIL Tests restart upon detected node failure. Generates fake failu - Run homme tests. + Run homme tests. Only works with the ACME version of the atmosphere component. 1 ndays 11