diff --git a/config/acme/config_archive.xml b/config/acme/config_archive.xml index fc2d71239b1..6761be060fa 100644 --- a/config/acme/config_archive.xml +++ b/config/acme/config_archive.xml @@ -2,11 +2,12 @@ \.[ri]\..* - \.r[sh]\.* + \.rh\d\.* + \.rs\.* \.h.*.nc$ nhfil - rpointer$NINST_STRING.atm + rpointer.atm$NINST_STRING $CASE.cam$NINST_STRING.r.$DATENAME.nc @@ -15,7 +16,7 @@ \.r.* unset - rpointer$NINST_STRING.atm + rpointer.atm$NINST_STRING $CASE.datm$NINST_STRING.r.$DATENAME.nc,$CASE.datm$NINST_STRING.rs1.$DATENAME.bin @@ -26,7 +27,7 @@ \.h.*.nc$ locfnh - rpointer$NINST_STRING.lnd + rpointer.lnd$NINST_STRING ./$CASE.clm2$NINST_STRING.r.$DATENAME.nc @@ -35,7 +36,7 @@ \.r.* unset - rpointer$NINST_STRING.lnd + rpointer.lnd$NINST_STRING $CASE.dlnd$NINST_STRING.r.$DATENAME.nc,$CASE.dlnd$NINST_STRING.rs1.$DATENAME.bin @@ -45,7 +46,7 @@ \.h.*.nc$ unset - rpointer$NINST_STRING.rof + rpointer.rof$NINST_STRING $CASE.rtm$NINST_STRING.r.$DATENAME.nc @@ -55,7 +56,7 @@ \.h.*.nc$ unset - rpointer$NINST_STRING.rof + rpointer.rof$NINST_STRING $CASE.mosart$NINST_STRING.r.$DATENAME.nc @@ -64,7 +65,7 @@ \.r.* unset - rpointer$NINST_STRING.rof + rpointer.rof$NINST_STRING $CASE.drof$NINST_STRING.r.$DATENAME.nc,$CASE.drof$NINST_STRING.rs1.$DATENAME.bin @@ -74,7 +75,7 @@ \.h.*.nc$ unset - rpointer$NINST_STRING.ice + rpointer.ice$NINST_STRING ./$CASE.cice$NINST_STRING.r.$DATENAME.nc @@ -84,7 +85,7 @@ \.hist.*\.*\.*\.nc unset - rpointer.ice + rpointer.ice$NINST_STRING @@ -92,7 +93,7 @@ \.r.* unset - rpointer$NINST_STRING.ice + rpointer.ice$NINST_STRING $CASE.dice$NINST_STRING.r.$DATENAME.nc,$CASE.dice$NINST_STRING.rs1.$DATENAME.bin @@ -102,11 +103,11 @@ \.h.*.nc$|\.d[dovt]\. unset - rpointer$NINST_STRING.ocn.restart + rpointer.ocn.restart$NINST_STRING ./$CASE.pop$NINST_STRING.r.$DATENAME.nc,RESTART_FMT=nc - rpointer$NINST_STRING.ocn.ovf + rpointer.ocn.ovf$NINST_STRING ./$CASE.pop$NINST_STRING.ro.$DATENAME @@ -116,7 +117,7 @@ \.hist.*\.*\.*\.nc unset - rpointer.ocn + rpointer.ocn$NINST_STRING @@ -124,7 +125,7 @@ \.r.* unset - rpointer$NINST_STRING.ocn + rpointer.ocn$NINST_STRING $CASE.docn$NINST_STRING.r.$DATENAME.nc,$CASE.docn$NINST_STRING.rs1.$DATENAME.bin @@ -135,7 +136,7 @@ \.initial_hist\..*\.nc$ unset - rpointer$NINST_STRING.glc + rpointer.glc$NINST_STRING ./$CASE.cism$NINST_STRING.r.$DATENAME.nc @@ -145,7 +146,7 @@ \.hist.* unset - rpointer.glc + rpointer.glc$NINST_STRING @@ -154,7 +155,7 @@ \.hi.* unset - rpointer$NINST_STRING.wav + rpointer.wav$NINST_STRING unset @@ -163,20 +164,11 @@ \.r.* unset - rpointer$NINST_STRING.wav + rpointer.wav$NINST_STRING $CASE.dwav$NINST_STRING.r.$DATENAME.nc,$CASE.dwav$NINST_STRING.rs1.$DATENAME.bin - - \.r\..* - \.h.*.nc$ - unset - - rpointer.cpl - $CASE.cpl.r.$DATENAME.nc - - inflate_restart.* @@ -194,4 +186,3 @@ - diff --git a/config/cesm/config_archive.xml b/config/cesm/config_archive.xml index 3566c870ef9..903aaf05f64 100644 --- a/config/cesm/config_archive.xml +++ b/config/cesm/config_archive.xml @@ -2,11 +2,12 @@ \.[ri]\..* - \.r[sh]\.* + \.rh\d\.* + \.rs\.* \.h.*.nc$ nhfil - rpointer$NINST_STRING.atm + rpointer.atm$NINST_STRING $CASE.cam$NINST_STRING.r.$DATENAME.nc @@ -15,7 +16,7 @@ \.r.* unset - rpointer$NINST_STRING.atm + rpointer.atm$NINST_STRING $CASE.datm$NINST_STRING.r.$DATENAME.nc,$CASE.datm$NINST_STRING.rs1.$DATENAME.bin @@ -26,7 +27,7 @@ \.h.*.nc$ locfnh - rpointer$NINST_STRING.lnd + rpointer.lnd$NINST_STRING ./$CASE.clm2$NINST_STRING.r.$DATENAME.nc @@ -35,7 +36,7 @@ \.r.* unset - rpointer$NINST_STRING.lnd + rpointer.lnd$NINST_STRING $CASE.dlnd$NINST_STRING.r.$DATENAME.nc,$CASE.dlnd$NINST_STRING.rs1.$DATENAME.bin @@ -45,7 +46,7 @@ \.h.*.nc$ locfnh - rpointer$NINST_STRING.rof + rpointer.rof$NINST_STRING $CASE.rtm$NINST_STRING.r.$DATENAME.nc @@ -55,7 +56,7 @@ \.h.*.nc$ locfnh - rpointer$NINST_STRING.rof + rpointer.rof$NINST_STRING $CASE.mosart$NINST_STRING.r.$DATENAME.nc @@ -66,7 +67,7 @@ \.h.*.nc$ unset - rpointer$NINST_STRING.ice + rpointer.ice$NINST_STRING ./$CASE.cice$NINST_STRING.r.$DATENAME.nc @@ -77,11 +78,11 @@ \.h.*.nc$|\.d[dovt]\. unset - rpointer$NINST_STRING.ocn.restart + rpointer.ocn.restart$NINST_STRING ./$CASE.pop$NINST_STRING.r.$DATENAME.nc,RESTART_FMT=nc - rpointer$NINST_STRING.ocn.ovf + rpointer.ocn.ovf$NINST_STRING ./$CASE.pop$NINST_STRING.ro.$DATENAME @@ -92,7 +93,7 @@ \.initial_hist\..*\.nc$ unset - rpointer$NINST_STRING.glc + rpointer.glc$NINST_STRING ./$CASE.cism$NINST_STRING.r.$DATENAME.nc @@ -102,7 +103,7 @@ \.hi\..*\.nc$ unset - rpointer$NINST_STRING.wav + rpointer.wav$NINST_STRING unset @@ -124,4 +125,3 @@ - diff --git a/config/config_tests.xml b/config/config_tests.xml index 43b49fc6c39..d19cd1394bc 100644 --- a/config/config_tests.xml +++ b/config/config_tests.xml @@ -273,6 +273,19 @@ NODEFAIL Tests restart upon detected node failure. Generates fake failu FALSE + + exact restart from startup, default 4 days + 7 days with restart from interim file + 1 + ndays + 11 + $STOP_N / 2 - 1 + $STOP_OPTION + $STOP_N + $STOP_OPTION + TRUE + FALSE + + exact restart from startup with different PIO methods, default 6 days + 5 days 1 diff --git a/scripts/lib/CIME/SystemTests/irt.py b/scripts/lib/CIME/SystemTests/irt.py new file mode 100644 index 00000000000..879781a57de --- /dev/null +++ b/scripts/lib/CIME/SystemTests/irt.py @@ -0,0 +1,55 @@ +""" +Implementation of the CIME IRT. (Interim Restart Test) +This test the model's restart capability as well as the short term archiver's interim restart capability + +(1) Do a Run of length N with restart at N/2 and DOUT_S_SAVE_INTERIM_RESTART set to TRUE +(2) Archive Run using ST archive tools +(3) Recover first interim restart to the case2 run directory +(4) Start case2 from restart and run to the end of case1 +(5) compare results. + +""" + +from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo +from CIME.XML.standard_module_setup import * +from CIME.case_st_archive import case_st_archive, restore_from_archive +from CIME.utils import ls_sorted_by_mtime + +logger = logging.getLogger(__name__) + +class IRT(SystemTestsCompareTwo): + + def __init__(self, case): + SystemTestsCompareTwo.__init__(self, case, + separate_builds=False, + run_two_suffix = 'restart', + run_one_description = 'initial', + run_two_description = 'restart', + multisubmit = False) + + def _case_one_setup(self): + stop_n = self._case1.get_value("STOP_N") + expect(stop_n >= 3,"STOP_N must be at least 3, STOP_N = {}".format(stop_n)) + + def _case_two_setup(self): + rest_n = self._case1.get_value("REST_N") + stop_n = self._case1.get_value("STOP_N") + stop_new = stop_n - rest_n + expect(stop_new > 0, "ERROR: stop_n value {:d} too short {:d} {:d}".format(stop_new,stop_n,rest_n)) + # hist_n is set to the stop_n value of case1 + self._case.set_value("HIST_N", stop_n) + self._case.set_value("STOP_N", stop_new) + self._case.set_value("CONTINUE_RUN",True) + self._case.set_value("REST_OPTION", "never") + + def _case_one_custom_postrun_action(self): + case_st_archive(self._case) + # Since preview namelist is run before _case_two_prerun_action, we need to do this here. + dout_s_root = self._case1.get_value("DOUT_S_ROOT") + restart_list = ls_sorted_by_mtime(os.path.join(dout_s_root,"rest")) + logger.info("Restart directory list is {}".format(restart_list)) + expect(len(restart_list) >=2,"Expected at least two restart directories") + # Get the older of the two restart directories + restore_from_archive(self._case2, + rest_dir=os.path.abspath( + os.path.join(dout_s_root, "rest", restart_list[0]))) diff --git a/scripts/lib/CIME/SystemTests/system_tests_compare_two.py b/scripts/lib/CIME/SystemTests/system_tests_compare_two.py index 25bfb98ae67..8c312df7d35 100644 --- a/scripts/lib/CIME/SystemTests/system_tests_compare_two.py +++ b/scripts/lib/CIME/SystemTests/system_tests_compare_two.py @@ -14,13 +14,24 @@ (2) _case_two_setup This method will be called to set up case 2, the "test" case -In addition, they MAY require the following method: +In addition, they MAY require the following methods: (1) _common_setup This method will be called to set up both cases. It should contain any setup that's needed in both cases. This is called before _case_one_setup or _case_two_setup. +(2) _case_one_custom_prerun_action(self): + Use this to do arbitrary actions immediately before running case one + +(3) _case_two_custom_prerun_action(self): + Use this to do arbitrary actions immediately before running case two + +(4) _case_one_custom_postrun_action(self): + Use this to do arbitrary actions immediately after running case one + +(5) _case_two_custom_postrun_action(self): + Use this to do arbitrary actions immediately after running case two """ from CIME.XML.standard_module_setup import * @@ -39,7 +50,8 @@ def __init__(self, separate_builds, run_two_suffix = 'test', run_one_description = '', - run_two_description = ''): + run_two_description = '', + multisubmit = False): """ Initialize a SystemTestsCompareTwo object. Individual test cases that inherit from SystemTestsCompareTwo MUST call this __init__ method. @@ -56,6 +68,8 @@ def __init__(self, when starting the first run. Defaults to ''. run_two_description (str, optional): Description printed to log file when starting the second run. Defaults to ''. + multisubmit (bool): Do first and second runs as different submissions. + Designed for tests with RESUBMIT=1 """ SystemTestsCommon.__init__(self, case) @@ -92,6 +106,8 @@ def __init__(self, self._case2 = None self._setup_cases_if_not_yet_done() + + self._multisubmit = multisubmit # ======================================================================== # Methods that MUST be implemented by specific tests that inherit from this # base class @@ -131,6 +147,30 @@ def _common_setup(self): """ pass + def _case_one_custom_prerun_action(self): + """ + Use to do arbitrary actions immediately before running case one + """ + pass + + def _case_two_custom_prerun_action(self): + """ + Use to do arbitrary actions immediately before running case two + """ + pass + + def _case_one_custom_postrun_action(self): + """ + Use to do arbitrary actions immediately after running case one + """ + pass + + def _case_two_custom_postrun_action(self): + """ + Use to do arbitrary actions immediately after running case two + """ + pass + # ======================================================================== # Main public methods # ======================================================================== @@ -160,21 +200,33 @@ def run_phase(self, success_change=False): # pylint: disable=arguments-differ Runs both phases of the two-phase test and compares their results If success_change is True, success requires some files to be different """ - - # First run - logger.info('Doing first run: ' + self._run_one_description) - self._activate_case1() + first_phase = self._case1.get_value("RESUBMIT") == 1 # Only relevant for multi-submit tests run_type = self._case1.get_value("RUN_TYPE") - self.run_indv(suffix = self._run_one_suffix) + # First run + if not self._multisubmit or first_phase: + logger.info('Doing first run: ' + self._run_one_description) + self._activate_case1() + self._case_one_custom_prerun_action() + self.run_indv(suffix = self._run_one_suffix) + self._case_one_custom_postrun_action() # Second run - logger.info('Doing second run: ' + self._run_two_description) - self._activate_case2() - # we need to make sure run2 is properly staged. - if run_type != "startup": - check_case(self._case2, self._caseroot2) - self._force_case2_settings() - self.run_indv(suffix = self._run_two_suffix) + if not self._multisubmit or not first_phase: + # Subtle issue: case1 is already in a writeable state since it tends to be opened + # with a with statement in all the API entrances in CIME. case2 was created via clone, + # not a with statement, so it's not in a writeable state, so we need to use a with + # statement here to put it in a writeable state. + with self._case2: + logger.info('Doing second run: ' + self._run_two_description) + self._activate_case2() + # we need to make sure run2 is properly staged. + if run_type != "startup": + check_case(self._case2, self._caseroot2) + self._force_case2_settings() + + self._case_two_custom_prerun_action() + self.run_indv(suffix = self._run_two_suffix) + self._case_two_custom_postrun_action() # Compare results # Case1 is the "main" case, and we need to do the comparisons from there @@ -236,7 +288,7 @@ def _setup_cases_if_not_yet_done(self): self._case1.get_value("CASE"), "case2") self._case2 = self._case1.create_clone( newcase = self._caseroot2, - keepexe = self._separate_builds==False, + keepexe = not self._separate_builds, cime_output_root = case2_output_root) self._setup_cases() except: diff --git a/scripts/lib/CIME/case_st_archive.py b/scripts/lib/CIME/case_st_archive.py index 50e01c152fa..3f469655bbb 100644 --- a/scripts/lib/CIME/case_st_archive.py +++ b/scripts/lib/CIME/case_st_archive.py @@ -7,7 +7,7 @@ from CIME.XML.standard_module_setup import * from CIME.case_submit import submit from CIME.XML.env_archive import EnvArchive -from CIME.utils import run_and_log_case_status +from CIME.utils import run_and_log_case_status, ls_sorted_by_mtime from os.path import isdir, join import datetime @@ -90,13 +90,16 @@ def _archive_rpointer_files(case, archive, archive_entry, archive_restdir, # loop through the possible rpointer files and contents for rpointer_file, rpointer_content in rpointer_items: - + temp_rpointer_file = rpointer_file + temp_rpointer_content = rpointer_content # put in a temporary setting for ninst_strings if they are empty # in order to have just one loop over ninst_strings below if rpointer_content is not 'unset': if not ninst_strings: ninst_strings = ["empty"] for ninst_string in ninst_strings: + rpointer_file = temp_rpointer_file + rpointer_content = temp_rpointer_content if ninst_string == 'empty': ninst_string = "" for key, value in [('$CASE', casename), @@ -384,16 +387,21 @@ def _archive_process(case, archive, last_date, archive_incomplete_logs, copy_onl archive_file_fn) ############################################################################### -def restore_from_archive(case): +def restore_from_archive(case, rest_dir=None): ############################################################################### """ - Take most recent archived restart files and load them into current case. + Take archived restart files and load them into current case. Use rest_dir if provided otherwise use most recent """ dout_sr = case.get_value("DOUT_S_ROOT") rundir = case.get_value("RUNDIR") - most_recent_rest = run_cmd_no_fail("ls -1dt {}/rest/* | head -1".format(dout_sr)) + if rest_dir is not None: + if not os.path.isabs(rest_dir): + rest_dir = os.path.join(dout_sr, "rest", rest_dir) + else: + rest_dir = ls_sorted_by_mtime(os.path.join(dout_sr, "rest"))[-1] - for item in glob.glob("{}/*".format(most_recent_rest)): + logger.info("Restoring from {} to {}".format(rest_dir, rundir)) + for item in glob.glob("{}/*".format(rest_dir)): base = os.path.basename(item) dst = os.path.join(rundir, base) if os.path.exists(dst): diff --git a/scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two.py b/scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two.py index ae4e837208f..0134059b782 100644 --- a/scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two.py +++ b/scripts/lib/CIME/tests/SystemTests/test_system_tests_compare_two.py @@ -396,3 +396,6 @@ def test_run2_fails(self): # Verify self.assertEqual(test_status.TEST_FAIL_STATUS, mytest._test_status.get_status(test_status.RUN_PHASE)) + +if __name__ == "__main__": + unittest.main(verbosity=2, catchbreak=True) diff --git a/scripts/lib/CIME/tests/case_fake.py b/scripts/lib/CIME/tests/case_fake.py index bba5aeedb99..da9effa7110 100644 --- a/scripts/lib/CIME/tests/case_fake.py +++ b/scripts/lib/CIME/tests/case_fake.py @@ -102,3 +102,9 @@ def set_rundir(self): inside CASEROOT) """ self.set_value('RUNDIR', os.path.join(self.get_value('CASEROOT'), 'run')) + + def __enter__(self): + pass + + def __exit__(self, *_): + pass diff --git a/scripts/lib/CIME/utils.py b/scripts/lib/CIME/utils.py index ac2b86cddc7..93c2c5cd915 100644 --- a/scripts/lib/CIME/utils.py +++ b/scripts/lib/CIME/utils.py @@ -1224,6 +1224,11 @@ def _get_most_recent_lid_impl(files): return sorted(results) +def ls_sorted_by_mtime(path): + ''' return list of path sorted by timestamp oldest first''' + mtime = lambda f: os.stat(os.path.join(path, f)).st_mtime + return list(sorted(os.listdir(path), key=mtime)) + def get_lids(case): model = case.get_value("MODEL") logdir = case.get_value("LOGDIR") diff --git a/scripts/lib/update_acme_tests.py b/scripts/lib/update_acme_tests.py index c134a2a1853..63f7706a7ec 100644 --- a/scripts/lib/update_acme_tests.py +++ b/scripts/lib/update_acme_tests.py @@ -43,7 +43,7 @@ "ERIO.f09_g16.X", "SEQ_Ln9.f19_g16_rx1.A", "ERS.ne30_g16_rx1.A", - "ERS_N2.f19_g16_rx1.A", + "IRT_N2.f19_g16_rx1.A", "ERR.f45_g37_rx1.A", "ERP.f45_g37_rx1.A", "SMS_D_Ln9.f19_g16_rx1.A", diff --git a/src/components/data_comps/datm/cime_config/config_archive.xml b/src/components/data_comps/datm/cime_config/config_archive.xml index 4b34208ddde..93f1f049db1 100644 --- a/src/components/data_comps/datm/cime_config/config_archive.xml +++ b/src/components/data_comps/datm/cime_config/config_archive.xml @@ -3,7 +3,7 @@ \.r.* unset - rpointer$NINST_STRING.atm + rpointer.atm$NINST_STRING $CASE.datm$NINST_STRING.r.$DATENAME.nc,$CASE.datm$NINST_STRING.rs1.$DATENAME.bin diff --git a/src/components/data_comps/dice/cime_config/config_archive.xml b/src/components/data_comps/dice/cime_config/config_archive.xml index 330deab5c7f..06dd1d802ba 100644 --- a/src/components/data_comps/dice/cime_config/config_archive.xml +++ b/src/components/data_comps/dice/cime_config/config_archive.xml @@ -3,7 +3,7 @@ \.r.* unset - rpointer$NINST_STRING.ice + rpointer.ice$NINST_STRING $CASE.dice$NINST_STRING.r.$DATENAME.nc,$CASE.dice$NINST_STRING.rs1.$DATENAME.bin diff --git a/src/components/data_comps/dlnd/cime_config/config_archive.xml b/src/components/data_comps/dlnd/cime_config/config_archive.xml index 1ccea9bdf31..bc2b683c7f9 100644 --- a/src/components/data_comps/dlnd/cime_config/config_archive.xml +++ b/src/components/data_comps/dlnd/cime_config/config_archive.xml @@ -3,7 +3,7 @@ \.r.* unset - rpointer$NINST_STRING.lnd + rpointer.lnd$NINST_STRING $CASE.dlnd$NINST_STRING.r.$DATENAME.nc,$CASE.dlnd$NINST_STRING.rs1.$DATENAME.bin diff --git a/src/components/data_comps/docn/cime_config/config_archive.xml b/src/components/data_comps/docn/cime_config/config_archive.xml index 69b7e07d38b..5d156b24164 100644 --- a/src/components/data_comps/docn/cime_config/config_archive.xml +++ b/src/components/data_comps/docn/cime_config/config_archive.xml @@ -3,7 +3,7 @@ \.r.* unset - rpointer$NINST_STRING.ocn + rpointer.ocn$NINST_STRING $CASE.docn$NINST_STRING.r.$DATENAME.nc,$CASE.docn$NINST_STRING.rs1.$DATENAME.bin diff --git a/src/components/data_comps/drof/cime_config/config_archive.xml b/src/components/data_comps/drof/cime_config/config_archive.xml index 804933fe8e1..9ab454c0f64 100644 --- a/src/components/data_comps/drof/cime_config/config_archive.xml +++ b/src/components/data_comps/drof/cime_config/config_archive.xml @@ -3,7 +3,7 @@ \.r.* unset - rpointer$NINST_STRING.rof + rpointer.rof$NINST_STRING $CASE.drof$NINST_STRING.r.$DATENAME.nc,$CASE.drof$NINST_STRING.rs1.$DATENAME.bin diff --git a/src/components/data_comps/dwav/cime_config/config_archive.xml b/src/components/data_comps/dwav/cime_config/config_archive.xml index 1c0dfdbe84b..5241c1e071a 100644 --- a/src/components/data_comps/dwav/cime_config/config_archive.xml +++ b/src/components/data_comps/dwav/cime_config/config_archive.xml @@ -3,7 +3,7 @@ \.r.* unset - rpointer$NINST_STRING.wav + rpointer.wav$NINST_STRING $CASE.dwav$NINST_STRING.r.$DATENAME.nc,$CASE.dwav$NINST_STRING.rs1.$DATENAME.bin diff --git a/src/drivers/mct/cime_config/config_archive.xml b/src/drivers/mct/cime_config/config_archive.xml index 168d40de2d6..5f59d56a82d 100644 --- a/src/drivers/mct/cime_config/config_archive.xml +++ b/src/drivers/mct/cime_config/config_archive.xml @@ -4,7 +4,7 @@ \.h.*.nc$ unset - rpointer.cpl + rpointer.drv $CASE.cpl.r.$DATENAME.nc