Merge pull request #1667 from billsacks/user_compset_fixes

User compset fixes
Main change is to have user-defined compsets still pick up a usermods
directory under [primary_component]/cime_config/usermods_dirs/. Along
with this, I have refactored where some other things are set that depend
on primary component. A key aim here was to have various things set in
the same place regardless of whether we're using a compset with or
without an alias. I find this makes it easier to understand the code,
and less likely to introduce discrepancies between user-defined and
alias-defined compsets in the future.

Test suite: scripts_regression_tests on yellowstone
Also, manual testing:

Compset with an alias: picks up PE settings and a usermods directory
User compset picks up PE settings
User compset picks up usermods directory, as in:
./create_newcase --case no_alias_0608d --compset 1850_DATM%GSWP3v1_CLM50%BGC-CROP_SICE_SOCN_RTM_CISM2%NOEVOLVE_SWAV --res f09_g16 --user-compset --user-mods-dir clm/clm5_OutputCrop
Test baseline: N/A
Test namelist changes: none
Test status: bit for bit
Fixes #1658
Fixes #1664
Fixes #1665

User interface changes?: none

Code review:jedwards

scripts/create_newcase

@@ -80,10 +80,9 @@ OR
                         "If the compset name is found as a supported compset, then it will be treated as such.")
 
     parser.add_argument("--pesfile",
-                        help="Only used and required for --user-compset argument."
-                        "Full pathname of the pes specification file"
-                        "This argument is required if --user-compset is True"
-                        "The file can follow either the config_pes.xml or the env_mach_pes.xml format")
+                        help="Full pathname of an optional pes specification "
+                        "file. The file can follow either the config_pes.xml or "
+                        "the env_mach_pes.xml format.")
 
     parser.add_argument("--user-grid", action="store_true",
                         help="If set, then the -grid argument is treated as a user specified grid."

scripts/lib/CIME/case.py

@@ -410,22 +410,24 @@ def clean_up_lookups(self, allow_undefined=False):
             if result is not None:
                 del self.lookups[key]
 
-    def _set_compset_and_pesfile(self, compset_name, files, user_compset=False, pesfile=None):
+    def _set_compset(self, compset_name, files, user_compset=False):
         """
         Loop through all the compset files and find the compset
         specifation file that matches either the input 'compset_name'.
         Note that the input compset name (i.e. compset_name) can be
-        either a longname or an alias.  This will also set the
-        compsets and pes specfication files.
+        either a longname or an alias. This will set various compset-related
+        info.
+
+        Returns a tuple: (compset_alias, science_support, component_defining_compset)
+        (For a user-defined compset - i.e., a compset without an alias - these
+        return values will be None, [], None.)
         """
         science_support = []
         compset_alias = None
+        component_defining_compset = None
         components = files.get_components("COMPSETS_SPEC_FILE")
         logger.debug(" Possible components for COMPSETS_SPEC_FILE are {}".format(components))
 
-        if pesfile is not None:
-            self._pesfile = pesfile
-
         # Loop through all of the files listed in COMPSETS_SPEC_FILE and find the file
         # that has a match for either the alias or the longname in that order
         for component in components:
@@ -438,28 +440,17 @@ def _set_compset_and_pesfile(self, compset_name, files, user_compset=False, pesf
                 compsets = Compsets(compsets_filename)
                 match, compset_alias, science_support = compsets.get_compset_match(name=compset_name)
                 if match is not None:
-                    if self._pesfile is None:
-                        self._pesfile = files.get_value("PES_SPEC_FILE"     , {"component":component})
-                        self.set_lookup_value("PES_SPEC_FILE"      ,
-                                              files.get_value("PES_SPEC_FILE"     , {"component":component}, resolved=False))
                     self._compsetsfile = compsets_filename
                     self._compsetname = match
-                    tests_filename    = files.get_value("TESTS_SPEC_FILE"   , {"component":component}, resolved=False)
-                    tests_mods_dir    = files.get_value("TESTS_MODS_DIR"    , {"component":component}, resolved=False)
-                    user_mods_dir     = files.get_value("USER_MODS_DIR"     , {"component":component}, resolved=False)
                     self.set_lookup_value("COMPSETS_SPEC_FILE" ,
                                    files.get_value("COMPSETS_SPEC_FILE", {"component":component}, resolved=False))
-                    self._primary_component = component
-                    self.set_lookup_value("TESTS_SPEC_FILE"    , tests_filename)
-                    self.set_lookup_value("TESTS_MODS_DIR"     , tests_mods_dir)
-                    self.set_lookup_value("USER_MODS_DIR"      , user_mods_dir)
+                    component_defining_compset = component
                     logger.info("Compset longname is {}".format(match))
                     logger.info("Compset specification file is {}".format(compsets_filename))
-                    logger.info("Pes     specification file is {}".format(self._pesfile))
                     if user_compset is True:
                         logger.info("Found a compset match for longname {} in alias {}".format(compset_name, compset_alias))
 
-                    return compset_alias, science_support
+                    return compset_alias, science_support, component_defining_compset
 
         if user_compset is True:
             self._compsetname = compset_name
@@ -468,7 +459,7 @@ def _set_compset_and_pesfile(self, compset_name, files, user_compset=False, pesf
                    "Could not find a compset match for either alias or longname in {}\n".format(compset_name)
                    + "You may need the --user-compset argument.")
 
-        return None, science_support
+        return None, science_support, None
 
     def _find_primary_component(self):
         """
@@ -522,6 +513,33 @@ def _find_primary_component(self):
         return primary_component
 
 
+    def _set_info_from_primary_component(self, files, pesfile=None):
+        """
+        Sets file and directory paths that depend on the primary component of
+        this compset.
+
+        Assumes that self._primary_component has already been set.
+        """
+
+        component = self._primary_component
+
+        if pesfile is None:
+            self._pesfile = files.get_value("PES_SPEC_FILE", {"component":component})
+            pesfile_unresolved = files.get_value("PES_SPEC_FILE", {"component":component}, resolved=False)
+            logger.info("Pes     specification file is {}".format(self._pesfile))
+        else:
+            self._pesfile = pesfile
+            pesfile_unresolved = pesfile
+        self.set_lookup_value("PES_SPEC_FILE", pesfile_unresolved)
+
+        tests_filename = files.get_value("TESTS_SPEC_FILE", {"component":component}, resolved=False)
+        tests_mods_dir = files.get_value("TESTS_MODS_DIR" , {"component":component}, resolved=False)
+        user_mods_dir  = files.get_value("USER_MODS_DIR"  , {"component":component}, resolved=False)
+        self.set_lookup_value("TESTS_SPEC_FILE", tests_filename)
+        self.set_lookup_value("TESTS_MODS_DIR" , tests_mods_dir)
+        self.set_lookup_value("USER_MODS_DIR"  , user_mods_dir)
+
+
     def get_compset_components(self):
         #If are doing a create_clone then, self._compsetname is not set yet
         components = []
@@ -605,13 +623,6 @@ def _get_component_config_data(self, files):
             for env_file in self._env_entryid_files:
                 env_file.add_elements_by_group(compobj, attributes=attlist)
 
-        if self._primary_component is None:
-            self._primary_component = self._find_primary_component()
-            if self._pesfile is None:
-                self._pesfile = files.get_value("PES_SPEC_FILE"     , {"component":self._primary_component})
-            logger.info("Pes specification file is {}".format(self._pesfile))
-            self.set_lookup_value("PES_SPEC_FILE", self._pesfile)
-
         self.clean_up_lookups()
 
     def _setup_mach_pes(self, pecount, ninst, machine_name, mpilib):
@@ -732,7 +743,8 @@ def configure(self, compset_name, grid_name, machine_name=None,
         # compset, pesfile, and compset components
         #--------------------------------------------
         files = Files()
-        compset_alias, science_support = self._set_compset_and_pesfile(compset_name, files, user_compset=user_compset, pesfile=pesfile)
+        compset_alias, science_support, component_defining_compset = self._set_compset(
+            compset_name, files, user_compset=user_compset)
 
         self._components = self.get_compset_components()
         #--------------------------------------------
@@ -754,6 +766,14 @@ def configure(self, compset_name, grid_name, machine_name=None,
         #--------------------------------------------
         self._get_component_config_data(files)
 
+        if component_defining_compset is None:
+            # This needs to be called after self.set_comp_classes, which is called
+            # from self._get_component_config_data
+            self._primary_component = self._find_primary_component()
+        else:
+            self._primary_component = component_defining_compset
+        self._set_info_from_primary_component(files, pesfile=pesfile)
+
         self.get_compset_var_settings()
 
         self.clean_up_lookups()
@@ -1268,18 +1288,25 @@ def get_build_threaded(self):
     def _check_testlists(self, compset_alias, grid_name, files):
         """
         CESM only: check the testlist file for tests of this compset grid combination
+
+        compset_alias should be None for a user-defined compset (i.e., a compset
+        without an alias)
         """
         if "TESTS_SPEC_FILE" in self.lookups:
             tests_spec_file = self.get_resolved_value(self.lookups["TESTS_SPEC_FILE"])
         else:
             tests_spec_file = self.get_value("TESTS_SPEC_FILE")
 
-        tests = Testlist(tests_spec_file, files)
-        testlist = tests.get_tests(compset=compset_alias, grid=grid_name)
         testcnt = 0
-        for test in testlist:
-            if test["category"] == "prealpha" or test["category"] == "prebeta" or "aux_" in test["category"]:
-                testcnt += 1
+        if compset_alias is not None:
+            # It's important that we not try to find matching tests if
+            # compset_alias is None, since compset=None tells get_tests to find
+            # tests of all compsets!
+            tests = Testlist(tests_spec_file, files)
+            testlist = tests.get_tests(compset=compset_alias, grid=grid_name)
+            for test in testlist:
+                if test["category"] == "prealpha" or test["category"] == "prebeta" or "aux_" in test["category"]:
+                    testcnt += 1
         if testcnt > 0:
             logger.info("\nThis compset and grid combination is not scientifically supported, however it is used in {:d} tests.\n".format(testcnt))
         else: