Merge branch 'jgfouca/branch-for-to-acme-11-13-2017-pr' into master (PR

#1896) Update cime to ESMCI 5.4.0-alpha.09 Feature list: Add optional COSTPES_PER_NODE field Bugfix for checking locked files, BUILD_COMPLETE should not cause check to fail [BFB] * jgfouca/branch-for-to-acme-11-13-2017-pr: fix pylint issue rename locked files needs to handle BUILD_COMPLETE better Switch melvin to use officially supported pfunit add optional COST_PER_NODE Remove hardcoded email options for lc_slurm, cascade, constance, sooty Fix cobalt indent Remove hardcoded mail option for PBS Revert "Revert the change to edison as this is now in E3SM-Project/E3SM#1888" Revert the change to edison as this is now in E3SM-Project/E3SM#1888 Undo/remove things that keep causing problems in the ESMCI merge. Update Slurm task-to-core affinities Efficiency updates for coupler bypass
ESMCI · Nov 14, 2017 · e96e7ce · e96e7ce
2 parents 159831a + 9442c93
commit e96e7ce
Show file tree

Hide file tree

Showing 8 changed files with 30 additions and 8 deletions.
diff --git a/config/acme/machines/config_compilers.xml b/config/acme/machines/config_compilers.xml
@@ -716,7 +716,7 @@ for mct, etc.
   <CONFIG_ARGS> --host=Linux </CONFIG_ARGS>
   <NETCDF_PATH>$(NETCDFROOT)</NETCDF_PATH>
   <PNETCDF_PATH>$(PNETCDFROOT)</PNETCDF_PATH>
-  <PFUNIT_PATH MPILIB="mpi-serial" compile_threaded="false">/sems-data-store/ACME/pfunit/pfunit-3.2.8</PFUNIT_PATH>
+  <PFUNIT_PATH MPILIB="mpi-serial" compile_threaded="false">$SEMS_PFUNIT_ROOT</PFUNIT_PATH>
   <ADD_SLIBS> $(shell $(NETCDF_PATH)/bin/nf-config --flibs) -lblas -llapack</ADD_SLIBS>
   <CXX_LIBS>-lstdc++ -lmpi_cxx</CXX_LIBS>
   <ALBANY_PATH>/projects/install/rhel6-x86_64/ACME/AlbanyTrilinos/Albany/build/install</ALBANY_PATH>

diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml
@@ -590,6 +590,7 @@
       <modules>
         <command name="purge"/>
         <command name="load">sems-env</command>
+        <command name="load">acme-env</command>
         <command name="load">sems-git</command>
         <command name="load">sems-python/2.7.9</command>
         <command name="load">sems-cmake/2.8.12</command>
@@ -602,6 +603,7 @@
       </modules>
       <modules mpilib="mpi-serial">
         <command name="load">sems-netcdf/4.4.1/exo</command>
+        <command name="load">acme-pfunit/3.2.8/base</command>
       </modules>
       <modules mpilib="!mpi-serial">
         <command name="load">sems-openmpi/1.8.7</command>

diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml
@@ -572,6 +572,7 @@
     <SUPPORTED_BY>cseg</SUPPORTED_BY>
     <MAX_TASKS_PER_NODE>256</MAX_TASKS_PER_NODE>
     <MAX_MPITASKS_PER_NODE>64</MAX_MPITASKS_PER_NODE>
+    <COSTPES_PER_NODE>68</COSTPES_PER_NODE>
     <mpirun mpilib="default">
       <executable>srun</executable>
       <arguments>
@@ -1843,6 +1844,7 @@
     <SUPPORTED_BY>cseg</SUPPORTED_BY>
     <MAX_TASKS_PER_NODE>30</MAX_TASKS_PER_NODE>
     <MAX_MPITASKS_PER_NODE>15</MAX_MPITASKS_PER_NODE>
+    <COSTPES_PER_NODE>16</COSTPES_PER_NODE>
     <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
     <mpirun mpilib="default" unit_testing="true">
       <!-- For unit testing, avoid all of the extra stuff that we use before and

diff --git a/config/xml_schemas/config_machines.xsd b/config/xml_schemas/config_machines.xsd
@@ -40,6 +40,7 @@
   <xs:element name="GMAKE_J" type="xs:integer"/>
   <xs:element name="MAX_TASKS_PER_NODE" type="xs:integer"/>
   <xs:element name="MAX_MPITASKS_PER_NODE" type="xs:integer"/>
+  <xs:element name="COSTPES_PER_NODE" type="xs:integer"/>
   <xs:element name="PROJECT_REQUIRED" type="xs:NCName"/>
   <xs:element name="GMAKE" type="xs:string"/>
   <xs:element name="TESTS" type="xs:string"/>
@@ -114,6 +115,8 @@
 	<!-- MAX_MPITASKS_PER_NODE: number of physical PES per shared node on
 	     this machine, in practice the MPI tasks per node will not exceed this value -->
         <xs:element ref="MAX_MPITASKS_PER_NODE" minOccurs="1" maxOccurs="1"/>
+	<!-- Optional cost factor per node unit -->
+        <xs:element ref="COSTPES_PER_NODE" minOccurs="0" maxOccurs="1"/>
 	<!-- PROJECT_REQUIRED: Does this machine require a project to be specified to
 	     the batch system?  See PROJECT above -->
         <xs:element ref="PROJECT_REQUIRED" minOccurs="0" maxOccurs="1"/>

diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py
@@ -1334,8 +1334,6 @@ def set_file(self, xmlfile):
         gfile = GenericXML(infile=xmlfile)
         ftype = gfile.get_id()
 
-        self.flush(flushall=True)
-
         logger.warning("setting case file to {}".format(xmlfile))
         components = self.get_value("COMP_CLASSES")
         new_env_file = None

diff --git a/scripts/lib/CIME/check_lockedfiles.py b/scripts/lib/CIME/check_lockedfiles.py
@@ -111,11 +111,14 @@ def check_lockedfiles(case):
                 continue
             diffs = f1obj.compare_xml(f2obj)
             if diffs:
+
                 logging.warning("File {} has been modified".format(lfile))
+                toggle_build_status = False
                 for key in diffs.keys():
-                    print("  found difference in {} : case {} locked {}"
-                          .format(key, repr(diffs[key][0]), repr(diffs[key][1])))
-
+                    if key != "BUILD_COMPLETE":
+                        print("  found difference in {} : case {} locked {}"
+                              .format(key, repr(diffs[key][0]), repr(diffs[key][1])))
+                        toggle_build_status = True
                 if objname == "env_mach_pes":
                     expect(False, "Invoke case.setup --reset ")
                 elif objname == "env_case":
@@ -128,8 +131,9 @@ def check_lockedfiles(case):
                         case.set_value("BUILD_STATUS", 2)
                         logging.critical("Changing PIO_VERSION requires running "
                                          "case.build --clean-all and rebuilding")
-                    else:
+                    elif toggle_build_status:
                         case.set_value("BUILD_STATUS", 1)
+                    f2obj.set_value("BUILD_COMPLETE", False)
                 elif objname == "env_batch":
                     expect(False, "Batch configuration has changed, please run case.setup --reset")
                 else:

diff --git a/scripts/lib/CIME/get_timing.py b/scripts/lib/CIME/get_timing.py
@@ -138,13 +138,18 @@ def _getTiming(self, inst=0):
         run_type = self.case.get_value("RUN_TYPE")
         stop_option = self.case.get_value("STOP_OPTION")
         stop_n = self.case.get_value("STOP_N")
+
         cost_pes = self.case.get_value("COST_PES")
+        costpes_per_node = self.case.get_value("COSTPES_PER_NODE")
+
         totalpes = self.case.get_value("TOTALPES")
         max_mpitasks_per_node = self.case.get_value("MAX_MPITASKS_PER_NODE")
         smt_factor = max(1,int(self.case.get_value("MAX_TASKS_PER_NODE") / max_mpitasks_per_node))
 
         if cost_pes > 0:
             pecost = cost_pes
+        elif costpes_per_node:
+            pecost = self.case.num_nodes * costpes_per_node
         else:
             pecost = totalpes
 
@@ -309,7 +314,7 @@ def _getTiming(self, inst=0):
 
         self.write("\n")
         self.write("  total pes active           : {} \n".format(totalpes*maxthrds*smt_factor ))
-        self.write("  pes per node               : {} \n".format(max_mpitasks_per_node))
+        self.write("  mpi tasks per node               : {} \n".format(max_mpitasks_per_node))
         self.write("  pe count for cost estimate : {} \n".format(pecost))
         self.write("\n")
 

diff --git a/src/drivers/mct/cime_config/config_component.xml b/src/drivers/mct/cime_config/config_component.xml
@@ -2063,6 +2063,14 @@
     <default_value>0</default_value>
     <group>mach_pes_last</group>
     <file>env_mach_pes.xml</file>
+    <desc>pes or cores per node for mpitasks </desc>
+  </entry>
+
+  <entry id="COSTPES_PER_NODE">
+    <type>integer</type>
+    <default_value>$MAX_MPITASKS_PER_NODE</default_value>
+    <group>mach_pes_last</group>
+    <file>env_mach_pes.xml</file>
     <desc>pes or cores per node for accounting purposes </desc>
   </entry>