diff --git a/config/acme/allactive/config_pesall.xml b/config/acme/allactive/config_pesall.xml index eabee41dbf5..ba70b4a52e2 100644 --- a/config/acme/allactive/config_pesall.xml +++ b/config/acme/allactive/config_pesall.xml @@ -4181,7 +4181,7 @@ ne30_ne30 grid on 45 nodes 32 ppn pure-MPI - 32 + 32 32 1350 @@ -6317,7 +6317,7 @@ -compset A_WCYCL* -res ne30_oEC* on 32 nodes pure-MPI - 32 + 32 32 675 @@ -6352,7 +6352,7 @@ -compset A_WCYCL* -res ne30_oEC* on 59 nodes pure-MPI - 32 + 32 32 1350 @@ -6387,7 +6387,7 @@ -compset A_WCYCL* -res ne30_oEC* on 115 nodes pure-MPI - 32 + 32 32 2700 diff --git a/config/acme/machines/config_batch.xml b/config/acme/machines/config_batch.xml index f34c1408151..f73bd1893f3 100644 --- a/config/acme/machines/config_batch.xml +++ b/config/acme/machines/config_batch.xml @@ -47,7 +47,7 @@ - + @@ -65,7 +65,7 @@ - + diff --git a/config/acme/machines/config_machines.xml b/config/acme/machines/config_machines.xml index 5af4f9ed271..4435dd73573 100644 --- a/config/acme/machines/config_machines.xml +++ b/config/acme/machines/config_machines.xml @@ -72,7 +72,7 @@ acme 8 24 - 24 + 24 acme TRUE /project/projectdirs/acme/inputdata @@ -196,7 +196,7 @@ acme 8 32 - 32 + 32 acme TRUE -D PIO_BUILD_TIMING:BOOL=ON @@ -333,7 +333,7 @@ acme 8 256 - 64 + 64 acme TRUE -D PIO_BUILD_TIMING:BOOL=ON @@ -469,7 +469,7 @@ 4 4 - 2 + 2 @@ -493,7 +493,7 @@ 4 4 - 2 + 2 mpirun @@ -524,7 +524,7 @@ 32 64 - 64 + 64 none mpirun @@ -591,7 +591,7 @@ 32 64 - 64 + 64 none mpirun @@ -657,7 +657,7 @@ make 32 64 - 64 + 64 none mpirun @@ -719,7 +719,7 @@ jgfouca at sandia dot gov 8 16 - 16 + 16 1 TRUE fy150001 @@ -794,7 +794,7 @@ jgfouca at sandia dot gov 8 16 - 16 + 16 1 TRUE fy150001 @@ -869,7 +869,7 @@ jgfouca at sandia dot gov 8 36 - 36 + 36 1 TRUE fy150001 @@ -935,7 +935,7 @@ acme 4 16 - 16 + 16 TRUE ACME @@ -1047,7 +1047,7 @@ acme 8 36 - 36 + 36 FALSE ACME @@ -1153,7 +1153,7 @@ acme 8 36 - 36 + 36 acme TRUE -D PIO_BUILD_TIMING:BOOL=ON @@ -1226,7 +1226,7 @@ cobalt jayesh -at- mcs.anl.gov 8 - 4 + 4 64 TRUE ClimateEnergy_2 @@ -1235,7 +1235,7 @@ /usr/bin/runjob --label short - --ranks-per-node $PES_PER_NODE + --ranks-per-node $MAX_MPITASKS_PER_NODE --np $TOTALPES --block $COBALT_PARTNAME $LOCARGS $ENV{BGQ_SMP_VARS} @@ -1280,7 +1280,7 @@ bogenschutz1 -at- llnl.gov 8 16 - 16 + 16 lc_slurm @@ -1336,7 +1336,7 @@ bogenschutz1 -at- llnl.gov 8 16 - 16 + 16 lc_slurm @@ -1397,7 +1397,7 @@ donahue5 -at- llnl.gov 8 36 - 36 + 36 lc_slurm @@ -1451,7 +1451,7 @@ cobalt mickelso -at- mcs.anl.gov 8 - 4 + 4 64 TRUE ClimateEnergy_2 @@ -1460,7 +1460,7 @@ /usr/bin/runjob --label short - --ranks-per-node $PES_PER_NODE + --ranks-per-node $MAX_MPITASKS_PER_NODE --np $TOTALPES --block $COBALT_PARTNAME $LOCARGS $ENV{BGQ_SMP_VARS} @@ -1505,7 +1505,7 @@ acme 8 64 - 64 + 64 OceanClimate TRUE -D PIO_BUILD_TIMING:BOOL=ON @@ -1513,7 +1513,7 @@ aprun -n $TOTALPES - -N $PES_PER_NODE + -N $MAX_MPITASKS_PER_NODE --cc depth -d $OMP_NUM_THREADS -e OMP_STACKSIZE=128M -e OMP_NUM_THREADS=$OMP_NUM_THREADS @@ -1601,7 +1601,7 @@ slurm balwinder.singh -at- pnnl.gov 8 - 8 + 8 FALSE @@ -1674,7 +1674,7 @@ balwinder.singh -at- pnnl.gov 8 16 - 16 + 16 TRUE @@ -1741,7 +1741,7 @@ balwinder.singh -at- pnnl.gov 8 24 - 24 + 24 FALSE @@ -1840,7 +1840,7 @@ pbs dmricciuto 8 - 8 + 8 8 /projects/cesm/devtools/mpich-3.0.4-gcc4.8.1/bin/mpirun @@ -1888,7 +1888,7 @@ pbs dmricciuto 32 - 32 + 32 32 /projects/cesm/devtools/mpich-3.0.4-gcc4.8.1/bin/mpirun @@ -1924,7 +1924,7 @@ yinj -at- ornl.gov 4 32 - 16 + 16 pbs FALSE @@ -1982,7 +1982,7 @@ $ENV{PROJWORK}/$PROJECT CNL pbs - 16 + 16 acme 8 @@ -1997,7 +1997,7 @@ @@ -2170,7 +2170,7 @@ pbs acme 8 - 16 + 16 32 TRUE -D PIO_BUILD_TIMING:BOOL=ON @@ -2180,7 +2180,7 @@ -j {{ hyperthreading }} -S {{ tasks_per_numa }} -n $TOTALPES - -N $PES_PER_NODE + -N $MAX_MPITASKS_PER_NODE -d $ENV{OMP_NUM_THREADS} -cc numa_node @@ -2320,7 +2320,7 @@ 4 36 - 32 + 32 climateacme TRUE luke.vanroekel @ gmail.com @@ -2342,7 +2342,7 @@ jonbob -at- lanl.gov 4 16 - 16 + 16 climateacme TRUE /lustre/scratch3/turquoise/$ENV{USER}/ACME/input_data @@ -2430,7 +2430,7 @@ jedwards@ucar.edu 8 30 - 15 + 15 TRUE /glade/scratch/$USER/acme/timing/performance_archive @@ -2542,7 +2542,7 @@ -n $TOTALPES -S {{ tasks_per_numa }} - -N $PES_PER_NODE + -N $MAX_MPITASKS_PER_NODE -d $ENV{OMP_NUM_THREADS} @@ -2581,7 +2581,7 @@ -n $TOTALPES -S {{ tasks_per_numa }} - -N $PES_PER_NODE + -N $MAX_MPITASKS_PER_NODE -d $ENV{OMP_NUM_THREADS} @@ -2606,20 +2606,20 @@ gbisht at lbl dot gov 4 16 - 16 + 16 TRUE mpirun -np $TOTALPES - -npernode $PES_PER_NODE + -npernode $MAX_MPITASKS_PER_NODE mpirun -np $TOTALPES - -npernode $PES_PER_NODE + -npernode $MAX_MPITASKS_PER_NODE @@ -2669,20 +2669,20 @@ gbisht at lbl dot gov 4 12 - 12 + 12 TRUE mpirun -np $TOTALPES - -npernode $PES_PER_NODE + -npernode $MAX_MPITASKS_PER_NODE mpirun -np $TOTALPES - -npernode $PES_PER_NODE + -npernode $MAX_MPITASKS_PER_NODE @@ -2730,7 +2730,7 @@ none rgknox at lbl gov 4 - 4 + 4 FALSE @@ -2739,7 +2739,7 @@ mpirun -np $TOTALPES - -npernode $PES_PER_NODE + -npernode $MAX_MPITASKS_PER_NODE @@ -2766,7 +2766,7 @@ lsf acme 32 - 20 + 20 160 TRUE csc249 @@ -2781,7 +2781,7 @@ --report-bindings --display-map diff --git a/config/acme/machines/config_pio.xml b/config/acme/machines/config_pio.xml index f76c751661a..b63fbdc6cd5 100644 --- a/config/acme/machines/config_pio.xml +++ b/config/acme/machines/config_pio.xml @@ -20,7 +20,7 @@ - $PES_PER_NODE + $MAX_MPITASKS_PER_NODE 60 128 64 diff --git a/config/acme/machines/userdefined_laptop_template/config_machines.xml b/config/acme/machines/userdefined_laptop_template/config_machines.xml index 5790446f74e..f6cc9587d0b 100644 --- a/config/acme/machines/userdefined_laptop_template/config_machines.xml +++ b/config/acme/machines/userdefined_laptop_template/config_machines.xml @@ -19,7 +19,7 @@ __YOUR_NAME_HERE__ 4 4 - 2 + 2 diff --git a/config/cesm/machines/config_batch.xml b/config/cesm/machines/config_batch.xml index afbfadb93e1..6f0c7abffd0 100644 --- a/config/cesm/machines/config_batch.xml +++ b/config/cesm/machines/config_batch.xml @@ -65,7 +65,7 @@ - + @@ -83,7 +83,7 @@ - + diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml index 0960de8e9dd..a5580a5a2b4 100644 --- a/config/cesm/machines/config_machines.xml +++ b/config/cesm/machines/config_machines.xml @@ -66,14 +66,14 @@ pbs cseg 32 - 16 + 16 TRUE aprun -n $TOTALPES - -N $PES_PER_NODE + -N $MAX_MPITASKS_PER_NODE -d $ENV{OMP_NUM_THREADS} @@ -145,12 +145,12 @@ lsf tcraig -at- ucar.edu 16 - 16 + 16 mpirun -hostfile $ENV{PBS_JOBID} - -ppn $PES_PER_NODE + -ppn $MAX_MPITASKS_PER_NODE -n $TOTALPES @@ -208,7 +208,7 @@ cseg 36 - 36 + 36 TRUE mpiexec_mpt @@ -312,7 +312,7 @@ slurm tcraig -at- ucar.edu 24 - 24 + 24 FALSE srun @@ -409,7 +409,7 @@ slurm cseg 64 - 32 + 32 srun @@ -512,7 +512,7 @@ slurm cseg 256 - 64 + 64 srun @@ -612,7 +612,7 @@ slurm tcraig -at- ucar.edu 12 - 12 + 12 srun @@ -669,7 +669,7 @@ slurm cseg 48 - 24 + 24 srun @@ -768,14 +768,14 @@ pbs julio -at- ucar.edu 24 - 24 + 24 aprun -j {{ hyperthreading }} -n $TOTALPES -S {{ tasks_per_numa }} - -N $PES_PER_NODE + -N $MAX_MPITASKS_PER_NODE -d $ENV{OMP_NUM_THREADS} @@ -834,7 +834,7 @@ pbs cseg 48 - 48 + 48 mpiexec @@ -889,7 +889,7 @@ Customize these fields as appropriate for your system, - particularly changing MAX_TASKS_PER_NODE and PES_PER_NODE to the + particularly changing MAX_TASKS_PER_NODE and MAX_MPITASKS_PER_NODE to the number of cores on your machine. You may also want to change instances of '$ENV{HOME}/projects' to your desired directory organization. You can use this in either of two ways: (1) @@ -918,7 +918,7 @@ none __YOUR_NAME_HERE__ 4 - 4 + 4 mpirun @@ -950,7 +950,7 @@ cseg 36 - 36 + 36 TRUE mpiexec_mpt @@ -1017,7 +1017,7 @@ none jgfouca at sandia dot gov 64 - 64 + 64 mpirun @@ -1069,14 +1069,14 @@ cobalt cseg 64 - 8 + 8 TRUE /usr/bin/runjob --label short - --ranks-per-node $PES_PER_NODE + --ranks-per-node $MAX_MPITASKS_PER_NODE --np $TOTALPES --block $COBALT_PARTNAME --envs OMP_WAIT_POLICY=active --envs BG_SMP_FAST_WAKEUP=yes $LOCARGS @@ -1120,7 +1120,7 @@ slurm tcraig -at- ucar.edu 32 - 32 + 32 FALSE mpiexec_mpt @@ -1165,7 +1165,7 @@ pbs fvitt -at- ucar.edu 28 - 28 + 28 TRUE mpiexec_mpt @@ -1219,7 +1219,7 @@ pbs fvitt -at- ucar.edu 24 - 24 + 24 TRUE mpiexec_mpt @@ -1273,7 +1273,7 @@ pbs fvitt -at- ucar.edu 16 - 16 + 16 TRUE mpiexec_mpt @@ -1327,7 +1327,7 @@ pbs fvitt -at- ucar.edu 20 - 20 + 20 TRUE mpiexec_mpt @@ -1381,12 +1381,12 @@ slurm edouard.davin -at- env.ethz.ch 32 - 32 + 32 aprun -n $TOTALPES - -N $PES_PER_NODE + -N $MAX_MPITASKS_PER_NODE -d $ENV{OMP_NUM_THREADS} @@ -1417,7 +1417,7 @@ none jgfouca at sandia dot gov 64 - 64 + 64 mpirun @@ -1469,14 +1469,14 @@ slurm jgfouca at sandia dot gov 16 - 16 + 16 TRUE mpirun -np $TOTALPES - -npernode $PES_PER_NODE + -npernode $MAX_MPITASKS_PER_NODE @@ -1531,7 +1531,7 @@ slurm cseg 16 - 16 + 16 ibrun @@ -1594,13 +1594,13 @@ cobalt_theta cseg 64 - 64 + 64 TRUE aprun -n $TOTALPES - -N $PES_PER_NODE + -N $MAX_MPITASKS_PER_NODE --cc depth -d $OMP_NUM_THREADS -e OMP_STACKSIZE=64M -e OMP_NUM_THREADS=$OMP_NUM_THREADS @@ -1682,7 +1682,7 @@ lsf cseg 30 - 15 + 15 TRUE - - + diff --git a/config/xml_schemas/config_machines_template.xml b/config/xml_schemas/config_machines_template.xml index a93a5391c18..2092185f3dd 100644 --- a/config/xml_schemas/config_machines_template.xml +++ b/config/xml_schemas/config_machines_template.xml @@ -75,12 +75,12 @@ + should always be >= MAX_MPITASKS_PER_NODE --> 36 - - 36 + 36 diff --git a/config/xml_schemas/config_pes.xsd b/config/xml_schemas/config_pes.xsd index 28000481e0f..f82a55c8f38 100644 --- a/config/xml_schemas/config_pes.xsd +++ b/config/xml_schemas/config_pes.xsd @@ -8,7 +8,7 @@ - + @@ -76,7 +76,7 @@ - + diff --git a/doc/source/users_guide/customizing-a-case.rst b/doc/source/users_guide/customizing-a-case.rst index 0ec88eceed2..7e2d12ab6c2 100644 --- a/doc/source/users_guide/customizing-a-case.rst +++ b/doc/source/users_guide/customizing-a-case.rst @@ -67,12 +67,12 @@ The entries in **env_mach_pes.xml** have the following meanings: MAX_TASKS_PER_MODE The total number of (MPI tasks) * (OpenMP threads) allowed on a node. This is defined in **config_machines.xml** and therefore given a default setting, but can be user modified. - PES_PER_NODE The maximum number of MPI tasks per node. + MAX_MPITASKS_PER_NODE The maximum number of MPI tasks per node. This is defined in **config_machines.xml** and therefore given a default setting, but can be user modified. NTASKS Total number of MPI tasks. A negative value indicates nodes rather than tasks, where - PES_PER_NODE * -NTASKS equals the number of MPI tasks. + MAX_MPITASKS_PER_NODE * -NTASKS equals the number of MPI tasks. NTHRDS Number of OpenMP threads per MPI task. ROOTPE The global MPI task of the component root task; if negative, indicates nodes rather than tasks. PSTRID The stride of MPI tasks across the global set of pes (for now set to 1). @@ -117,9 +117,9 @@ If you had set ``ROOTPE_OCN`` to 64 in this example, a total of 176 processors w **Example 2** ---------------- -If a component has **NTASKS=-2**, **NTHRDS=4** and **ROOTPE=0**, **PES_PER_NODE=4**, **MAX_TASKS_PER_NODE=4**, it will run on (8 MPI tasks * 4 threads) = 32 hardware processors on 8 nodes. +If a component has **NTASKS=-2**, **NTHRDS=4** and **ROOTPE=0**, **MAX_MPITASKS_PER_NODE=4**, **MAX_TASKS_PER_NODE=4**, it will run on (8 MPI tasks * 4 threads) = 32 hardware processors on 8 nodes. -If you intended 2 nodes INSTEAD of 8 nodes, then you would change **PES_PER_NODE=1** (using **xmlchange**). +If you intended 2 nodes INSTEAD of 8 nodes, then you would change **MAX_MPITASKS_PER_NODE=1** (using **xmlchange**). **Note**: **env_mach_pes.xml** *cannot* be modified after **case.setup** has been invoked without first running the following: diff --git a/doc/source/users_guide/porting-cime.rst b/doc/source/users_guide/porting-cime.rst index beb6598e9b5..052915c7e2d 100644 --- a/doc/source/users_guide/porting-cime.rst +++ b/doc/source/users_guide/porting-cime.rst @@ -117,7 +117,7 @@ Each ```` tag requires the following input: - ``BATCH_SYSTEM``: batch system used on this machine (none is okay) - ``SUPPORTED_BY``: contact information for support for this system - ``MAX_TASKS_PER_NODE``: maximum number of threads/tasks per shared memory node on the machine -- ``PES_PER_NODE``: number of physical PES per shared node on the machine. In practice the MPI tasks per node will not exceed this value. +- ``MAX_MPITASKS_PER_NODE``: number of physical PES per shared node on the machine. In practice the MPI tasks per node will not exceed this value. - ``PROJECT_REQUIRED``: Does this machine require a project to be specified to the batch system? - ``mpirun``: The mpi exec to start a job on this machine. This is itself an element that has sub-elements that must be filled: diff --git a/scripts/create_test b/scripts/create_test index 856ff051cb2..f81b2402380 100755 --- a/scripts/create_test +++ b/scripts/create_test @@ -195,7 +195,7 @@ OR parser.add_argument("--proc-pool", type=int, default=None, help="The size of the processor pool that create_test can use. Default " - "is PES_PER_NODE + 25 percent.") + "is MAX_MPITASKS_PER_NODE + 25 percent.") parser.add_argument("--walltime", default=os.getenv("CIME_GLOBAL_WALLTIME"), help="Set the wallclock limit for all tests in the suite. " @@ -429,7 +429,7 @@ def single_submit_impl(machine_name, test_id, proc_pool, project, args, job_cost submit_cmd = env_batch.get_value("batch_submit", subgroup=None) submit_args = env_batch.get_submit_args(case, "case.run") - tasks_per_node = mach.get_value("PES_PER_NODE") + tasks_per_node = mach.get_value("MAX_MPITASKS_PER_NODE") num_nodes = int(math.ceil(float(proc_pool) / tasks_per_node)) if wall_time is None: wall_time = compute_total_time(job_cost_map, proc_pool) diff --git a/scripts/lib/CIME/XML/env_mach_pes.py b/scripts/lib/CIME/XML/env_mach_pes.py index 2995aeefc7b..e1c063a861a 100644 --- a/scripts/lib/CIME/XML/env_mach_pes.py +++ b/scripts/lib/CIME/XML/env_mach_pes.py @@ -19,7 +19,7 @@ def __init__(self, case_root=None, infile="env_mach_pes.xml", components=None): schema = os.path.join(get_cime_root(), "config", "xml_schemas", "env_mach_pes.xsd") EnvBase.__init__(self, case_root, infile, schema=schema) - def get_value(self, vid, attribute=None, resolved=True, subgroup=None, pes_per_node=None): # pylint: disable=arguments-differ + def get_value(self, vid, attribute=None, resolved=True, subgroup=None, MAX_MPITASKS_PER_NODE=None): # pylint: disable=arguments-differ # Special variable NINST_MAX is used to determine the number of # drivers in multi-driver mode. if vid == "NINST_MAX": @@ -31,10 +31,10 @@ def get_value(self, vid, attribute=None, resolved=True, subgroup=None, pes_per_n value = EnvBase.get_value(self, vid, attribute, resolved, subgroup) if "NTASKS" in vid or "ROOTPE" in vid: - if pes_per_node is None: - pes_per_node = self.get_value("PES_PER_NODE") + if MAX_MPITASKS_PER_NODE is None: + MAX_MPITASKS_PER_NODE = self.get_value("MAX_MPITASKS_PER_NODE") if value is not None and value < 0: - value = -1*value*pes_per_node + value = -1*value*MAX_MPITASKS_PER_NODE return value @@ -71,7 +71,7 @@ def get_cost_pes(self, totaltasks, max_thread_count, machine=None): figure out the value of COST_PES which is the pe value used to estimate model cost """ expect(totaltasks > 0,"totaltasks > 0 expected totaltasks = {}".format(totaltasks)) - pespn = self.get_value("PES_PER_NODE") + pespn = self.get_value("MAX_MPITASKS_PER_NODE") num_nodes, spare_nodes = self.get_total_nodes(totaltasks, max_thread_count) num_nodes += spare_nodes # This is hardcoded because on yellowstone by default we @@ -98,7 +98,7 @@ def get_total_tasks(self, comp_classes): def get_tasks_per_node(self, total_tasks, max_thread_count): expect(total_tasks > 0,"totaltasks > 0 expected totaltasks = {}".format(total_tasks)) tasks_per_node = min(self.get_value("MAX_TASKS_PER_NODE")/ max_thread_count, - self.get_value("PES_PER_NODE"), total_tasks) + self.get_value("MAX_MPITASKS_PER_NODE"), total_tasks) return tasks_per_node if tasks_per_node > 0 else 1 def get_total_nodes(self, total_tasks, max_thread_count): diff --git a/scripts/lib/CIME/XML/machines.py b/scripts/lib/CIME/XML/machines.py index e28a2e84d9f..fc39ac556b4 100644 --- a/scripts/lib/CIME/XML/machines.py +++ b/scripts/lib/CIME/XML/machines.py @@ -298,13 +298,13 @@ def print_values(self): os_ = machine.find("OS") compilers = machine.find("COMPILERS") max_tasks_per_node = machine.find("MAX_TASKS_PER_NODE") - pes_per_node = machine.find("PES_PER_NODE") + MAX_MPITASKS_PER_NODE = machine.find("MAX_MPITASKS_PER_NODE") print( " {} : {} ".format(name , desc.text)) print( " os ", os_.text) print( " compilers ",compilers.text) - if pes_per_node is not None: - print( " pes/node ",pes_per_node.text) + if MAX_MPITASKS_PER_NODE is not None: + print( " pes/node ",MAX_MPITASKS_PER_NODE.text) if max_tasks_per_node is not None: print( " max_tasks/node ",max_tasks_per_node.text) @@ -318,10 +318,10 @@ def return_all_values(self): os_ = machine.find("OS") compilers = machine.find("COMPILERS") max_tasks_per_node = machine.find("MAX_TASKS_PER_NODE") - pes_per_node = machine.find("PES_PER_NODE") + MAX_MPITASKS_PER_NODE = machine.find("MAX_MPITASKS_PER_NODE") ppn = '' - if pes_per_node is not None: - ppn = pes_per_node.text + if MAX_MPITASKS_PER_NODE is not None: + ppn = MAX_MPITASKS_PER_NODE.text max_tasks_pn = '' if max_tasks_per_node is not None: diff --git a/scripts/lib/CIME/aprun.py b/scripts/lib/CIME/aprun.py index 272344325ed..c09574912d2 100755 --- a/scripts/lib/CIME/aprun.py +++ b/scripts/lib/CIME/aprun.py @@ -11,7 +11,7 @@ ############################################################################### def _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, - max_tasks_per_node, pes_per_node, + max_tasks_per_node, MAX_MPITASKS_PER_NODE, pio_numtasks, pio_async_interface, compiler, machine, run_exe): ############################################################################### @@ -23,23 +23,23 @@ def _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, >>> rootpes = [0, 0, 512, 0, 680, 512, 512, 0, 0] >>> pstrids = [1, 1, 1, 1, 1, 1, 1, 1, 1] >>> max_tasks_per_node = 16 - >>> pes_per_node = 16 + >>> MAX_MPITASKS_PER_NODE = 16 >>> pio_numtasks = -1 >>> pio_async_interface = False >>> compiler = "pgi" >>> machine = "titan" >>> run_exe = "acme.exe" - >>> _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, max_tasks_per_node, pes_per_node, pio_numtasks, pio_async_interface, compiler, machine, run_exe) + >>> _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, max_tasks_per_node, MAX_MPITASKS_PER_NODE, pio_numtasks, pio_async_interface, compiler, machine, run_exe) (' -S 4 -n 680 -N 8 -d 2 acme.exe : -S 2 -n 128 -N 4 -d 4 acme.exe ', 117) >>> compiler = "intel" - >>> _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, max_tasks_per_node, pes_per_node, pio_numtasks, pio_async_interface, compiler, machine, run_exe) + >>> _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, max_tasks_per_node, MAX_MPITASKS_PER_NODE, pio_numtasks, pio_async_interface, compiler, machine, run_exe) (' -S 4 -cc numa_node -n 680 -N 8 -d 2 acme.exe : -S 2 -cc numa_node -n 128 -N 4 -d 4 acme.exe ', 117) >>> ntasks = [64, 64, 64, 64, 64, 64, 64, 64, 1] >>> nthreads = [1, 1, 1, 1, 1, 1, 1, 1, 1] >>> rootpes = [0, 0, 0, 0, 0, 0, 0, 0, 0] >>> pstrids = [1, 1, 1, 1, 1, 1, 1, 1, 1] - >>> _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, max_tasks_per_node, pes_per_node, pio_numtasks, pio_async_interface, compiler, machine, run_exe) + >>> _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, max_tasks_per_node, MAX_MPITASKS_PER_NODE, pio_numtasks, pio_async_interface, compiler, machine, run_exe) (' -S 8 -cc numa_node -n 64 -N 16 -d 1 acme.exe ', 4) """ max_tasks_per_node = 1 if max_tasks_per_node < 1 else max_tasks_per_node @@ -51,7 +51,7 @@ def _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, # Check if we need to add pio's tasks to the total task count if pio_async_interface: - total_tasks += pio_numtasks if pio_numtasks > 0 else pes_per_node + total_tasks += pio_numtasks if pio_numtasks > 0 else MAX_MPITASKS_PER_NODE # Compute max threads for each mpi task maxt = [0] * total_tasks @@ -74,7 +74,7 @@ def _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, 0, 1, maxt[0], maxt[0], 0, "" for c1 in xrange(1, total_tasks): if maxt[c1] != thread_count: - tasks_per_node = min(pes_per_node, max_tasks_per_node / thread_count) + tasks_per_node = min(MAX_MPITASKS_PER_NODE, max_tasks_per_node / thread_count) tasks_per_node = min(task_count, tasks_per_node) @@ -98,8 +98,8 @@ def _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, else: task_count += 1 - if pes_per_node > 0: - tasks_per_node = min(pes_per_node, max_tasks_per_node / thread_count) + if MAX_MPITASKS_PER_NODE > 0: + tasks_per_node = min(MAX_MPITASKS_PER_NODE, max_tasks_per_node / thread_count) else: tasks_per_node = max_tasks_per_node / thread_count @@ -135,7 +135,7 @@ def get_aprun_cmd_for_case(case, run_exe): return _get_aprun_cmd_for_case_impl(ntasks, nthreads, rootpes, pstrids, case.get_value("MAX_TASKS_PER_NODE"), - case.get_value("PES_PER_NODE"), + case.get_value("MAX_MPITASKS_PER_NODE"), case.get_value("PIO_NUMTASKS"), case.get_value("PIO_ASYNC_INTERFACE"), case.get_value("COMPILER"), diff --git a/scripts/lib/CIME/case.py b/scripts/lib/CIME/case.py index b14be26ca61..972619ccd3a 100644 --- a/scripts/lib/CIME/case.py +++ b/scripts/lib/CIME/case.py @@ -129,7 +129,7 @@ def initialize_derived_attributes(self): env_mach_pes = self.get_env("mach_pes") env_mach_spec = self.get_env('mach_specific') comp_classes = self.get_values("COMP_CLASSES") - pes_per_node = self.get_value("PES_PER_NODE") + MAX_MPITASKS_PER_NODE = self.get_value("MAX_MPITASKS_PER_NODE") self.total_tasks = env_mach_pes.get_total_tasks(comp_classes) self.thread_count = env_mach_pes.get_max_thread_count(comp_classes) @@ -137,10 +137,10 @@ def initialize_derived_attributes(self): logger.debug("total_tasks {} thread_count {}".format(self.total_tasks, self.thread_count)) self.tasks_per_numa = int(math.ceil(self.tasks_per_node / 2.0)) - smt_factor = max(1,int(self.get_value("MAX_TASKS_PER_NODE") / pes_per_node)) + smt_factor = max(1,int(self.get_value("MAX_TASKS_PER_NODE") / MAX_MPITASKS_PER_NODE)) threads_per_node = self.tasks_per_node * self.thread_count - threads_per_core = 1 if (threads_per_node <= pes_per_node) else smt_factor + threads_per_core = 1 if (threads_per_node <= MAX_MPITASKS_PER_NODE) else smt_factor self.cores_per_task = self.thread_count / threads_per_core mpi_attribs = { @@ -719,10 +719,10 @@ def _setup_mach_pes(self, pecount, multi_driver, ninst, machine_name, mpilib): mach_pes_obj.set_value(rootpe_str, rootpe) pesize = 1 - pes_per_node = self.get_value("PES_PER_NODE") + MAX_MPITASKS_PER_NODE = self.get_value("MAX_MPITASKS_PER_NODE") for val in totaltasks: if val < 0: - val = -1*val*pes_per_node + val = -1*val*MAX_MPITASKS_PER_NODE if val > pesize: pesize = val if multi_driver: diff --git a/scripts/lib/CIME/get_timing.py b/scripts/lib/CIME/get_timing.py index 330e365215e..2e128bf56ad 100644 --- a/scripts/lib/CIME/get_timing.py +++ b/scripts/lib/CIME/get_timing.py @@ -137,8 +137,8 @@ def _getTiming(self, inst=0): stop_n = self.case.get_value("STOP_N") cost_pes = self.case.get_value("COST_PES") totalpes = self.case.get_value("TOTALPES") - pes_per_node = self.case.get_value("PES_PER_NODE") - smt_factor = max(1,int(self.case.get_value("MAX_TASKS_PER_NODE") / pes_per_node)) + MAX_MPITASKS_PER_NODE = self.case.get_value("MAX_MPITASKS_PER_NODE") + smt_factor = max(1,int(self.case.get_value("MAX_TASKS_PER_NODE") / MAX_MPITASKS_PER_NODE)) if cost_pes > 0: pecost = cost_pes @@ -306,7 +306,7 @@ def _getTiming(self, inst=0): self.write("\n") self.write(" total pes active : {} \n".format(totalpes*maxthrds*smt_factor )) - self.write(" pes per node : {} \n".format(pes_per_node)) + self.write(" pes per node : {} \n".format(MAX_MPITASKS_PER_NODE)) self.write(" pe count for cost estimate : {} \n".format(pecost)) self.write("\n") diff --git a/scripts/lib/CIME/test_scheduler.py b/scripts/lib/CIME/test_scheduler.py index f2cf884b13a..af7cac13433 100644 --- a/scripts/lib/CIME/test_scheduler.py +++ b/scripts/lib/CIME/test_scheduler.py @@ -199,7 +199,7 @@ def __init__(self, test_names, test_data=None, # Oversubscribe by 1/4 if proc_pool is None: - pes = int(self._machobj.get_value("PES_PER_NODE")) + pes = int(self._machobj.get_value("MAX_MPITASKS_PER_NODE")) self._proc_pool = int(pes * 1.25) else: self._proc_pool = int(proc_pool) diff --git a/scripts/query_config b/scripts/query_config index f734d69ed3c..4b2fb3a6094 100755 --- a/scripts/query_config +++ b/scripts/query_config @@ -295,7 +295,7 @@ class Machines(CIME.XML.machines.Machines): os_ = machine.find("OS") compilers = machine.find("COMPILERS") max_tasks_per_node = machine.find("MAX_TASKS_PER_NODE") - pes_per_node = machine.find("PES_PER_NODE") + MAX_MPITASKS_PER_NODE = machine.find("MAX_MPITASKS_PER_NODE") current_machine = self.probe_machine_name(warn=False) if not single_machine: @@ -303,16 +303,16 @@ class Machines(CIME.XML.machines.Machines): print " {} : {} ".format(name, desc.text) print " os ", os_.text print " compilers ",compilers.text - if pes_per_node is not None: - print " pes/node ",pes_per_node.text + if MAX_MPITASKS_PER_NODE is not None: + print " pes/node ",MAX_MPITASKS_PER_NODE.text if max_tasks_per_node is not None: print " max_tasks/node ",max_tasks_per_node.text elif single_machine and machine_name in name: print " {} : {} ".format(name, desc.text) print " os ", os_.text print " compilers ",compilers.text - if pes_per_node is not None: - print " pes/node ",pes_per_node.text + if MAX_MPITASKS_PER_NODE is not None: + print " pes/node ",MAX_MPITASKS_PER_NODE.text if max_tasks_per_node is not None: print " max_tasks/node ",max_tasks_per_node.text diff --git a/src/drivers/mct/cime_config/config_component.xml b/src/drivers/mct/cime_config/config_component.xml index 7c4456af959..ddfda8864a8 100644 --- a/src/drivers/mct/cime_config/config_component.xml +++ b/src/drivers/mct/cime_config/config_component.xml @@ -949,7 +949,7 @@ 0 mach_pes_last env_mach_pes.xml - pes or cores used relative to PES_PER_NODE for accounting (0 means TOTALPES is valid) + pes or cores used relative to MAX_MPITASKS_PER_NODE for accounting (0 means TOTALPES is valid) @@ -2032,7 +2032,7 @@ maximum number of tasks/ threads allowed per node - + integer 0 mach_pes_last