Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

v11: Update Intel MPI flags #632

Open
wants to merge 4 commits into
base: develop
Choose a base branch
from
Open

v11: Update Intel MPI flags #632

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0a13fd9
Update Intel MPI flags
mathomp4 Jul 24, 2024
519d946
Use shm:ofi on SLES15
mathomp4 Jul 24, 2024
f498b52
Merge branch 'develop' into feature/mathomp4/update-intelmpi-flags-v11
mathomp4 Jul 31, 2024
6464869
Merge branch 'develop' into feature/mathomp4/update-intelmpi-flags-v11
mathomp4 Sep 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 17 additions & 31 deletions gcm_setup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2188,9 +2188,6 @@ endif
else if( $MPI_STACK == intelmpi ) then

cat > $HOMDIR/SETENV.commands << EOF
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_GATHERV 3

# This flag prints out the Intel MPI state. Uncomment if needed
#setenv I_MPI_DEBUG 9
EOF
Expand All @@ -2200,7 +2197,24 @@ EOF
# specific compared to the above adjustments
if ( $SITE == 'NCCS' ) then

# Some flags we know work on SLES15 and Milan (see below)
# For safety, we keep the old SLES12 flags for that system
#
# NOTE: When Cascade Lake is on SLES15, the following flags
# might need to be Milan only

if ("$BUILT_ON_SLES15" == "TRUE") then
cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_FABRICS shm:ofi
setenv I_MPI_OFI_PROVIDER psm3
EOF

else

cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_GATHERV 3

setenv I_MPI_SHM_HEAP_VSIZE 512
setenv PSM2_MEMORY large
EOF
Expand All @@ -2216,34 +2230,6 @@ EOF

endif # if NOT Singularity

# Testing by Bill Putman found these to be
# useful flags with Intel MPI on SLES15 on the
# Milan nodes.
# Note 1: Testing by NCCS shows the PSM3 provider
# runs on the Infiniband fabric. Tests show it runs
# up to C720.
# Note 2: When the Cascade Lakes are moved to
# SLES15, these will need to be Milan-only flags
# as Intel MPI will probably work just fine with
# Intel chips.
if ("$BUILT_ON_SLES15" == "TRUE") then
cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_FALLBACK 0
setenv I_MPI_FABRICS ofi
setenv I_MPI_OFI_PROVIDER psm3
setenv I_MPI_ADJUST_SCATTER 2
setenv I_MPI_ADJUST_SCATTERV 2
setenv I_MPI_ADJUST_GATHER 2
setenv I_MPI_ADJUST_GATHERV 3
setenv I_MPI_ADJUST_ALLGATHER 3
setenv I_MPI_ADJUST_ALLGATHERV 3
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_REDUCE 10
setenv I_MPI_ADJUST_BCAST 11
setenv I_MPI_ADJUST_REDUCE_SCATTER 4
setenv I_MPI_ADJUST_BARRIER 9
EOF

endif # if SLES15

endif # if NCCS
Expand Down
52 changes: 19 additions & 33 deletions geoschemchem_setup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1763,7 +1763,7 @@ if( $SITE == 'NAS' ) then
setenv CHMDIR ${BOUNDARY_DIR}/fvInput_nc3 # locations of Aerosol Chemistry BCs
setenv WRKDIR /nobackup/$LOGNAME # user work directory
setenv COUPLEDIR ${BOUNDARY_DIR}/bcs_shared/make_bcs_inputs/ocean # Coupled Ocean/Atmos Forcing
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files

else if( $SITE == 'NCCS' ) then
setenv BATCH_CMD "sbatch" # SLURM Batch command
Expand Down Expand Up @@ -1839,7 +1839,7 @@ else if( $SITE == 'AWS' | $SITE == 'Azure' ) then
setenv BOUNDARY_DIR /ford1/share/gmao_SIteam/ModelData
setenv BC_BASE ${BOUNDARY_DIR}/bcs_shared/fvInput/ExtData/esm/tiles
setenv BCSDIR ${BC_BASE}/${LSM_BCS} # location of Boundary Conditions
setenv REPLAY_ANA_EXPID REPLAY_UNSUPPORTED # Default Analysis Experiment for REPLAY
setenv REPLAY_ANA_EXPID REPLAY_UNSUPPORTED # Default Analysis Experiment for REPLAY
setenv REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for REPLAY
setenv M2_REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for M2 REPLAY

Expand Down Expand Up @@ -2218,9 +2218,6 @@ endif
else if( $MPI_STACK == intelmpi ) then

cat > $HOMDIR/SETENV.commands << EOF
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_GATHERV 3

# This flag prints out the Intel MPI state. Uncomment if needed
#setenv I_MPI_DEBUG 9
EOF
Expand All @@ -2230,7 +2227,24 @@ EOF
# specific compared to the above adjustments
if ( $SITE == 'NCCS' ) then

# Some flags we know work on SLES15 and Milan (see below)
# For safety, we keep the old SLES12 flags for that system
#
# NOTE: When Cascade Lake is on SLES15, the following flags
# might need to be Milan only

if ("$BUILT_ON_SLES15" == "TRUE") then
cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_FABRICS shm:ofi
setenv I_MPI_OFI_PROVIDER psm3
EOF

else

cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_GATHERV 3

setenv I_MPI_SHM_HEAP_VSIZE 512
setenv PSM2_MEMORY large
EOF
Expand All @@ -2246,34 +2260,6 @@ EOF

endif # if NOT Singularity

# Testing by Bill Putman found these to be
# useful flags with Intel MPI on SLES15 on the
# Milan nodes.
# Note 1: Testing by NCCS shows the PSM3 provider
# runs on the Infiniband fabric. Tests show it runs
# up to C720.
# Note 2: When the Cascade Lakes are moved to
# SLES15, these will need to be Milan-only flags
# as Intel MPI will probably work just fine with
# Intel chips.
if ("$BUILT_ON_SLES15" == "TRUE") then
cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_FALLBACK 0
setenv I_MPI_FABRICS ofi
setenv I_MPI_OFI_PROVIDER psm3
setenv I_MPI_ADJUST_SCATTER 2
setenv I_MPI_ADJUST_SCATTERV 2
setenv I_MPI_ADJUST_GATHER 2
setenv I_MPI_ADJUST_GATHERV 3
setenv I_MPI_ADJUST_ALLGATHER 3
setenv I_MPI_ADJUST_ALLGATHERV 3
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_REDUCE 10
setenv I_MPI_ADJUST_BCAST 11
setenv I_MPI_ADJUST_REDUCE_SCATTER 4
setenv I_MPI_ADJUST_BARRIER 9
EOF

endif # if SLES15

endif # if NCCS
Expand Down
54 changes: 20 additions & 34 deletions gmichem_setup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1888,7 +1888,7 @@ if( $SITE == 'NAS' ) then
setenv BATCH_TIME "PBS -l walltime=" # PBS Syntax for walltime
setenv BATCH_JOBNAME "PBS -N " # PBS Syntax for job name
setenv BATCH_OUTPUTNAME "PBS -o " # PBS Syntax for job output name
setenv BATCH_JOINOUTERR "PBS -j oe -k oed" # PBS Syntax for joining output and error
setenv BATCH_JOINOUTERR "PBS -j oe -k oed" # PBS Syntax for joining output and error
setenv RUN_FT "6:00:00" # Wallclock Time for gcm_forecast.j
setenv RUN_T "8:00:00" # Wallclock Time for gcm_run.j
setenv POST_T "8:00:00" # Wallclock Time for gcm_post.j
Expand Down Expand Up @@ -1928,7 +1928,7 @@ if( $SITE == 'NAS' ) then
setenv CHMDIR ${BOUNDARY_DIR}/fvInput_nc3 # locations of Aerosol Chemistry BCs
setenv WRKDIR /nobackup/$LOGNAME # user work directory
setenv COUPLEDIR ${BOUNDARY_DIR}/bcs_shared/make_bcs_inputs/ocean # Coupled Ocean/Atmos Forcing
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files

else if( $SITE == 'NCCS' ) then
setenv BATCH_CMD "sbatch" # SLURM Batch command
Expand Down Expand Up @@ -2004,7 +2004,7 @@ else if( $SITE == 'AWS' | $SITE == 'Azure' ) then
setenv BOUNDARY_DIR /ford1/share/gmao_SIteam/ModelData
setenv BC_BASE ${BOUNDARY_DIR}/bcs_shared/fvInput/ExtData/esm/tiles
setenv BCSDIR ${BC_BASE}/${LSM_BCS} # location of Boundary Conditions
setenv REPLAY_ANA_EXPID REPLAY_UNSUPPORTED # Default Analysis Experiment for REPLAY
setenv REPLAY_ANA_EXPID REPLAY_UNSUPPORTED # Default Analysis Experiment for REPLAY
setenv REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for REPLAY
setenv M2_REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for M2 REPLAY

Expand Down Expand Up @@ -2389,9 +2389,6 @@ endif
else if( $MPI_STACK == intelmpi ) then

cat > $HOMDIR/SETENV.commands << EOF
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_GATHERV 3

# This flag prints out the Intel MPI state. Uncomment if needed
#setenv I_MPI_DEBUG 9
EOF
Expand All @@ -2401,7 +2398,24 @@ EOF
# specific compared to the above adjustments
if ( $SITE == 'NCCS' ) then

# Some flags we know work on SLES15 and Milan (see below)
# For safety, we keep the old SLES12 flags for that system
#
# NOTE: When Cascade Lake is on SLES15, the following flags
# might need to be Milan only

if ("$BUILT_ON_SLES15" == "TRUE") then
cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_FABRICS shm:ofi
setenv I_MPI_OFI_PROVIDER psm3
EOF

else

cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_GATHERV 3

setenv I_MPI_SHM_HEAP_VSIZE 512
setenv PSM2_MEMORY large
EOF
Expand All @@ -2417,34 +2431,6 @@ EOF

endif # if NOT Singularity

# Testing by Bill Putman found these to be
# useful flags with Intel MPI on SLES15 on the
# Milan nodes.
# Note 1: Testing by NCCS shows the PSM3 provider
# runs on the Infiniband fabric. Tests show it runs
# up to C720.
# Note 2: When the Cascade Lakes are moved to
# SLES15, these will need to be Milan-only flags
# as Intel MPI will probably work just fine with
# Intel chips.
if ("$BUILT_ON_SLES15" == "TRUE") then
cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_FALLBACK 0
setenv I_MPI_FABRICS ofi
setenv I_MPI_OFI_PROVIDER psm3
setenv I_MPI_ADJUST_SCATTER 2
setenv I_MPI_ADJUST_SCATTERV 2
setenv I_MPI_ADJUST_GATHER 2
setenv I_MPI_ADJUST_GATHERV 3
setenv I_MPI_ADJUST_ALLGATHER 3
setenv I_MPI_ADJUST_ALLGATHERV 3
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_REDUCE 10
setenv I_MPI_ADJUST_BCAST 11
setenv I_MPI_ADJUST_REDUCE_SCATTER 4
setenv I_MPI_ADJUST_BARRIER 9
EOF

endif # if SLES15

endif # if NCCS
Expand Down
52 changes: 19 additions & 33 deletions stratchem_setup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1748,7 +1748,7 @@ if( $SITE == 'NAS' ) then
setenv CHMDIR ${BOUNDARY_DIR}/fvInput_nc3 # locations of Aerosol Chemistry BCs
setenv WRKDIR /nobackup/$LOGNAME # user work directory
setenv COUPLEDIR ${BOUNDARY_DIR}/bcs_shared/make_bcs_inputs/ocean # Coupled Ocean/Atmos Forcing
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files
setenv GWDRSDIR ${BOUNDARY_DIR}/GWD_RIDGE # Location of GWD_RIDGE files

else if( $SITE == 'NCCS' ) then
setenv BATCH_CMD "sbatch" # SLURM Batch command
Expand Down Expand Up @@ -1824,7 +1824,7 @@ else if( $SITE == 'AWS' | $SITE == 'Azure' ) then
setenv BOUNDARY_DIR /ford1/share/gmao_SIteam/ModelData
setenv BC_BASE ${BOUNDARY_DIR}/bcs_shared/fvInput/ExtData/esm/tiles
setenv BCSDIR ${BC_BASE}/${LSM_BCS} # location of Boundary Conditions
setenv REPLAY_ANA_EXPID REPLAY_UNSUPPORTED # Default Analysis Experiment for REPLAY
setenv REPLAY_ANA_EXPID REPLAY_UNSUPPORTED # Default Analysis Experiment for REPLAY
setenv REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for REPLAY
setenv M2_REPLAY_ANA_LOCATION REPLAY_UNSUPPORTED # Default Analysis Location for M2 REPLAY

Expand Down Expand Up @@ -2203,9 +2203,6 @@ endif
else if( $MPI_STACK == intelmpi ) then

cat > $HOMDIR/SETENV.commands << EOF
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_GATHERV 3

# This flag prints out the Intel MPI state. Uncomment if needed
#setenv I_MPI_DEBUG 9
EOF
Expand All @@ -2215,7 +2212,24 @@ EOF
# specific compared to the above adjustments
if ( $SITE == 'NCCS' ) then

# Some flags we know work on SLES15 and Milan (see below)
# For safety, we keep the old SLES12 flags for that system
#
# NOTE: When Cascade Lake is on SLES15, the following flags
# might need to be Milan only

if ("$BUILT_ON_SLES15" == "TRUE") then
cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_FABRICS shm:ofi
setenv I_MPI_OFI_PROVIDER psm3
EOF

else

cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_GATHERV 3

setenv I_MPI_SHM_HEAP_VSIZE 512
setenv PSM2_MEMORY large
EOF
Expand All @@ -2231,34 +2245,6 @@ EOF

endif # if NOT Singularity

# Testing by Bill Putman found these to be
# useful flags with Intel MPI on SLES15 on the
# Milan nodes.
# Note 1: Testing by NCCS shows the PSM3 provider
# runs on the Infiniband fabric. Tests show it runs
# up to C720.
# Note 2: When the Cascade Lakes are moved to
# SLES15, these will need to be Milan-only flags
# as Intel MPI will probably work just fine with
# Intel chips.
if ("$BUILT_ON_SLES15" == "TRUE") then
cat >> $HOMDIR/SETENV.commands << EOF
setenv I_MPI_FALLBACK 0
setenv I_MPI_FABRICS ofi
setenv I_MPI_OFI_PROVIDER psm3
setenv I_MPI_ADJUST_SCATTER 2
setenv I_MPI_ADJUST_SCATTERV 2
setenv I_MPI_ADJUST_GATHER 2
setenv I_MPI_ADJUST_GATHERV 3
setenv I_MPI_ADJUST_ALLGATHER 3
setenv I_MPI_ADJUST_ALLGATHERV 3
setenv I_MPI_ADJUST_ALLREDUCE 12
setenv I_MPI_ADJUST_REDUCE 10
setenv I_MPI_ADJUST_BCAST 11
setenv I_MPI_ADJUST_REDUCE_SCATTER 4
setenv I_MPI_ADJUST_BARRIER 9
EOF

endif # if SLES15

endif # if NCCS
Expand Down
Loading