Merge pull request #143 from xylar/update-frontier

Updates to Frontier
E3SM-Project · Jan 30, 2024 · 445ac07 · 445ac07
2 parents 8e0cb59 + 33fe455
commit 445ac07
Show file tree

Hide file tree

Showing 10 changed files with 659 additions and 1 deletion.
diff --git a/mache/machines/frontier.cfg b/mache/machines/frontier.cfg
@@ -41,7 +41,7 @@ system = slurm
 parallel_executable = srun
 
 # cores per node on the machine
-cores_per_node = 56
+cores_per_node = 64
 
 # account for running diagnostics jobs
 account = cli115

diff --git a/mache/spack/frontier_crayclang_mpich.csh b/mache/spack/frontier_crayclang_mpich.csh
@@ -0,0 +1,20 @@
+module reset >& /dev/null
+module switch PrgEnv-cray PrgEnv-cray/8.3.3 >& /dev/null
+module switch cce cce/15.0.1 >& /dev/null
+module switch craype craype/2.7.20 >& /dev/null
+
+{% if e3sm_lapack %}
+module load cray-libsci/22.12.1.1
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+module load cray-hdf5-parallel/1.12.2.1
+module load cray-netcdf-hdf5parallel/4.9.0.1
+module load cray-parallel-netcdf/1.12.3.1
+{% endif %}
+
+{% if e3sm_hdf5_netcdf %}
+setenv NETCDF_C_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX
+setenv NETCDF_FORTRAN_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX
+setenv PNETCDF_PATH $CRAY_PARALLEL_NETCDF_PREFIX
+{% endif %}
+setenv HDF5_USE_FILE_LOCKING FALSE
diff --git a/mache/spack/frontier_crayclang_mpich.sh b/mache/spack/frontier_crayclang_mpich.sh
@@ -0,0 +1,20 @@
+module reset >& /dev/null
+module switch PrgEnv-cray PrgEnv-cray/8.3.3 >& /dev/null
+module switch cce cce/15.0.1 >& /dev/null
+module switch craype craype/2.7.20 >& /dev/null
+
+{% if e3sm_lapack %}
+module load cray-libsci/22.12.1.1
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+module load cray-hdf5-parallel/1.12.2.1
+module load cray-netcdf-hdf5parallel/4.9.0.1
+module load cray-parallel-netcdf/1.12.3.1
+{% endif %}
+
+{% if e3sm_hdf5_netcdf %}
+export NETCDF_C_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX
+export NETCDF_FORTRAN_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX
+export PNETCDF_PATH=$CRAY_PARALLEL_NETCDF_PREFIX
+{% endif %}
+export HDF5_USE_FILE_LOCKING=FALSE
diff --git a/mache/spack/frontier_crayclang_mpich.yaml b/mache/spack/frontier_crayclang_mpich.yaml
@@ -0,0 +1,175 @@
+spack:
+  specs:
+  - cray-mpich
+{% if e3sm_lapack %}
+  - cray-libsci
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+  - hdf5
+  - netcdf-c
+  - netcdf-fortran
+  - parallel-netcdf
+{% endif %}
+{{ specs }}
+  concretizer:
+    unify: when_possible
+  packages:
+    all:
+      compiler: [[email protected]]
+      providers:
+        mpi: [[email protected]]
+{% if e3sm_lapack %}
+        lapack: [[email protected]]
+{% endif %}
+    autoconf:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    automake:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    bzip2:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    cmake:
+      externals:
+      - spec: [email protected]
+        prefix: /sw/frontier/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/cmake-3.23.2-4r4mpiba7cwdw2hlakh5i7tchi64s3qd
+      buildable: false
+    curl:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    gettext:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    gmake:
+      externals:
+      - spec: [email protected]
+        prefix: /sw/frontier/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/gmake-4.3-32qt54ujpdxcat7ceg4vcjlmf2ze25j5
+      buildable: false
+    libxml2:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    m4:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    ncurses:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    openssl:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    perl:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    python:
+      externals:
+      - spec: [email protected]
+        prefix: /opt/cray/pe/python/3.9.13.1
+        modules:
+        - cray-python/3.9.13.1
+      buildable: false
+    tar:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    xz:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    cray-mpich:
+      externals:
+      - spec: [email protected]
+        prefix: /opt/cray/pe/mpich/8.1.23/ofi/crayclang/10.0
+        modules:
+        - libfabric/1.15.2.0
+        - cray-mpich/8.1.23
+      buildable: false
+    libfabric:
+      externals:
+      - spec: [email protected]
+        prefix: /opt/cray/libfabric/1.15.2.0
+        modules:
+        - libfabric/1.15.2.0
+      buildable: false
+{% if e3sm_lapack %}
+    cray-libsci:
+      externals:
+      - spec: [email protected]
+        prefix: /opt/cray/pe/libsci/22.12.1.1/crayclang/9.0/x86_64
+        modules:
+        - cray-libsci/22.12.1.1
+      buildable: false
+{% endif %}
+{% if e3sm_hdf5_netcdf or system_hdf5_netcdf %}
+    hdf5:
+      externals:
+      - spec: [email protected]~cxx+fortran+hl~java+mpi+shared
+        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.1/crayclang/14.0
+      buildable: false
+    parallel-netcdf:
+      externals:
+      - spec: [email protected]+cxx+fortran+pic+shared
+        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.1/crayclang/14.0
+      buildable: false
+    netcdf-c:
+      externals:
+      - spec: [email protected]+mpi~parallel-netcdf
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.1/crayclang/14.0
+      buildable: false
+    netcdf-fortran:
+      externals:
+      - spec: [email protected]
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.1/crayclang/14.0
+      buildable: false
+{% endif %}
+  config:
+    install_missing_compilers: false
+  compilers:
+  - compiler:
+      spec: [email protected]
+      paths:
+        cc: cc
+        cxx: CC
+        f77: ftn
+        fc: ftn
+      flags: {}
+      operating_system: sles15
+      target: any
+      modules:
+      - PrgEnv-cray/8.3.3
+      - cce/15.0.1
+      - craype/2.7.20
+      - libfabric/1.15.2.0
+      environment:
+        set:
+          RFE_811452_DISABLE: '1'
+        append_path:
+          LD_LIBRARY_PATH: /opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64
+        prepend_path:
+          LD_LIBRARY_PATH: /opt/cray/pe/pmi/6.1.8/lib
+          PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.22__gd067c3f.shasta/lib64/pkgconfig"
+      extra_rpaths:
+      - /opt/cray/pe/gcc-libs
+      - /opt/cray/gcc-libs
diff --git a/mache/spack/frontier_crayclanggpu_mpich.csh b/mache/spack/frontier_crayclanggpu_mpich.csh
@@ -0,0 +1,22 @@
+module reset >& /dev/null
+module switch PrgEnv-cray PrgEnv-cray/8.3.3 >& /dev/null
+module switch cce cce/15.0.1 >& /dev/null
+module switch craype craype/2.7.20 >& /dev/null
+module load craype-accel-amd-gfx90a
+module load rocm/5.4.0
+
+{% if e3sm_lapack %}
+module load cray-libsci/22.12.1.1
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+module load cray-hdf5-parallel/1.12.2.1
+module load cray-netcdf-hdf5parallel/4.9.0.1
+module load cray-parallel-netcdf/1.12.3.1
+{% endif %}
+
+{% if e3sm_hdf5_netcdf %}
+setenv NETCDF_C_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX
+setenv NETCDF_FORTRAN_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX
+setenv PNETCDF_PATH $CRAY_PARALLEL_NETCDF_PREFIX
+{% endif %}
+setenv HDF5_USE_FILE_LOCKING FALSE
diff --git a/mache/spack/frontier_crayclanggpu_mpich.sh b/mache/spack/frontier_crayclanggpu_mpich.sh
@@ -0,0 +1,22 @@
+module reset >& /dev/null
+module switch PrgEnv-cray PrgEnv-cray/8.3.3 >& /dev/null
+module switch cce cce/15.0.1 >& /dev/null
+module switch craype craype/2.7.20 >& /dev/null
+module load craype-accel-amd-gfx90a
+module load rocm/5.4.0
+
+{% if e3sm_lapack %}
+module load cray-libsci/22.12.1.1
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+module load cray-hdf5-parallel/1.12.2.1
+module load cray-netcdf-hdf5parallel/4.9.0.1
+module load cray-parallel-netcdf/1.12.3.1
+{% endif %}
+
+{% if e3sm_hdf5_netcdf %}
+export NETCDF_C_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX
+export NETCDF_FORTRAN_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX
+export PNETCDF_PATH=$CRAY_PARALLEL_NETCDF_PREFIX
+{% endif %}
+export HDF5_USE_FILE_LOCKING=FALSE