From 04716dcfa01f2b4b84e40068d38e50690975ea0c Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sat, 24 Dec 2022 06:49:02 -0800 Subject: [PATCH 1/6] Update docs with new gnu-cray and intel-cray build targets --- docs/developers_guide/machines/cori.rst | 4 ++-- docs/developers_guide/machines/index.rst | 6 +++--- docs/developers_guide/machines/perlmutter.rst | 2 +- docs/users_guide/machines/cori.rst | 4 ++-- docs/users_guide/machines/perlmutter.rst | 2 +- utils/matrix/setup_matrix.py | 6 +++--- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/developers_guide/machines/cori.rst b/docs/developers_guide/machines/cori.rst index 62347f79a7..df3a440c96 100644 --- a/docs/developers_guide/machines/cori.rst +++ b/docs/developers_guide/machines/cori.rst @@ -16,7 +16,7 @@ Then, you can build the MPAS model with .. code-block:: bash - make [DEBUG=true] [OPENMP=true] intel-nersc + make [DEBUG=true] [OPENMP=true] intel-cray cori-haswell, gnu ----------------- @@ -31,4 +31,4 @@ Then, you can build the MPAS model with .. code-block:: bash - make [DEBUG=true] [OPENMP=true] [ALBANY=true] gnu-nersc + make [DEBUG=true] [OPENMP=true] [ALBANY=true] gnu-cray diff --git a/docs/developers_guide/machines/index.rst b/docs/developers_guide/machines/index.rst index 405d80b733..417a5565c5 100644 --- a/docs/developers_guide/machines/index.rst +++ b/docs/developers_guide/machines/index.rst @@ -61,11 +61,11 @@ the MPAS model. | +------------+-----------+-------------------+ | | gnu | openmpi | gfortran | +--------------+------------+-----------+-------------------+ -| cori-haswell | intel | mpt | intel-nersc | +| cori-haswell | intel | mpt | intel-cray | | +------------+-----------+-------------------+ -| | gnu | mpt | gnu-nersc | +| | gnu | mpt | gnu-cray | +--------------+------------+-----------+-------------------+ -| pm-cpu | gnu | mpich | gnu-nersc | +| pm-cpu | gnu | mpich | gnu-cray | +--------------+------------+-----------+-------------------+ Below are specifics for each supported machine diff --git a/docs/developers_guide/machines/perlmutter.rst b/docs/developers_guide/machines/perlmutter.rst index 46ba35f306..63d371858d 100644 --- a/docs/developers_guide/machines/perlmutter.rst +++ b/docs/developers_guide/machines/perlmutter.rst @@ -15,4 +15,4 @@ Then, you can build the MPAS model with .. code-block:: bash - make [DEBUG=true] gnu-nersc + make [DEBUG=true] gnu-cray diff --git a/docs/users_guide/machines/cori.rst b/docs/users_guide/machines/cori.rst index 696565db9c..6f6bd598a7 100644 --- a/docs/users_guide/machines/cori.rst +++ b/docs/users_guide/machines/cori.rst @@ -138,7 +138,7 @@ To build the MPAS model with .. code-block:: bash - make [DEBUG=true] [OPENMP=true] intel-nersc + make [DEBUG=true] [OPENMP=true] intel-cray Gnu on Cori-Haswell @@ -155,7 +155,7 @@ To build the MPAS model with .. code-block:: bash - make [DEBUG=true] [OPENMP=true] [ALBANY=true] gnu-nersc + make [DEBUG=true] [OPENMP=true] [ALBANY=true] gnu-cray Jupyter notebook on remote data diff --git a/docs/users_guide/machines/perlmutter.rst b/docs/users_guide/machines/perlmutter.rst index 861f78cbb8..ba1e61a628 100644 --- a/docs/users_guide/machines/perlmutter.rst +++ b/docs/users_guide/machines/perlmutter.rst @@ -152,7 +152,7 @@ To build the MPAS model with .. code-block:: bash - make [DEBUG=true] [OPENMP=true] [ALBANY=true] gnu-nersc + make [DEBUG=true] [OPENMP=true] [ALBANY=true] gnu-cray Jupyter notebook on remote data diff --git a/utils/matrix/setup_matrix.py b/utils/matrix/setup_matrix.py index 205e18b2fe..1c79632364 100755 --- a/utils/matrix/setup_matrix.py +++ b/utils/matrix/setup_matrix.py @@ -36,10 +36,10 @@ ('intel', 'impi'): 'intel-mpi', ('gnu', 'openmpi'): 'gfortran'}, 'cori-haswell': { - ('intel', 'mpt'): 'intel-nersc', - ('gnu', 'mpt'): 'gnu-nersc'}, + ('intel', 'mpt'): 'intel-cray', + ('gnu', 'mpt'): 'gnu-cray'}, 'pm-cpu': { - ('gnu', 'mpich'): 'gnu-nersc'}, + ('gnu', 'mpich'): 'gnu-cray'}, 'conda-linux': { ('gfortran', 'mpich'): 'gfortran', ('gfortran', 'openmpi'): 'gfortran'}, From 7b7b9f686ab584db227d7c50eba4c04695a66eaf Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sat, 24 Dec 2022 03:37:04 -0800 Subject: [PATCH 2/6] Update to 1.2.0-alpha.4 and mache 1.10.0 --- compass/version.py | 2 +- conda/compass_env/spec-file.template | 2 +- conda/configure_compass_env.py | 2 +- conda/recipe/meta.yaml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compass/version.py b/compass/version.py index 44c2e1d388..84f88e39e7 100644 --- a/compass/version.py +++ b/compass/version.py @@ -1 +1 @@ -__version__ = '1.2.0-alpha.3' +__version__ = '1.2.0-alpha.4' diff --git a/conda/compass_env/spec-file.template b/conda/compass_env/spec-file.template index ae7df49f39..49b54ea202 100644 --- a/conda/compass_env/spec-file.template +++ b/conda/compass_env/spec-file.template @@ -15,7 +15,7 @@ jigsaw=0.9.14 jigsawpy=0.3.3 jupyter lxml -mache=1.9.0 +mache=1.10.0 matplotlib-base metis mpas_tools=0.17.0 diff --git a/conda/configure_compass_env.py b/conda/configure_compass_env.py index ac9fe29da3..a7fd4938c7 100755 --- a/conda/configure_compass_env.py +++ b/conda/configure_compass_env.py @@ -53,7 +53,7 @@ def setup_install_env(env_name, activate_base, use_local, logger, recreate, channels = '--use-local' else: channels = '' - packages = 'progressbar2 jinja2 "mache=1.9.0"' + packages = 'progressbar2 jinja2 "mache=1.10.0"' if recreate or not os.path.exists(env_path): print('Setting up a conda environment for installing compass\n') commands = '{}; ' \ diff --git a/conda/recipe/meta.yaml b/conda/recipe/meta.yaml index 991a1b3a39..2d77664ab3 100644 --- a/conda/recipe/meta.yaml +++ b/conda/recipe/meta.yaml @@ -1,5 +1,5 @@ {% set name = "compass" %} -{% set version = "1.2.0alpha.3" %} +{% set version = "1.2.0alpha.4" %} {% set build = 0 %} {% if mpi == "nompi" %} @@ -52,7 +52,7 @@ requirements: - jigsawpy 0.3.3 - jupyter - lxml - - mache 1.9.0 + - mache 1.10.0 - matplotlib-base - metis - mpas_tools 0.17.0 From c9604f11947dab27f3d3eaf1d5664b81267122db Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 9 Jan 2023 04:01:45 -0800 Subject: [PATCH 3/6] set `$LD_LIBRARY_PATH` after creating a new spack env This is needed because externally built codes like MPAS components can't find the spack libraries at runtime without it. --- conda/bootstrap.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index a94b38f153..e8daf9e7dc 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -354,7 +354,7 @@ def get_env_vars(machine, compiler, mpilib): def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, - spack_base, spack_template_path, env_vars, tmpdir): + spack_base, spack_template_path, env_vars, tmpdir, logger): albany = config.get('deploy', 'albany') esmf = config.get('deploy', 'esmf') @@ -412,6 +412,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, files = glob.glob(os.path.join(include_path, f'{prefix}*')) for filename in files: os.remove(filename) + set_ld_library_path(spack_branch_base, spack_env, logger) spack_script = get_spack_script( spack_path=spack_branch_base, env_name=spack_env, compiler=compiler, @@ -460,6 +461,15 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, return spack_branch_base, spack_script, env_vars +def set_ld_library_path(spack_branch_base, spack_env, logger): + commands = \ + f'source {spack_branch_base}/share/spack/setup-env.sh; ' \ + f'spack env activate {spack_env}; ' \ + f'spack config add modules:prefix_inspections:lib:[LD_LIBRARY_PATH]; ' \ + f'spack config add modules:prefix_inspections:lib64:[LD_LIBRARY_PATH]' + check_call(commands, logger=logger) + + def write_load_compass(template_path, activ_path, conda_base, env_type, activ_suffix, prefix, env_name, spack_script, machine, env_vars, env_only, source_path, without_openmp): @@ -863,7 +873,7 @@ def main(): spack_branch_base, spack_script, env_vars = build_spack_env( config, args.update_spack, machine, compiler, mpi, spack_env, spack_base, spack_template_path, env_vars, - args.tmpdir) + args.tmpdir, logger) spack_script = f'echo Loading Spack environment...\n' \ f'{spack_script}\n' \ f'echo Done.\n' \ From e111370525a97fd7cf119207e0ecdd3585fc251f Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 11 Jan 2023 05:57:53 -0600 Subject: [PATCH 4/6] Add more unsupported compilers and MPIs Don't support oneapi-ifx compiler on Chrysalis Don't support various compilers on Perlmutter --- conda/unsupported.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conda/unsupported.txt b/conda/unsupported.txt index 7cf00c65b1..0f4c6ead20 100644 --- a/conda/unsupported.txt +++ b/conda/unsupported.txt @@ -3,9 +3,15 @@ # no spack available anvil, gnu, impi chrysalis, gnu, impi +chrysalis, oneapi-ifx, openmpi +chrysalis, oneapi-ifx, impi compy, intel, mvapich2 compy, pgi, impi compy, pgi, mvapich2 +pm-cpu, nvidia, mpich +pm-cpu, aocc, mpich +pm-cpu, amdclang, mpich + # compiles but tests unreliable (errors or hanging), # see https://github.com/MPAS-Dev/compass/issues/336 From 2c4747172eeb9bd141b681d42b5bc031b686be7c Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 13 Jan 2023 07:53:17 -0800 Subject: [PATCH 5/6] Require C++17 standard for Albany (and Trilinos) --- conda/bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index e8daf9e7dc..c1eeac696e 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -392,7 +392,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, specs.append(f'scorpio@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc') if albany != 'None': - specs.append(f'albany@{albany}+mpas') + specs.append(f'albany@{albany}+mpas+cxx17') yaml_template = f'{spack_template_path}/{machine}_{compiler}_{mpi}.yaml' if not os.path.exists(yaml_template): From 2bd90a02af3aa14b4778225c322008ed96d7bd1e Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sat, 14 Jan 2023 02:36:19 -0600 Subject: [PATCH 6/6] Disable Albany with MVAPICH on Anvil It is failing 9 integration tests, whereas OpenMPI is working well. --- conda/albany_supported.txt | 1 - docs/users_guide/machines/anvil.rst | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/conda/albany_supported.txt b/conda/albany_supported.txt index afc6b6e5bb..d40679c7b7 100644 --- a/conda/albany_supported.txt +++ b/conda/albany_supported.txt @@ -1,6 +1,5 @@ # a list of supported machine, compiler and mpi combinations for Albany -anvil, gnu, mvapich anvil, gnu, openmpi chrysalis, gnu, openmpi compy, gnu, openmpi diff --git a/docs/users_guide/machines/anvil.rst b/docs/users_guide/machines/anvil.rst index a4a3539925..f304fb7d56 100644 --- a/docs/users_guide/machines/anvil.rst +++ b/docs/users_guide/machines/anvil.rst @@ -105,7 +105,7 @@ variables: .. code-block:: bash - source /lcrc/soft/climate/compass/anvil/load_latest_compass_gnu_mvapich.sh + source /lcrc/soft/climate/compass/anvil/load_latest_compass_gnu_openmpi.sh To build the MPAS model with