From b0514426a32909b1ad70912c979878a2bcc616cb Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 16 May 2023 07:40:40 -0600 Subject: [PATCH 01/20] Update to 1.2.0-alpha.6 This merge also updates mache to 1.15.0, which brings in support for external machines (unknown to mache) and support for intel on Perlmutter --- compass/version.py | 2 +- conda/compass_env/spec-file.template | 2 +- conda/configure_compass_env.py | 2 +- conda/recipe/meta.yaml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compass/version.py b/compass/version.py index cbbfb1a865..447b7d027d 100644 --- a/compass/version.py +++ b/compass/version.py @@ -1 +1 @@ -__version__ = '1.2.0-alpha.5' +__version__ = '1.2.0-alpha.6' diff --git a/conda/compass_env/spec-file.template b/conda/compass_env/spec-file.template index 8b48c90add..d53a75c59a 100644 --- a/conda/compass_env/spec-file.template +++ b/conda/compass_env/spec-file.template @@ -18,7 +18,7 @@ jigsawpy=0.3.3 jupyter lxml {% if include_mache %} -mache=1.14.0 +mache=1.15.0 {% endif %} matplotlib-base metis diff --git a/conda/configure_compass_env.py b/conda/configure_compass_env.py index cff86bbbeb..efb8933fdb 100755 --- a/conda/configure_compass_env.py +++ b/conda/configure_compass_env.py @@ -101,7 +101,7 @@ def main(): if local_mache: mache = '' else: - mache = '"mache=1.14.0"' + mache = '"mache=1.15.0"' setup_install_env(env_name, activate_base, args.use_local, logger, args.recreate, conda_base, mache) diff --git a/conda/recipe/meta.yaml b/conda/recipe/meta.yaml index 58f0011773..e8a9231f6f 100644 --- a/conda/recipe/meta.yaml +++ b/conda/recipe/meta.yaml @@ -1,5 +1,5 @@ {% set name = "compass" %} -{% set version = "1.2.0alpha.5" %} +{% set version = "1.2.0alpha.6" %} {% set build = 0 %} {% if mpi == "nompi" %} @@ -54,7 +54,7 @@ requirements: - jigsawpy 0.3.3 - jupyter - lxml - - mache 1.14.0 + - mache 1.15.0 - matplotlib-base - metis - mpas_tools 0.19.0 From f0dfb8d7f440eb6a07b2494279ff147d39f8671a Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 17 May 2023 17:11:45 -0600 Subject: [PATCH 02/20] Name the spack clone the same as the spack env This way, different users can create different spack clones. Multiple users cannot modify the same clone. --- conda/bootstrap.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index 57bc1e3fa2..429b60244e 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -14,7 +14,6 @@ from jinja2 import Template from mache import MachineInfo, discover_machine from mache.spack import get_spack_script, make_spack_env -from mache.version import __version__ as mache_version from shared import ( check_call, get_conda_base, @@ -394,7 +393,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, petsc = config.get('deploy', 'petsc') scorpio = config.get('deploy', 'scorpio') - spack_branch_base = f'{spack_base}/spack_for_mache_{mache_version}' + spack_branch_base = f'{spack_base}/{spack_env}' specs = list() From 84ab6bf6985eabe0d3b5520aa8f515bb055b1ac6 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 25 May 2023 23:29:34 -0500 Subject: [PATCH 03/20] Drop pio from esmf spec --- conda/bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index 429b60244e..f8620241b1 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -413,7 +413,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, f'parallel-netcdf "@{pnetcdf}+cxx+fortran"']) if esmf != 'None': - specs.append(f'esmf "@{esmf}+mpi+netcdf~pio+pnetcdf"') + specs.append(f'esmf "@{esmf}+mpi+netcdf+pnetcdf"') if lapack != 'None': specs.append(f'netlib-lapack "@{lapack}"') include_e3sm_lapack = False From a8926268db9e39ce2dd34d029f1e013801c616bd Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 25 May 2023 19:43:17 -0500 Subject: [PATCH 04/20] Fix quotes in spack env. yaml --- conda/bootstrap.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index f8620241b1..d7a3074f93 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -398,7 +398,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, specs = list() if cmake != 'None': - specs.append(f'cmake "@{cmake}"') + specs.append(f'"cmake@{cmake}"') e3sm_hdf5_netcdf = config.getboolean('deploy', 'use_e3sm_hdf5_netcdf') if not e3sm_hdf5_netcdf: @@ -407,28 +407,28 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, netcdf_fortran = config.get('deploy', 'netcdf_fortran') pnetcdf = config.get('deploy', 'pnetcdf') specs.extend([ - f'hdf5 "@{hdf5}+cxx+fortran+hl+mpi+shared"', - f'netcdf-c "@{netcdf_c}+mpi~parallel-netcdf"', - f'netcdf-fortran "@{netcdf_fortran}"', - f'parallel-netcdf "@{pnetcdf}+cxx+fortran"']) + f'"hdf5@{hdf5}+cxx+fortran+hl+mpi+shared"', + f'"netcdf-c@{netcdf_c}+mpi~parallel-netcdf"', + f'"netcdf-fortran@{netcdf_fortran}"', + f'"parallel-netcdf@{pnetcdf}+cxx+fortran"']) if esmf != 'None': - specs.append(f'esmf "@{esmf}+mpi+netcdf+pnetcdf"') + specs.append(f'"esmf@{esmf}+mpi+netcdf+pnetcdf"') if lapack != 'None': - specs.append(f'netlib-lapack "@{lapack}"') + specs.append(f'"netlib-lapack@{lapack}"') include_e3sm_lapack = False else: include_e3sm_lapack = True if petsc != 'None': - specs.append(f'petsc "@{petsc}+mpi+batch"') + specs.append(f'"petsc@{petsc}+mpi+batch"') if scorpio != 'None': specs.append( - f'scorpio ' - f'"@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc"') + f'"scorpio' + f'@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc"') if albany != 'None': - specs.append(f'albany "@{albany}+mpas"') + specs.append(f'"albany@{albany}+mpas"') yaml_template = f'{spack_template_path}/{machine}_{compiler}_{mpi}.yaml' if not os.path.exists(yaml_template): From 7d8db08a1a3ab6b18a6e8f9a9c02326cb10158a3 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 26 May 2023 17:32:44 -0500 Subject: [PATCH 05/20] Switch from SCORPIO to ParallelIO Update to latest ESMF Set `HAVE_ADIOS=false` in the load script --- conda/bootstrap.py | 17 ++++++++++++++--- conda/default.cfg | 6 ++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index d7a3074f93..71e58cfe80 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -383,8 +383,9 @@ def get_env_vars(machine, compiler, mpilib): return env_vars -def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, - spack_base, spack_template_path, env_vars, tmpdir, logger): +def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901 + spack_env, spack_base, spack_template_path, env_vars, + tmpdir, logger): albany = config.get('deploy', 'albany') cmake = config.get('deploy', 'cmake') @@ -392,6 +393,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, lapack = config.get('deploy', 'lapack') petsc = config.get('deploy', 'petsc') scorpio = config.get('deploy', 'scorpio') + parallelio = config.get('deploy', 'parallelio') spack_branch_base = f'{spack_base}/{spack_env}' @@ -413,7 +415,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, f'"parallel-netcdf@{pnetcdf}+cxx+fortran"']) if esmf != 'None': - specs.append(f'"esmf@{esmf}+mpi+netcdf+pnetcdf"') + specs.append(f'"esmf@{esmf}+mpi+netcdf+pnetcdf+external-parallelio"') if lapack != 'None': specs.append(f'"netlib-lapack@{lapack}"') include_e3sm_lapack = False @@ -427,6 +429,11 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, f'"scorpio' f'@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc"') + if parallelio != 'None': + specs.append( + f'"parallelio' + f'@{parallelio}+pnetcdf~timing"') + if albany != 'None': specs.append(f'"albany@{albany}+mpas"') @@ -464,6 +471,10 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, f'{spack_env}/.spack-env/view' env_vars = f'{env_vars}' \ f'export PIO={spack_view}\n' + if parallelio != 'None': + env_vars = \ + f'{env_vars}' \ + f'export HAVE_ADIOS=false\n' if albany != 'None': albany_flag_filename = f'{spack_view}/export_albany.in' if not os.path.exists(albany_flag_filename): diff --git a/conda/default.cfg b/conda/default.cfg index b8cf19a440..3f730bdd0d 100644 --- a/conda/default.cfg +++ b/conda/default.cfg @@ -23,11 +23,13 @@ mpi = nompi albany = develop # cmake newer than 3.23.0 needed for Trilinos cmake = 3.23.0: -esmf = 8.2.0 +esmf = 8.4.2 hdf5 = 1.12.1 lapack = 3.9.1 netcdf_c = 4.8.1 netcdf_fortran = 4.5.3 petsc = 3.16.1 pnetcdf = 1.12.2 -scorpio = 1.3.2 +# scorpio = 1.3.2 +scorpio = None +parallelio = 2.5.10 From 9a5ec5f6c940ab094948e556e7f909a7fb2185a4 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 15 May 2023 21:30:52 -0600 Subject: [PATCH 06/20] For known machines, use config file in spack builds This will allow support for machines that are unknown to mache. --- conda/bootstrap.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index 71e58cfe80..c7a622314c 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -440,13 +440,21 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901 yaml_template = f'{spack_template_path}/{machine}_{compiler}_{mpi}.yaml' if not os.path.exists(yaml_template): yaml_template = None + + if machine is not None: + here = os.path.abspath(os.path.dirname(__file__)) + machine_config = os.path.join(here, '..', 'compass', 'machines', + f'{machine}.cfg') + else: + machine_config = None + if update_spack: home_dir = os.path.expanduser('~') log_message(logger, 'Removing ~/.spack for safety') safe_rmtree(os.path.join(home_dir, '.spack')) make_spack_env(spack_path=spack_branch_base, env_name=spack_env, spack_specs=specs, compiler=compiler, mpi=mpi, - machine=machine, + machine=machine, config_file=machine_config, include_e3sm_lapack=include_e3sm_lapack, include_e3sm_hdf5_netcdf=e3sm_hdf5_netcdf, yaml_template=yaml_template, tmpdir=tmpdir) @@ -462,7 +470,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901 spack_script = get_spack_script( spack_path=spack_branch_base, env_name=spack_env, compiler=compiler, - mpi=mpi, shell='sh', machine=machine, + mpi=mpi, shell='sh', machine=machine, config_file=machine_config, include_e3sm_lapack=include_e3sm_lapack, include_e3sm_hdf5_netcdf=e3sm_hdf5_netcdf, yaml_template=yaml_template) From 7bce0ffe903b798cc74c2b98eefb8c83a022fd7a Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 15 May 2023 21:32:42 -0600 Subject: [PATCH 07/20] Add support for new machine `morpheus` This is my laptop as an example of a machine not supported by mache. --- compass/machines/morpheus.cfg | 53 +++++++++++++++++++++++++++ conda/spack/morpheus_gnu_openmpi.yaml | 38 +++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 compass/machines/morpheus.cfg create mode 100644 conda/spack/morpheus_gnu_openmpi.yaml diff --git a/compass/machines/morpheus.cfg b/compass/machines/morpheus.cfg new file mode 100644 index 0000000000..ebe2830bb9 --- /dev/null +++ b/compass/machines/morpheus.cfg @@ -0,0 +1,53 @@ +# The parallel section describes options related to running jobs in parallel +[parallel] + +# parallel system of execution: slurm, cobalt or single_node +system = single_node + +# whether to use mpirun or srun to run a task +parallel_executable = mpirun + +# cores per node on the machine +cores_per_node = 8 + + +# Config options related to spack environments +[spack] + +# whether to load modules from the spack yaml file before loading the spack +# environment +modules_before = False + +# whether to load modules from the spack yaml file after loading the spack +# environment +modules_after = False + + +# The paths section describes paths that are used within the ocean core test +# cases. +[paths] + +# A shared root directory where MPAS standalone data can be found +database_root = /home/xylar/data/mpas/mpas_standalonedata + +# the path to the base conda environment where compass environments have +# been created +compass_envs = /home/xylar/data/mpas/compass_envs + + +# Options related to deploying a compass conda environment on supported +# machines +[deploy] + +# the compiler set to use for system libraries and MPAS builds +compiler = gnu + +# the system MPI library to use for gnu compiler +mpi_gnu = openmpi + +# the base path for spack environments used by compass +spack = /home/xylar/data/mpas/spack + +# whether to use the same modules for hdf5, netcdf-c, netcdf-fortran and +# pnetcdf as E3SM (spack modules are used otherwise) +use_e3sm_hdf5_netcdf = False diff --git a/conda/spack/morpheus_gnu_openmpi.yaml b/conda/spack/morpheus_gnu_openmpi.yaml new file mode 100644 index 0000000000..6e5a4aa8d7 --- /dev/null +++ b/conda/spack/morpheus_gnu_openmpi.yaml @@ -0,0 +1,38 @@ +spack: + specs: + - gcc + - openmpi +{{ specs }} + concretizer: + unify: true + packages: + all: + compiler: [gcc@11.3.0] + providers: + mpi: [openmpi] + curl: + externals: + - spec: curl@7.81.0 + prefix: /usr + buildable: false + gcc: + externals: + - spec: gcc@11.3.0 + prefix: /usr + buildable: false + config: + install_missing_compilers: false + compilers: + - compiler: + spec: gcc@11.3.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: ubuntu22.04 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] From b75efb37233933048342b1f42733cec3851f371b Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 15 May 2023 21:54:06 -0600 Subject: [PATCH 08/20] Skip exception when getting config from mache To support machines that aren't known to mache, we need to suppress exceptions when trying to read mache config options for a machine. --- compass/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compass/setup.py b/compass/setup.py index a0c2cb6145..02ad6d686e 100644 --- a/compass/setup.py +++ b/compass/setup.py @@ -432,7 +432,8 @@ def _get_basic_config(config_file, machine, mpas_model_path, mpas_core): # add the E3SM config options from mache if machine is not None: - config.add_from_package('mache.machines', f'{machine}.cfg') + config.add_from_package('mache.machines', f'{machine}.cfg', + exception=False) # add the compass machine config file if machine is None: From 445f5d1de28403625af9dc4a8b19846d904d6285 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 4 Jun 2023 23:17:38 -0600 Subject: [PATCH 09/20] Support config option for identifying known machines This merge adds a config option `hostname_contains` to a new config section `discovery` that can be used to identify the machine by hostname if `mache` is unable to identify it. This capability has been added both when calling the configure script and when setting up test cases. --- compass/machines/__init__.py | 54 +++++++++++++++++++++++++++++++++++ compass/machines/morpheus.cfg | 7 +++++ compass/setup.py | 3 +- conda/bootstrap.py | 52 +++++++++++++++++++++++++++++++-- 4 files changed, 111 insertions(+), 5 deletions(-) diff --git a/compass/machines/__init__.py b/compass/machines/__init__.py index e69de29bb2..688ad26c74 100644 --- a/compass/machines/__init__.py +++ b/compass/machines/__init__.py @@ -0,0 +1,54 @@ +import os +import socket + +try: + from importlib.resources import contents as imp_res_contents +except ImportError: + # python<=3.8 + from importlib_resources import contents as imp_res_contents + +from mache import discover_machine as mache_discover_machine + +from compass.config import CompassConfigParser + + +def discover_machine(quiet=False): + """ + Figure out the machine from the host name + + Parameters + ---------- + quiet : bool, optional + Whether to print warnings if the machine name is ambiguous + + Returns + ------- + machine : str + The name of the current machine + """ + machine = mache_discover_machine(quiet=quiet) + if machine is None: + possible_hosts = _get_possible_hosts() + hostname = socket.gethostname() + for possible_machine, hostname_contains in possible_hosts.items(): + if hostname_contains in hostname: + machine = possible_machine + break + return machine + + +def _get_possible_hosts(): + machine_contents = imp_res_contents('compass.machines') + possible_hosts = dict() + for filename in machine_contents: + if filename.endswith('.cfg'): + machine = os.path.split(filename)[0] + config = CompassConfigParser() + config.add_from_package('compass.machines', filename) + if config.has_section('discovery') and \ + config.has_option('discovery', 'hostname_contains'): + hostname_contains = config.get('discovery', + 'hostname_contains') + possible_hosts[machine] = hostname_contains + + return possible_hosts diff --git a/compass/machines/morpheus.cfg b/compass/machines/morpheus.cfg index ebe2830bb9..f44918fffa 100644 --- a/compass/machines/morpheus.cfg +++ b/compass/machines/morpheus.cfg @@ -51,3 +51,10 @@ spack = /home/xylar/data/mpas/spack # whether to use the same modules for hdf5, netcdf-c, netcdf-fortran and # pnetcdf as E3SM (spack modules are used otherwise) use_e3sm_hdf5_netcdf = False + + +# Options related to machine discovery +[discovery] + +# a substring used to identify this machine from its hostname +hostname_contains = morpheus diff --git a/compass/setup.py b/compass/setup.py index 02ad6d686e..70fa193bc1 100644 --- a/compass/setup.py +++ b/compass/setup.py @@ -4,12 +4,11 @@ import sys import warnings -from mache import discover_machine - from compass import provenance from compass.config import CompassConfigParser from compass.io import symlink from compass.job import write_job_script +from compass.machines import discover_machine from compass.mpas_cores import get_mpas_cores diff --git a/conda/bootstrap.py b/conda/bootstrap.py index c7a622314c..4bf55fcfc8 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -6,13 +6,15 @@ import os import platform import shutil +import socket import stat import subprocess from configparser import ConfigParser import progressbar from jinja2 import Template -from mache import MachineInfo, discover_machine +from mache import MachineInfo +from mache import discover_machine as mache_discover_machine from mache.spack import get_spack_script, make_spack_env from shared import ( check_call, @@ -824,6 +826,50 @@ def safe_rmtree(path): pass +def discover_machine(quiet=False): + """ + Figure out the machine from the host name + + Parameters + ---------- + quiet : bool, optional + Whether to print warnings if the machine name is ambiguous + + Returns + ------- + machine : str + The name of the current machine + """ + machine = mache_discover_machine(quiet=quiet) + if machine is None: + possible_hosts = get_possible_hosts() + hostname = socket.gethostname() + for possible_machine, hostname_contains in possible_hosts.items(): + if hostname_contains in hostname: + machine = possible_machine + break + return machine + + +def get_possible_hosts(): + here = os.path.abspath(os.path.dirname(__file__)) + files = sorted(glob.glob(os.path.join( + here, '..', 'compass', 'machines', '*.cfg'))) + + possible_hosts = dict() + for filename in files: + machine = os.path.splitext(os.path.split(filename)[1])[0] + config = ConfigParser() + config.read(filename) + if config.has_section('discovery') and \ + config.has_option('discovery', 'hostname_contains'): + hostname_contains = config.get('discovery', + 'hostname_contains') + possible_hosts[machine] = hostname_contains + + return possible_hosts + + def main(): # noqa: C901 args = parse_args(bootstrap=True) @@ -848,7 +894,7 @@ def main(): # noqa: C901 else: machine = args.machine - e3sm_machine = machine is not None + known_machine = machine is not None if machine is None and not args.env_only: if platform.system() == 'Linux': @@ -913,7 +959,7 @@ def main(): # noqa: C901 if args.spack_base is not None: spack_base = args.spack_base - elif e3sm_machine and compiler is not None: + elif known_machine and compiler is not None: spack_base = get_spack_base(args.spack_base, config) else: spack_base = None From ad9c19ee171d8d2d1618db6175bb804b80367bad Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 5 Jun 2023 09:18:42 -0600 Subject: [PATCH 10/20] Add support for new machine eligos --- compass/machines/eligos.cfg | 60 +++++++++++++++++++++++++++++ conda/spack/eligos_gnu_openmpi.yaml | 38 ++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 compass/machines/eligos.cfg create mode 100644 conda/spack/eligos_gnu_openmpi.yaml diff --git a/compass/machines/eligos.cfg b/compass/machines/eligos.cfg new file mode 100644 index 0000000000..aa1a52bdaa --- /dev/null +++ b/compass/machines/eligos.cfg @@ -0,0 +1,60 @@ +# The parallel section describes options related to running jobs in parallel +[parallel] + +# parallel system of execution: slurm, cobalt or single_node +system = single_node + +# whether to use mpirun or srun to run a task +parallel_executable = mpirun + +# cores per node on the machine +cores_per_node = 4 + + +# Config options related to spack environments +[spack] + +# whether to load modules from the spack yaml file before loading the spack +# environment +modules_before = False + +# whether to load modules from the spack yaml file after loading the spack +# environment +modules_after = False + + +# The paths section describes paths that are used within the ocean core test +# cases. +[paths] + +# A shared root directory where MPAS standalone data can be found +database_root = /home/xylar/data/mpas/mpas_standalonedata + +# the path to the base conda environment where compass environments have +# been created +compass_envs = /home/xylar/data/mpas/compass_envs + + +# Options related to deploying a compass conda environment on supported +# machines +[deploy] + +# the compiler set to use for system libraries and MPAS builds +compiler = gnu + +# the system MPI library to use for gnu compiler +mpi_gnu = openmpi + +# the base path for spack environments used by compass +spack = /home/xylar/data/mpas/spack + +# whether to use the same modules for hdf5, netcdf-c, netcdf-fortran and +# pnetcdf as E3SM (spack modules are used otherwise) +use_e3sm_hdf5_netcdf = False + + +# Options related to machine discovery +[discovery] + +# a substring used to identify this machine from its hostname +hostname_contains = eligos diff --git a/conda/spack/eligos_gnu_openmpi.yaml b/conda/spack/eligos_gnu_openmpi.yaml new file mode 100644 index 0000000000..355346e0ba --- /dev/null +++ b/conda/spack/eligos_gnu_openmpi.yaml @@ -0,0 +1,38 @@ +spack: + specs: + - gcc + - openmpi +{{ specs }} + concretizer: + unify: true + packages: + all: + compiler: [gcc@9.4.0] + providers: + mpi: [openmpi] + curl: + externals: + - spec: curl@7.68.0 + prefix: /usr + buildable: false + gcc: + externals: + - spec: gcc@9.4.0 + prefix: /usr + buildable: false + config: + install_missing_compilers: false + compilers: + - compiler: + spec: gcc@9.4.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: ubuntu20.04 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] From 84dac6d303ce432e3d8c154f400fae10d4b41907 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 5 Jun 2023 11:11:46 -0600 Subject: [PATCH 11/20] Add documentation on adding a supported machine --- docs/developers_guide/machines/index.rst | 282 +++++++++++++++++++++++ 1 file changed, 282 insertions(+) diff --git a/docs/developers_guide/machines/index.rst b/docs/developers_guide/machines/index.rst index 7cb96e1c1f..d24e6332ef 100644 --- a/docs/developers_guide/machines/index.rst +++ b/docs/developers_guide/machines/index.rst @@ -117,3 +117,285 @@ Under OSX, you can build the MPAS model with .. code-block:: bash make gfortran-clang + +.. _dev_add_supported_machine: + +Adding a New Supported Machine +------------------------------ + +If you want to add a new supported machine, you need to add both a config file +and a yaml file describing your machine, as detailed below. + +Adding a Machine Config File +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The first step in adding a new supported machine to add a config file in +``compass/machines``. The config file needs to describe the parallel +environment and some paths where shared Spack environments will be installed +and shared data will be downloaded. The easiest place to start is one of the +examples provided (machines ``morpheus`` and ``eligos`` for now, but more +will be added soon.) + +.. code-block:: cfg + + # The parallel section describes options related to running jobs in parallel + [parallel] + + # parallel system of execution: slurm, cobalt or single_node + system = single_node + + # whether to use mpirun or srun to run a task + parallel_executable = mpirun + + # cores per node on the machine + cores_per_node = 8 + + + # Config options related to spack environments + [spack] + + # whether to load modules from the spack yaml file before loading the spack + # environment + modules_before = False + + # whether to load modules from the spack yaml file after loading the spack + # environment + modules_after = False + + + # The paths section describes paths that are used within the ocean core test + # cases. + [paths] + + # A shared root directory where MPAS standalone data can be found + database_root = /home/xylar/data/mpas/mpas_standalonedata + + # the path to the base conda environment where compass environments have + # been created + compass_envs = /home/xylar/data/mpas/compass_envs + + + # Options related to deploying a compass conda environment on supported + # machines + [deploy] + + # the compiler set to use for system libraries and MPAS builds + compiler = gnu + + # the system MPI library to use for gnu compiler + mpi_gnu = openmpi + + # the base path for spack environments used by compass + spack = /home/xylar/data/mpas/spack + + # whether to use the same modules for hdf5, netcdf-c, netcdf-fortran and + # pnetcdf as E3SM (spack modules are used otherwise) + use_e3sm_hdf5_netcdf = False + + + # Options related to machine discovery + [discovery] + + # a substring used to identify this machine from its hostname + hostname_contains = morpheus + + +The ``[parallel]`` section should describe the type of parallel queuing +system (currently only ``slurm`` or ``single_node`` are supported), the number +of cores per node and the command for running an MPI executable (typically +``srun`` for Slurm and ``mpirun`` for a "single node" machine like a laptop or +workstation. + +The ``[spack]`` section has some config options to do with loading system +modules before or after loading a Spack environment. On a "single node" +machine, you typically don't have modules so both ``modules_before`` and +``modules_after`` can be set to ``False``. On a high-performance computing +(HPC) machine, you may find it is safest to load modules after the Spack +environment to ensure that certain paths and environment variables are set the +way the modules have them, rather than the way that Spack would have them. +The recommended starting point would be ``modules_before = False`` and +``modules_after = True``, but could be adjusted as needed if the right shared +libraries aren't being found when you try to build an MPAS component. + +In the ``[paths]`` section, you will first give a path where you would like +to store shared data files used in compass test cases in ``database_root``. +Compass will create this directory if it doesn't exist. Then, you can specify +``compass_envs`` as a path where shared conda environments will be installed +for compass releases. If developers always create their own conda +environments, this path will never be used. + +In ``[deploy]``, you will specify config options used in setting up conda +and Spack environments for developers. The ``compiler`` is the default +compiler to use for your system. You must supply a corresponding +``mpi_`` for each supported compiler (not just the default compiler) +that specifies the default MPI library for that compiler. If you only support +one compiler and MPI library, that's pretty simple: ``compiler`` is the name +of the compiler (e.g. ``intel`` or ``gnu``) and ``mpi_`` is the +MPI library (e.g. ``compiler_gnu = mpich`` or ``compiler_intel = openmpi``). +The ``spack`` option specifies a path where Spack environment will be created. +The option ``use_e3sm_hdf5_netcdf = False`` indicates that you will not use +the E3SM default modules for HDF5 and NetCDF libraries (which are not available +for machines installed in the way described here). + +Finally, ``[discovery]`` allows you to add a ``hostname_contains`` that is used +to automatically identify your machine based on its hostname. If your machine +has multiple login nodes with different hostnames, hopefully, a string common +to all login nodes can be used here. If your machine has a unique hostname, +simply give that. This option saves developers from having to specify +``--machine `` each time they setup compass environments or test +cases. + + +Describing a Spack Environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The next step is to create a template YAML file that can be used to create +`Spack environments `_ +for your machine. Compass uses Spack environments to build packages that need +MPI support or which should be build for some other reason with system +compilers rather than coming from pre-built conda packages. Using a Spack +environment allows these packages to be built together in a consistent way that +is not guaranteed if you try to install dependencies one-by-one. In Spack +parlance, this is known as +`unified concretization `_. + +To do this, you will create a file ``conda/spack/__.yaml`` +similar to the following example for an Ubuntu laptop: + +.. code-block:: + + spack: + specs: + - gcc + - openmpi + {{ specs }} + concretizer: + unify: true + packages: + all: + compiler: [gcc@11.3.0] + providers: + mpi: [openmpi] + curl: + externals: + - spec: curl@7.81.0 + prefix: /usr + buildable: false + gcc: + externals: + - spec: gcc@11.3.0 + prefix: /usr + buildable: false + config: + install_missing_compilers: false + compilers: + - compiler: + spec: gcc@11.3.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: ubuntu22.04 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + + +Typically your system will already have compilers if nothing else, and this is +what we assume here. Give the appropriate path (replace ``/usr`` with the +appropriate path on your system). We have had better luck with ``gcc`` than +other compilers like Intel so far so for new supported machines so that's our +recommendation. Use ``gcc --version`` to determine the version and replace +``11.3.0`` with this number. + +Finally, you might need to update the ``target`` and ``operating_system``. +This is a bit of a "catch 22" in that you can use Spack to find this out but +compass is designed to clone and set up Spack for you so we assume you don't +have it yet. For now, make your best guess using the info on +`this page`_ +and correct it later if necessary. + +You may need to load a system module to get the compilers and potentially other +libraries such as MPI, HDF5, and NetCDF-C if you prefer to use system modules +rather than having Spack build them. If this is the case, the best way to do +this is to add a file +``conda/spack/__.sh`` along these lines: + +.. code-block:: bash + + module purge + module load perl/5.32.0-bsnc6lt + module load gcc/9.2.0-ugetvbp + module load openmpi/4.1.3-sxfyy4k + module load intel-mkl/2020.4.304-n3b5fye + module load hdf5/1.10.7-j3zxncu + module load netcdf-c/4.4.1-7ohuiwq + module load netcdf-fortran/4.4.4-k2zu3y5 + module load parallel-netcdf/1.11.0-mirrcz7 + +These modules will be loaded either before or after the spack environment, +depending on the ``modules_before`` and ``modules_after`` config options above. +You can also add modules in your YAML file but this shouldn't be necessary. + +For examples from various supported machines, compilers and MPI libraries, see the +`mache spack directory `_. + +Building the Spack Environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The next step is to try setting up compass and asking it to build the Spack +environment with a command something like: + +.. code-block:: bash + + ./conda/configure_compass_env.py --verbose --update_spack --conda -c gnu -i openmpi ... + +The ``--update_spack`` flag tells compass to create (or update) a Spack +environment. You can specify a directory for testing Spack with the +``--spack`` flag. You can specify a temporary directory for building spack +packages with ``--tmpdir`` (this directory must already exist). This is useful +if your ``/tmp`` space is small (Spack will use several GB of temporary space). + + +Creating the Spack environment may take anywhere from minutes to hours, +depending on your system. + +If dependencies don't build as expected, you may get an error message +suggesting that your ``operating_system`` or ``target`` aren't right. Here's +an example: + +.. code-block:: + + ==> Error: concretization failed for the following reasons: + + 1. zlib compiler '%gcc@9.4.0' incompatible with 'os=ubuntu20.04' + 2. readline compiler '%gcc@9.4.0' incompatible with 'os=ubuntu20.04' + 3. pkgconf compiler '%gcc@9.4.0' incompatible with 'os=ubuntu20.04' + +In this example, I had specified ``operating_system: ubuntu22.04`` in the YAML +file but in fact my operating system is ``ubuntu20.04`` as shown in the error +message. + +You can run: + +.. code-block:: bash + + source $SPACKDIR/share/spack/setup-env.sh + spack arch -o + spack arch -g + +where ``$SPACKDIR`` is the directory where the Spack repository was cloned +by compass (you should see ``Cloning into <$SPACKDIR>`` in the terminal, which +will hopefully help you find the right directory). This should hopefully give +you something close to what Spack wants. If you get something like +``x86_64_v4`` for the target, use ``x86_64`` instead. + +If you are getting other error messages, do your best to debug them but also +feel free to get in touch with the compass development team and we'll help if +we can. + +If you get everything working well, please feel free to make a pull request +into the compass main repo to add your supported machine. From c6de1b98ad712d87bdb243b15755774bc345b7f9 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 6 Jun 2023 07:40:56 -0600 Subject: [PATCH 12/20] Update spack package versions --- conda/default.cfg | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/conda/default.cfg b/conda/default.cfg index 3f730bdd0d..b9f3eefc47 100644 --- a/conda/default.cfg +++ b/conda/default.cfg @@ -24,12 +24,12 @@ albany = develop # cmake newer than 3.23.0 needed for Trilinos cmake = 3.23.0: esmf = 8.4.2 -hdf5 = 1.12.1 +hdf5 = 1.14.1 lapack = 3.9.1 -netcdf_c = 4.8.1 -netcdf_fortran = 4.5.3 -petsc = 3.16.1 -pnetcdf = 1.12.2 -# scorpio = 1.3.2 +netcdf_c = 4.9.2 +netcdf_fortran = 4.6.0 +petsc = 3.19.1 +pnetcdf = 1.12.3 +# scorpio = 1.4.1 scorpio = None parallelio = 2.5.10 From fc8c4f910f9a81fcac48c4a20be9c9ab7aae6151 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 6 Jun 2023 23:56:14 -0500 Subject: [PATCH 13/20] Update pyremap version This brings in pyremap 1.0.0, which supports specifying the NetCDF format and xarray output engine. --- conda/compass_env/spec-file.template | 2 +- conda/recipe/meta.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/compass_env/spec-file.template b/conda/compass_env/spec-file.template index d53a75c59a..552e45bf67 100644 --- a/conda/compass_env/spec-file.template +++ b/conda/compass_env/spec-file.template @@ -32,7 +32,7 @@ otps=2021.10 progressbar2 pyamg >=4.2.2 pyproj -pyremap>=0.0.13,<0.1.0 +pyremap>=1.0.0,<2.0.0 requests # having pip check problems with this version scikit-image !=0.20.0 diff --git a/conda/recipe/meta.yaml b/conda/recipe/meta.yaml index e8a9231f6f..29ca97cf7e 100644 --- a/conda/recipe/meta.yaml +++ b/conda/recipe/meta.yaml @@ -65,7 +65,7 @@ requirements: - progressbar2 - pyamg >=4.2.2 - pyproj - - pyremap >=0.0.13,<0.1.0 + - pyremap >=1.0.0,<2.0.0 - requests - scipy >=1.8.0 - shapely >=2.0,<3.0 From ebac565142b15efaea667db66345bb8691aaf18d Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 7 Jun 2023 15:38:34 -0700 Subject: [PATCH 14/20] Add ncview to the development environment --- conda/compass_env/spec-file.template | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conda/compass_env/spec-file.template b/conda/compass_env/spec-file.template index 552e45bf67..e325d18da0 100644 --- a/conda/compass_env/spec-file.template +++ b/conda/compass_env/spec-file.template @@ -72,3 +72,6 @@ m2r mock sphinx sphinx_rtd_theme + +# Visualization +ncview From 45c968bdc3c2dc92440b6290f219e61ab0b7f55d Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 19 Jun 2023 09:36:12 +0200 Subject: [PATCH 15/20] Update developer's guide --- docs/developers_guide/deploying_spack.rst | 8 ++++---- docs/developers_guide/quick_start.rst | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/developers_guide/deploying_spack.rst b/docs/developers_guide/deploying_spack.rst index 0e35e12c75..f417107324 100644 --- a/docs/developers_guide/deploying_spack.rst +++ b/docs/developers_guide/deploying_spack.rst @@ -285,7 +285,7 @@ Troubleshooting spack If you encounter an error like: .. code-block:: none - ==> spack env activate dev_compass_1_2_0-alpha_4_gnu_mpich + ==> spack env activate dev_compass_1_2_0-alpha_6_gnu_mpich ==> Error: Package 'armpl' not found. You may need to run 'spack clean -m'. @@ -295,7 +295,7 @@ get the ``spack`` command, e.g.: .. code-block:: bash - source ${PSCRATCH}/spack_test/spack_for_mache_1.12.0/share/spack/setup-env.sh + source ${PSCRATCH}/spack_test/dev_compass_1_2_0-alpha_6_gnu_mpich/share/spack/setup-env.sh Then run the suggested command: @@ -323,7 +323,7 @@ and serves as kind of a baseline for other testing. source load_dev_compass_1.2.0-alpha.5_chrysalis_intel_openmpi.sh git submodule update --init --recursive cd E3SM-Project/components/mpas-ocean - # this will build with scorpio and openmp + # this will build with PIO and OpenMP make ifort compass suite -s -c ocean -t pr -p . \ -w /lcrc/group/e3sm/ac.xylar/compass/test_20230202/ocean_pr_chrys_intel_openmpi @@ -392,7 +392,7 @@ branch. cd ../nonhydro_chrys_intel_openmpi git submodule update --init --recursive cd components/mpas-ocean - # this will build with scorpio, Netlib LAPACK and PETSc + # this will build with PIO, Netlib LAPACK and PETSc make ifort compass list | grep nonhydro # update these numbers for the 2 nonhydro test cases diff --git a/docs/developers_guide/quick_start.rst b/docs/developers_guide/quick_start.rst index 5bd20ef586..705508641e 100644 --- a/docs/developers_guide/quick_start.rst +++ b/docs/developers_guide/quick_start.rst @@ -181,8 +181,8 @@ you, this script will also: ``--spack`` flag can be used to point to a location for the spack repo to be checked out. Without this flag, a default location is used. Spack is used to build several libraries with system compilers and MPI library, including: - `SCORPIO `_ (parallel i/o for MPAS - components) `ESMF `_ (making mapping files + `ParallelIO `_ (parallel i/o in NetCDF + format) `ESMF `_ (making mapping files in parallel), `Trilinos `_, `Albany `_, `Netlib-LAPACK `_ and @@ -385,8 +385,8 @@ Creating/updating only the compass environment For some workflows (e.g. for MALI development with the Albany library when the MALI build environment has been created outside of ``compass``, for example, -on an unsupported machine), you may -only want to create the conda environment and not build SCORPIO, ESMF or +on an unsupported machine), you may only want to create the conda environment +and not build PIO, ESMF or other packages with system compilers, or include any system modules or environment variables in your activation script. In such cases, run with the ``--env_only`` flag: @@ -529,7 +529,7 @@ Code style for compass guidelines, the easiest way to stay in compliance as a developer writing new code or modifying existing code is to use a PEP8 style checker. One option is to use an IDE with a PEP8 style checker built in, such as -`PyCharm `_. See +`PyCharm `_. See `this tutorial `_ for some tips on checking code style in PyCharm. From 2f4289cf764e752949b8188b8843fdbf1016e4fe Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 29 Jun 2023 09:58:48 -0500 Subject: [PATCH 16/20] Switch back to SCORPIO Build ESMF without PNetCDF and with internal ParallelIO --- conda/bootstrap.py | 2 +- conda/default.cfg | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index 4bf55fcfc8..3eb9072691 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -417,7 +417,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901 f'"parallel-netcdf@{pnetcdf}+cxx+fortran"']) if esmf != 'None': - specs.append(f'"esmf@{esmf}+mpi+netcdf+pnetcdf+external-parallelio"') + specs.append(f'"esmf@{esmf}+mpi+netcdf~pnetcdf~external-parallelio"') if lapack != 'None': specs.append(f'"netlib-lapack@{lapack}"') include_e3sm_lapack = False diff --git a/conda/default.cfg b/conda/default.cfg index b9f3eefc47..580505bc8e 100644 --- a/conda/default.cfg +++ b/conda/default.cfg @@ -30,6 +30,6 @@ netcdf_c = 4.9.2 netcdf_fortran = 4.6.0 petsc = 3.19.1 pnetcdf = 1.12.3 -# scorpio = 1.4.1 -scorpio = None -parallelio = 2.5.10 +scorpio = 1.4.1 +# parallelio = 2.5.10 +parallelio = None From 3e166e6fd64b0101122bb4f3f1b97aa70246f597 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 21 Jul 2023 10:46:47 +0200 Subject: [PATCH 17/20] Add hack to make sure scorpio, not esmf, provides libpio*.a --- conda/bootstrap.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index 3eb9072691..858561b48d 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -426,10 +426,19 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901 if petsc != 'None': specs.append(f'"petsc@{petsc}+mpi+batch"') + custom_spack = '' if scorpio != 'None': specs.append( f'"scorpio' f'@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc"') + # make sure scorpio, not esmf, libraries are linked + lib_path = f'{spack_branch_base}/var/spack/environments/' \ + f'{spack_env}/.spack-env/view/lib' + scorpio_lib_path = '$(spack find --format "{prefix}" scorpio)' + custom_spack = \ + f'{custom_spack}' \ + f'ln -sfn {scorpio_lib_path}/lib/libpioc.a {lib_path}\n' \ + f'ln -sfn {scorpio_lib_path}/lib/libpiof.a {lib_path}\n' if parallelio != 'None': specs.append( @@ -459,7 +468,8 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901 machine=machine, config_file=machine_config, include_e3sm_lapack=include_e3sm_lapack, include_e3sm_hdf5_netcdf=e3sm_hdf5_netcdf, - yaml_template=yaml_template, tmpdir=tmpdir) + yaml_template=yaml_template, tmpdir=tmpdir, + custom_spack=custom_spack) # remove ESMC/ESMF include files that interfere with MPAS time keeping include_path = f'{spack_branch_base}/var/spack/environments/' \ From 83c80e52226cdc4cd93bae92640f4a798eb19fcb Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Sun, 23 Jul 2023 16:15:06 +0200 Subject: [PATCH 18/20] Update to mache 1.16.0 --- conda/compass_env/spec-file.template | 2 +- conda/configure_compass_env.py | 2 +- conda/recipe/meta.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conda/compass_env/spec-file.template b/conda/compass_env/spec-file.template index e325d18da0..0a3f4bbff0 100644 --- a/conda/compass_env/spec-file.template +++ b/conda/compass_env/spec-file.template @@ -18,7 +18,7 @@ jigsawpy=0.3.3 jupyter lxml {% if include_mache %} -mache=1.15.0 +mache=1.16.0 {% endif %} matplotlib-base metis diff --git a/conda/configure_compass_env.py b/conda/configure_compass_env.py index efb8933fdb..28bf66c368 100755 --- a/conda/configure_compass_env.py +++ b/conda/configure_compass_env.py @@ -101,7 +101,7 @@ def main(): if local_mache: mache = '' else: - mache = '"mache=1.15.0"' + mache = '"mache=1.16.0"' setup_install_env(env_name, activate_base, args.use_local, logger, args.recreate, conda_base, mache) diff --git a/conda/recipe/meta.yaml b/conda/recipe/meta.yaml index 29ca97cf7e..7fcc7f71dc 100644 --- a/conda/recipe/meta.yaml +++ b/conda/recipe/meta.yaml @@ -54,7 +54,7 @@ requirements: - jigsawpy 0.3.3 - jupyter - lxml - - mache 1.15.0 + - mache 1.16.0 - matplotlib-base - metis - mpas_tools 0.19.0 From 71e15327829bf7b049fbae599bf960a3b454c8d9 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 24 Jul 2023 11:14:59 +0200 Subject: [PATCH 19/20] Remove --with_netlib_lapack flag This is tedious to type and the suffix causes environment names that are too long (spack can't create a valid shebang). --- conda/albany_supported.txt | 1 + conda/bootstrap.py | 7 +------ conda/petsc_supported.txt | 1 + conda/shared.py | 8 ++------ docs/developers_guide/deploying_spack.rst | 18 +++++++++--------- docs/developers_guide/quick_start.rst | 8 ++++---- 6 files changed, 18 insertions(+), 25 deletions(-) diff --git a/conda/albany_supported.txt b/conda/albany_supported.txt index bd9ff79467..12ce2817af 100644 --- a/conda/albany_supported.txt +++ b/conda/albany_supported.txt @@ -3,3 +3,4 @@ chicoma-cpu, gnu, mpich chrysalis, gnu, openmpi pm-cpu, gnu, mpich +morpheus, gnu, openmpi diff --git a/conda/bootstrap.py b/conda/bootstrap.py index 858561b48d..6e38a76257 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -185,11 +185,6 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path, else: config.set('deploy', 'albany', 'None') - if args.with_netlib_lapack: - lib_suffix = f'{lib_suffix}_netlib_lapack' - else: - config.set('deploy', 'lapack', 'None') - if args.with_petsc: lib_suffix = f'{lib_suffix}_petsc' log_message( @@ -198,6 +193,7 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path, args.without_openmp = True else: config.set('deploy', 'petsc', 'None') + config.set('deploy', 'lapack', 'None') activ_suffix = f'{activ_suffix}{lib_suffix}' @@ -1039,7 +1035,6 @@ def main(): # noqa: C901 check_env(script_filename, conda_env_name, logger) if env_type == 'release' and not (args.with_albany or - args.with_netlib_lapack or args.with_petsc): # make a symlink to the activation script link = os.path.join(activ_path, diff --git a/conda/petsc_supported.txt b/conda/petsc_supported.txt index 5bc50ab6ab..8aff04a7d0 100644 --- a/conda/petsc_supported.txt +++ b/conda/petsc_supported.txt @@ -7,4 +7,5 @@ chicoma-cpu, gnu, mpich chrysalis, intel, openmpi chrysalis, gnu, openmpi compy, intel, impi +morpheus, gnu, openmpi pm-cpu, gnu, mpich diff --git a/conda/shared.py b/conda/shared.py index c2c5358ed4..308ca24d6f 100644 --- a/conda/shared.py +++ b/conda/shared.py @@ -56,14 +56,10 @@ def parse_args(bootstrap): action='store_true', help="Whether to include albany in the spack " "environment") - parser.add_argument("--with_netlib_lapack", dest="with_netlib_lapack", - action='store_true', - help="Whether to include Netlib-LAPACK in the spack " - "environment") parser.add_argument("--with_petsc", dest="with_petsc", action='store_true', - help="Whether to include PETSc in the spack " - "environment") + help="Whether to include PETSc and Netlib-LAPACK in " + "the spack environment") parser.add_argument("--without_openmp", dest="without_openmp", action='store_true', help="If this flag is included, OPENMP=false will " diff --git a/docs/developers_guide/deploying_spack.rst b/docs/developers_guide/deploying_spack.rst index f417107324..1ea685a3e6 100644 --- a/docs/developers_guide/deploying_spack.rst +++ b/docs/developers_guide/deploying_spack.rst @@ -229,8 +229,8 @@ discussed below. Testing spack with PETSc (and Netlib LAPACK) -------------------------------------------- -If you want to build PETSc (and Netlib LAPACK), use the ``--with_petsc`` and -``--with_netlib_lapack`` flags. Currently, this only works with some +If you want to build PETSc (and Netlib LAPACK), use the ``--with_petsc`` flag. +Currently, this only works with some compilers, but that may be more that I was trying to limit the amount of work for the compass support team. There is a file, `petsc_supported.txt `_, @@ -249,10 +249,10 @@ Here is an example: --spack /lcrc/group/e3sm/${USER}/spack_test \ --tmpdir ${TMPDIR} \ --compiler intel gnu \ - --mpi openmpi openmpi \ - --with_netlib_lapack \ + --mpi openmpi \ --with_petsc \ - --recreate + --recreate \ + --verbose Testing spack with Albany ------------------------- @@ -383,7 +383,7 @@ branch. .. code-block:: bash # source whichever load script is appropriate - source load_dev_compass_1.2.0-alpha.5_chrysalis_intel_openmpi_netlib_lapack_petsc.sh + source load_dev_compass_1.2.0-alpha.5_chrysalis_intel_openmpi_petsc.sh git submodule update --init cd E3SM-Project git remote add scalandr/E3SM git@github.com:scalandr/E3SM.git @@ -473,7 +473,7 @@ Deploying spack with PETSc (and Netlib LAPACK) --update_spack \ --tmpdir ${TMPDIR} \ --compiler intel gnu \ - --mpi openmpi openmpi \ - --with_netlib_lapack \ + --mpi openmpi \ --with_petsc \ - --recreate + --recreate \ + --verbose diff --git a/docs/developers_guide/quick_start.rst b/docs/developers_guide/quick_start.rst index 705508641e..c7cded691e 100644 --- a/docs/developers_guide/quick_start.rst +++ b/docs/developers_guide/quick_start.rst @@ -89,7 +89,7 @@ If you are on one of the :ref:`dev_supported_machines`, run: ./conda/configure_compass_env.py --conda \ -c [--mpi ] [-m ] [--with_albany] \ - [--with_netlib_lapack] [--with_petsc] + [--with_petsc] The ```` is typically ``~/mambaforge``. This is the location where you would like to install Mambaforge or where it is @@ -125,8 +125,8 @@ will be linked in to MPAS-Ocean. Environments with PETSc and Netlib-LAPACK ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you are working with MPAS-Ocean test cases that need PETSC and -Netlib-LAPACK, you should specify ``--with_petsc --with_netlib_lapack`` to +If you are working with MPAS-Ocean test cases that need PETSc and +Netlib-LAPACK, you should specify ``--with_petsc`` to point to Spack environments where these libraries are included. Appropriate environment variables for pointing to these libraries will be build into the resulting load script (see below). @@ -192,7 +192,7 @@ you, this script will also: * with the ``--with_albany`` flag, creates or uses an existing Spack environment that includes Albany and Trilinos. -* with the ``--with_petsc --with_netlib_lapack`` flags, creates or uses an +* with the ``--with_petsc`` flag, creates or uses an existing Spack environment that includes PETSc and Netlib-LAPACK. * make an activation script called ``load_*.sh``, where the details of the From 625485cb7c65ccbc3d34ac1f6c695d21aca889f2 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Mon, 24 Jul 2023 18:34:41 +0200 Subject: [PATCH 20/20] Remove Intel-MPI from supported configs on Chrysalis Also remove Gnu from Compy in the docs (already marked as unsuppored). --- conda/unsupported.txt | 4 ++++ docs/developers_guide/machines/index.rst | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda/unsupported.txt b/conda/unsupported.txt index b5b897d69f..9103260d04 100644 --- a/conda/unsupported.txt +++ b/conda/unsupported.txt @@ -20,3 +20,7 @@ pm-cpu, amdclang, mpich # compiles but tests unreliable (errors or hanging), # see https://github.com/MPAS-Dev/compass/issues/336 anvil, intel, mvapich + + +# can't build ESMF +chrysalis, intel, impi diff --git a/docs/developers_guide/machines/index.rst b/docs/developers_guide/machines/index.rst index d24e6332ef..cee7ac556a 100644 --- a/docs/developers_guide/machines/index.rst +++ b/docs/developers_guide/machines/index.rst @@ -54,14 +54,10 @@ the MPAS model. | chicoma-cpu | gnu | mpich | gnu-cray | +--------------+------------+-----------+-------------------+ | chrysalis | intel | openmpi | ifort | -| | +-----------+-------------------+ -| | | impi | intel-mpi | | +------------+-----------+-------------------+ | | gnu | openmpi | gfortran | +--------------+------------+-----------+-------------------+ | compy | intel | impi | intel-mpi | -| +------------+-----------+-------------------+ -| | gnu | openmpi | gfortran | +--------------+------------+-----------+-------------------+ | pm-cpu | gnu | mpich | gnu-cray | +--------------+------------+-----------+-------------------+