diff --git a/compass/machines/__init__.py b/compass/machines/__init__.py index e69de29bb2..688ad26c74 100644 --- a/compass/machines/__init__.py +++ b/compass/machines/__init__.py @@ -0,0 +1,54 @@ +import os +import socket + +try: + from importlib.resources import contents as imp_res_contents +except ImportError: + # python<=3.8 + from importlib_resources import contents as imp_res_contents + +from mache import discover_machine as mache_discover_machine + +from compass.config import CompassConfigParser + + +def discover_machine(quiet=False): + """ + Figure out the machine from the host name + + Parameters + ---------- + quiet : bool, optional + Whether to print warnings if the machine name is ambiguous + + Returns + ------- + machine : str + The name of the current machine + """ + machine = mache_discover_machine(quiet=quiet) + if machine is None: + possible_hosts = _get_possible_hosts() + hostname = socket.gethostname() + for possible_machine, hostname_contains in possible_hosts.items(): + if hostname_contains in hostname: + machine = possible_machine + break + return machine + + +def _get_possible_hosts(): + machine_contents = imp_res_contents('compass.machines') + possible_hosts = dict() + for filename in machine_contents: + if filename.endswith('.cfg'): + machine = os.path.split(filename)[0] + config = CompassConfigParser() + config.add_from_package('compass.machines', filename) + if config.has_section('discovery') and \ + config.has_option('discovery', 'hostname_contains'): + hostname_contains = config.get('discovery', + 'hostname_contains') + possible_hosts[machine] = hostname_contains + + return possible_hosts diff --git a/compass/machines/eligos.cfg b/compass/machines/eligos.cfg new file mode 100644 index 0000000000..aa1a52bdaa --- /dev/null +++ b/compass/machines/eligos.cfg @@ -0,0 +1,60 @@ +# The parallel section describes options related to running jobs in parallel +[parallel] + +# parallel system of execution: slurm, cobalt or single_node +system = single_node + +# whether to use mpirun or srun to run a task +parallel_executable = mpirun + +# cores per node on the machine +cores_per_node = 4 + + +# Config options related to spack environments +[spack] + +# whether to load modules from the spack yaml file before loading the spack +# environment +modules_before = False + +# whether to load modules from the spack yaml file after loading the spack +# environment +modules_after = False + + +# The paths section describes paths that are used within the ocean core test +# cases. +[paths] + +# A shared root directory where MPAS standalone data can be found +database_root = /home/xylar/data/mpas/mpas_standalonedata + +# the path to the base conda environment where compass environments have +# been created +compass_envs = /home/xylar/data/mpas/compass_envs + + +# Options related to deploying a compass conda environment on supported +# machines +[deploy] + +# the compiler set to use for system libraries and MPAS builds +compiler = gnu + +# the system MPI library to use for gnu compiler +mpi_gnu = openmpi + +# the base path for spack environments used by compass +spack = /home/xylar/data/mpas/spack + +# whether to use the same modules for hdf5, netcdf-c, netcdf-fortran and +# pnetcdf as E3SM (spack modules are used otherwise) +use_e3sm_hdf5_netcdf = False + + +# Options related to machine discovery +[discovery] + +# a substring used to identify this machine from its hostname +hostname_contains = eligos diff --git a/compass/machines/morpheus.cfg b/compass/machines/morpheus.cfg new file mode 100644 index 0000000000..f44918fffa --- /dev/null +++ b/compass/machines/morpheus.cfg @@ -0,0 +1,60 @@ +# The parallel section describes options related to running jobs in parallel +[parallel] + +# parallel system of execution: slurm, cobalt or single_node +system = single_node + +# whether to use mpirun or srun to run a task +parallel_executable = mpirun + +# cores per node on the machine +cores_per_node = 8 + + +# Config options related to spack environments +[spack] + +# whether to load modules from the spack yaml file before loading the spack +# environment +modules_before = False + +# whether to load modules from the spack yaml file after loading the spack +# environment +modules_after = False + + +# The paths section describes paths that are used within the ocean core test +# cases. +[paths] + +# A shared root directory where MPAS standalone data can be found +database_root = /home/xylar/data/mpas/mpas_standalonedata + +# the path to the base conda environment where compass environments have +# been created +compass_envs = /home/xylar/data/mpas/compass_envs + + +# Options related to deploying a compass conda environment on supported +# machines +[deploy] + +# the compiler set to use for system libraries and MPAS builds +compiler = gnu + +# the system MPI library to use for gnu compiler +mpi_gnu = openmpi + +# the base path for spack environments used by compass +spack = /home/xylar/data/mpas/spack + +# whether to use the same modules for hdf5, netcdf-c, netcdf-fortran and +# pnetcdf as E3SM (spack modules are used otherwise) +use_e3sm_hdf5_netcdf = False + + +# Options related to machine discovery +[discovery] + +# a substring used to identify this machine from its hostname +hostname_contains = morpheus diff --git a/compass/setup.py b/compass/setup.py index a0c2cb6145..70fa193bc1 100644 --- a/compass/setup.py +++ b/compass/setup.py @@ -4,12 +4,11 @@ import sys import warnings -from mache import discover_machine - from compass import provenance from compass.config import CompassConfigParser from compass.io import symlink from compass.job import write_job_script +from compass.machines import discover_machine from compass.mpas_cores import get_mpas_cores @@ -432,7 +431,8 @@ def _get_basic_config(config_file, machine, mpas_model_path, mpas_core): # add the E3SM config options from mache if machine is not None: - config.add_from_package('mache.machines', f'{machine}.cfg') + config.add_from_package('mache.machines', f'{machine}.cfg', + exception=False) # add the compass machine config file if machine is None: diff --git a/compass/version.py b/compass/version.py index cbbfb1a865..447b7d027d 100644 --- a/compass/version.py +++ b/compass/version.py @@ -1 +1 @@ -__version__ = '1.2.0-alpha.5' +__version__ = '1.2.0-alpha.6' diff --git a/conda/albany_supported.txt b/conda/albany_supported.txt index bd9ff79467..12ce2817af 100644 --- a/conda/albany_supported.txt +++ b/conda/albany_supported.txt @@ -3,3 +3,4 @@ chicoma-cpu, gnu, mpich chrysalis, gnu, openmpi pm-cpu, gnu, mpich +morpheus, gnu, openmpi diff --git a/conda/bootstrap.py b/conda/bootstrap.py index 57bc1e3fa2..6e38a76257 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -6,15 +6,16 @@ import os import platform import shutil +import socket import stat import subprocess from configparser import ConfigParser import progressbar from jinja2 import Template -from mache import MachineInfo, discover_machine +from mache import MachineInfo +from mache import discover_machine as mache_discover_machine from mache.spack import get_spack_script, make_spack_env -from mache.version import __version__ as mache_version from shared import ( check_call, get_conda_base, @@ -184,11 +185,6 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path, else: config.set('deploy', 'albany', 'None') - if args.with_netlib_lapack: - lib_suffix = f'{lib_suffix}_netlib_lapack' - else: - config.set('deploy', 'lapack', 'None') - if args.with_petsc: lib_suffix = f'{lib_suffix}_petsc' log_message( @@ -197,6 +193,7 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path, args.without_openmp = True else: config.set('deploy', 'petsc', 'None') + config.set('deploy', 'lapack', 'None') activ_suffix = f'{activ_suffix}{lib_suffix}' @@ -384,8 +381,9 @@ def get_env_vars(machine, compiler, mpilib): return env_vars -def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, - spack_base, spack_template_path, env_vars, tmpdir, logger): +def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901 + spack_env, spack_base, spack_template_path, env_vars, + tmpdir, logger): albany = config.get('deploy', 'albany') cmake = config.get('deploy', 'cmake') @@ -393,13 +391,14 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, lapack = config.get('deploy', 'lapack') petsc = config.get('deploy', 'petsc') scorpio = config.get('deploy', 'scorpio') + parallelio = config.get('deploy', 'parallelio') - spack_branch_base = f'{spack_base}/spack_for_mache_{mache_version}' + spack_branch_base = f'{spack_base}/{spack_env}' specs = list() if cmake != 'None': - specs.append(f'cmake "@{cmake}"') + specs.append(f'"cmake@{cmake}"') e3sm_hdf5_netcdf = config.getboolean('deploy', 'use_e3sm_hdf5_netcdf') if not e3sm_hdf5_netcdf: @@ -408,42 +407,65 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, netcdf_fortran = config.get('deploy', 'netcdf_fortran') pnetcdf = config.get('deploy', 'pnetcdf') specs.extend([ - f'hdf5 "@{hdf5}+cxx+fortran+hl+mpi+shared"', - f'netcdf-c "@{netcdf_c}+mpi~parallel-netcdf"', - f'netcdf-fortran "@{netcdf_fortran}"', - f'parallel-netcdf "@{pnetcdf}+cxx+fortran"']) + f'"hdf5@{hdf5}+cxx+fortran+hl+mpi+shared"', + f'"netcdf-c@{netcdf_c}+mpi~parallel-netcdf"', + f'"netcdf-fortran@{netcdf_fortran}"', + f'"parallel-netcdf@{pnetcdf}+cxx+fortran"']) if esmf != 'None': - specs.append(f'esmf "@{esmf}+mpi+netcdf~pio+pnetcdf"') + specs.append(f'"esmf@{esmf}+mpi+netcdf~pnetcdf~external-parallelio"') if lapack != 'None': - specs.append(f'netlib-lapack "@{lapack}"') + specs.append(f'"netlib-lapack@{lapack}"') include_e3sm_lapack = False else: include_e3sm_lapack = True if petsc != 'None': - specs.append(f'petsc "@{petsc}+mpi+batch"') + specs.append(f'"petsc@{petsc}+mpi+batch"') + custom_spack = '' if scorpio != 'None': specs.append( - f'scorpio ' - f'"@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc"') + f'"scorpio' + f'@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc"') + # make sure scorpio, not esmf, libraries are linked + lib_path = f'{spack_branch_base}/var/spack/environments/' \ + f'{spack_env}/.spack-env/view/lib' + scorpio_lib_path = '$(spack find --format "{prefix}" scorpio)' + custom_spack = \ + f'{custom_spack}' \ + f'ln -sfn {scorpio_lib_path}/lib/libpioc.a {lib_path}\n' \ + f'ln -sfn {scorpio_lib_path}/lib/libpiof.a {lib_path}\n' + + if parallelio != 'None': + specs.append( + f'"parallelio' + f'@{parallelio}+pnetcdf~timing"') if albany != 'None': - specs.append(f'albany "@{albany}+mpas"') + specs.append(f'"albany@{albany}+mpas"') yaml_template = f'{spack_template_path}/{machine}_{compiler}_{mpi}.yaml' if not os.path.exists(yaml_template): yaml_template = None + + if machine is not None: + here = os.path.abspath(os.path.dirname(__file__)) + machine_config = os.path.join(here, '..', 'compass', 'machines', + f'{machine}.cfg') + else: + machine_config = None + if update_spack: home_dir = os.path.expanduser('~') log_message(logger, 'Removing ~/.spack for safety') safe_rmtree(os.path.join(home_dir, '.spack')) make_spack_env(spack_path=spack_branch_base, env_name=spack_env, spack_specs=specs, compiler=compiler, mpi=mpi, - machine=machine, + machine=machine, config_file=machine_config, include_e3sm_lapack=include_e3sm_lapack, include_e3sm_hdf5_netcdf=e3sm_hdf5_netcdf, - yaml_template=yaml_template, tmpdir=tmpdir) + yaml_template=yaml_template, tmpdir=tmpdir, + custom_spack=custom_spack) # remove ESMC/ESMF include files that interfere with MPAS time keeping include_path = f'{spack_branch_base}/var/spack/environments/' \ @@ -456,7 +478,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, spack_script = get_spack_script( spack_path=spack_branch_base, env_name=spack_env, compiler=compiler, - mpi=mpi, shell='sh', machine=machine, + mpi=mpi, shell='sh', machine=machine, config_file=machine_config, include_e3sm_lapack=include_e3sm_lapack, include_e3sm_hdf5_netcdf=e3sm_hdf5_netcdf, yaml_template=yaml_template) @@ -465,6 +487,10 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, f'{spack_env}/.spack-env/view' env_vars = f'{env_vars}' \ f'export PIO={spack_view}\n' + if parallelio != 'None': + env_vars = \ + f'{env_vars}' \ + f'export HAVE_ADIOS=false\n' if albany != 'None': albany_flag_filename = f'{spack_view}/export_albany.in' if not os.path.exists(albany_flag_filename): @@ -806,6 +832,50 @@ def safe_rmtree(path): pass +def discover_machine(quiet=False): + """ + Figure out the machine from the host name + + Parameters + ---------- + quiet : bool, optional + Whether to print warnings if the machine name is ambiguous + + Returns + ------- + machine : str + The name of the current machine + """ + machine = mache_discover_machine(quiet=quiet) + if machine is None: + possible_hosts = get_possible_hosts() + hostname = socket.gethostname() + for possible_machine, hostname_contains in possible_hosts.items(): + if hostname_contains in hostname: + machine = possible_machine + break + return machine + + +def get_possible_hosts(): + here = os.path.abspath(os.path.dirname(__file__)) + files = sorted(glob.glob(os.path.join( + here, '..', 'compass', 'machines', '*.cfg'))) + + possible_hosts = dict() + for filename in files: + machine = os.path.splitext(os.path.split(filename)[1])[0] + config = ConfigParser() + config.read(filename) + if config.has_section('discovery') and \ + config.has_option('discovery', 'hostname_contains'): + hostname_contains = config.get('discovery', + 'hostname_contains') + possible_hosts[machine] = hostname_contains + + return possible_hosts + + def main(): # noqa: C901 args = parse_args(bootstrap=True) @@ -830,7 +900,7 @@ def main(): # noqa: C901 else: machine = args.machine - e3sm_machine = machine is not None + known_machine = machine is not None if machine is None and not args.env_only: if platform.system() == 'Linux': @@ -895,7 +965,7 @@ def main(): # noqa: C901 if args.spack_base is not None: spack_base = args.spack_base - elif e3sm_machine and compiler is not None: + elif known_machine and compiler is not None: spack_base = get_spack_base(args.spack_base, config) else: spack_base = None @@ -965,7 +1035,6 @@ def main(): # noqa: C901 check_env(script_filename, conda_env_name, logger) if env_type == 'release' and not (args.with_albany or - args.with_netlib_lapack or args.with_petsc): # make a symlink to the activation script link = os.path.join(activ_path, diff --git a/conda/compass_env/spec-file.template b/conda/compass_env/spec-file.template index 8b48c90add..0a3f4bbff0 100644 --- a/conda/compass_env/spec-file.template +++ b/conda/compass_env/spec-file.template @@ -18,7 +18,7 @@ jigsawpy=0.3.3 jupyter lxml {% if include_mache %} -mache=1.14.0 +mache=1.16.0 {% endif %} matplotlib-base metis @@ -32,7 +32,7 @@ otps=2021.10 progressbar2 pyamg >=4.2.2 pyproj -pyremap>=0.0.13,<0.1.0 +pyremap>=1.0.0,<2.0.0 requests # having pip check problems with this version scikit-image !=0.20.0 @@ -72,3 +72,6 @@ m2r mock sphinx sphinx_rtd_theme + +# Visualization +ncview diff --git a/conda/configure_compass_env.py b/conda/configure_compass_env.py index cff86bbbeb..28bf66c368 100755 --- a/conda/configure_compass_env.py +++ b/conda/configure_compass_env.py @@ -101,7 +101,7 @@ def main(): if local_mache: mache = '' else: - mache = '"mache=1.14.0"' + mache = '"mache=1.16.0"' setup_install_env(env_name, activate_base, args.use_local, logger, args.recreate, conda_base, mache) diff --git a/conda/default.cfg b/conda/default.cfg index b8cf19a440..580505bc8e 100644 --- a/conda/default.cfg +++ b/conda/default.cfg @@ -23,11 +23,13 @@ mpi = nompi albany = develop # cmake newer than 3.23.0 needed for Trilinos cmake = 3.23.0: -esmf = 8.2.0 -hdf5 = 1.12.1 +esmf = 8.4.2 +hdf5 = 1.14.1 lapack = 3.9.1 -netcdf_c = 4.8.1 -netcdf_fortran = 4.5.3 -petsc = 3.16.1 -pnetcdf = 1.12.2 -scorpio = 1.3.2 +netcdf_c = 4.9.2 +netcdf_fortran = 4.6.0 +petsc = 3.19.1 +pnetcdf = 1.12.3 +scorpio = 1.4.1 +# parallelio = 2.5.10 +parallelio = None diff --git a/conda/petsc_supported.txt b/conda/petsc_supported.txt index 5bc50ab6ab..8aff04a7d0 100644 --- a/conda/petsc_supported.txt +++ b/conda/petsc_supported.txt @@ -7,4 +7,5 @@ chicoma-cpu, gnu, mpich chrysalis, intel, openmpi chrysalis, gnu, openmpi compy, intel, impi +morpheus, gnu, openmpi pm-cpu, gnu, mpich diff --git a/conda/recipe/meta.yaml b/conda/recipe/meta.yaml index 58f0011773..7fcc7f71dc 100644 --- a/conda/recipe/meta.yaml +++ b/conda/recipe/meta.yaml @@ -1,5 +1,5 @@ {% set name = "compass" %} -{% set version = "1.2.0alpha.5" %} +{% set version = "1.2.0alpha.6" %} {% set build = 0 %} {% if mpi == "nompi" %} @@ -54,7 +54,7 @@ requirements: - jigsawpy 0.3.3 - jupyter - lxml - - mache 1.14.0 + - mache 1.16.0 - matplotlib-base - metis - mpas_tools 0.19.0 @@ -65,7 +65,7 @@ requirements: - progressbar2 - pyamg >=4.2.2 - pyproj - - pyremap >=0.0.13,<0.1.0 + - pyremap >=1.0.0,<2.0.0 - requests - scipy >=1.8.0 - shapely >=2.0,<3.0 diff --git a/conda/shared.py b/conda/shared.py index c2c5358ed4..308ca24d6f 100644 --- a/conda/shared.py +++ b/conda/shared.py @@ -56,14 +56,10 @@ def parse_args(bootstrap): action='store_true', help="Whether to include albany in the spack " "environment") - parser.add_argument("--with_netlib_lapack", dest="with_netlib_lapack", - action='store_true', - help="Whether to include Netlib-LAPACK in the spack " - "environment") parser.add_argument("--with_petsc", dest="with_petsc", action='store_true', - help="Whether to include PETSc in the spack " - "environment") + help="Whether to include PETSc and Netlib-LAPACK in " + "the spack environment") parser.add_argument("--without_openmp", dest="without_openmp", action='store_true', help="If this flag is included, OPENMP=false will " diff --git a/conda/spack/eligos_gnu_openmpi.yaml b/conda/spack/eligos_gnu_openmpi.yaml new file mode 100644 index 0000000000..355346e0ba --- /dev/null +++ b/conda/spack/eligos_gnu_openmpi.yaml @@ -0,0 +1,38 @@ +spack: + specs: + - gcc + - openmpi +{{ specs }} + concretizer: + unify: true + packages: + all: + compiler: [gcc@9.4.0] + providers: + mpi: [openmpi] + curl: + externals: + - spec: curl@7.68.0 + prefix: /usr + buildable: false + gcc: + externals: + - spec: gcc@9.4.0 + prefix: /usr + buildable: false + config: + install_missing_compilers: false + compilers: + - compiler: + spec: gcc@9.4.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: ubuntu20.04 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] diff --git a/conda/spack/morpheus_gnu_openmpi.yaml b/conda/spack/morpheus_gnu_openmpi.yaml new file mode 100644 index 0000000000..6e5a4aa8d7 --- /dev/null +++ b/conda/spack/morpheus_gnu_openmpi.yaml @@ -0,0 +1,38 @@ +spack: + specs: + - gcc + - openmpi +{{ specs }} + concretizer: + unify: true + packages: + all: + compiler: [gcc@11.3.0] + providers: + mpi: [openmpi] + curl: + externals: + - spec: curl@7.81.0 + prefix: /usr + buildable: false + gcc: + externals: + - spec: gcc@11.3.0 + prefix: /usr + buildable: false + config: + install_missing_compilers: false + compilers: + - compiler: + spec: gcc@11.3.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: ubuntu22.04 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] diff --git a/conda/unsupported.txt b/conda/unsupported.txt index b5b897d69f..9103260d04 100644 --- a/conda/unsupported.txt +++ b/conda/unsupported.txt @@ -20,3 +20,7 @@ pm-cpu, amdclang, mpich # compiles but tests unreliable (errors or hanging), # see https://github.com/MPAS-Dev/compass/issues/336 anvil, intel, mvapich + + +# can't build ESMF +chrysalis, intel, impi diff --git a/docs/developers_guide/deploying_spack.rst b/docs/developers_guide/deploying_spack.rst index 0e35e12c75..1ea685a3e6 100644 --- a/docs/developers_guide/deploying_spack.rst +++ b/docs/developers_guide/deploying_spack.rst @@ -229,8 +229,8 @@ discussed below. Testing spack with PETSc (and Netlib LAPACK) -------------------------------------------- -If you want to build PETSc (and Netlib LAPACK), use the ``--with_petsc`` and -``--with_netlib_lapack`` flags. Currently, this only works with some +If you want to build PETSc (and Netlib LAPACK), use the ``--with_petsc`` flag. +Currently, this only works with some compilers, but that may be more that I was trying to limit the amount of work for the compass support team. There is a file, `petsc_supported.txt `_, @@ -249,10 +249,10 @@ Here is an example: --spack /lcrc/group/e3sm/${USER}/spack_test \ --tmpdir ${TMPDIR} \ --compiler intel gnu \ - --mpi openmpi openmpi \ - --with_netlib_lapack \ + --mpi openmpi \ --with_petsc \ - --recreate + --recreate \ + --verbose Testing spack with Albany ------------------------- @@ -285,7 +285,7 @@ Troubleshooting spack If you encounter an error like: .. code-block:: none - ==> spack env activate dev_compass_1_2_0-alpha_4_gnu_mpich + ==> spack env activate dev_compass_1_2_0-alpha_6_gnu_mpich ==> Error: Package 'armpl' not found. You may need to run 'spack clean -m'. @@ -295,7 +295,7 @@ get the ``spack`` command, e.g.: .. code-block:: bash - source ${PSCRATCH}/spack_test/spack_for_mache_1.12.0/share/spack/setup-env.sh + source ${PSCRATCH}/spack_test/dev_compass_1_2_0-alpha_6_gnu_mpich/share/spack/setup-env.sh Then run the suggested command: @@ -323,7 +323,7 @@ and serves as kind of a baseline for other testing. source load_dev_compass_1.2.0-alpha.5_chrysalis_intel_openmpi.sh git submodule update --init --recursive cd E3SM-Project/components/mpas-ocean - # this will build with scorpio and openmp + # this will build with PIO and OpenMP make ifort compass suite -s -c ocean -t pr -p . \ -w /lcrc/group/e3sm/ac.xylar/compass/test_20230202/ocean_pr_chrys_intel_openmpi @@ -383,7 +383,7 @@ branch. .. code-block:: bash # source whichever load script is appropriate - source load_dev_compass_1.2.0-alpha.5_chrysalis_intel_openmpi_netlib_lapack_petsc.sh + source load_dev_compass_1.2.0-alpha.5_chrysalis_intel_openmpi_petsc.sh git submodule update --init cd E3SM-Project git remote add scalandr/E3SM git@github.com:scalandr/E3SM.git @@ -392,7 +392,7 @@ branch. cd ../nonhydro_chrys_intel_openmpi git submodule update --init --recursive cd components/mpas-ocean - # this will build with scorpio, Netlib LAPACK and PETSc + # this will build with PIO, Netlib LAPACK and PETSc make ifort compass list | grep nonhydro # update these numbers for the 2 nonhydro test cases @@ -473,7 +473,7 @@ Deploying spack with PETSc (and Netlib LAPACK) --update_spack \ --tmpdir ${TMPDIR} \ --compiler intel gnu \ - --mpi openmpi openmpi \ - --with_netlib_lapack \ + --mpi openmpi \ --with_petsc \ - --recreate + --recreate \ + --verbose diff --git a/docs/developers_guide/machines/index.rst b/docs/developers_guide/machines/index.rst index 7cb96e1c1f..cee7ac556a 100644 --- a/docs/developers_guide/machines/index.rst +++ b/docs/developers_guide/machines/index.rst @@ -54,14 +54,10 @@ the MPAS model. | chicoma-cpu | gnu | mpich | gnu-cray | +--------------+------------+-----------+-------------------+ | chrysalis | intel | openmpi | ifort | -| | +-----------+-------------------+ -| | | impi | intel-mpi | | +------------+-----------+-------------------+ | | gnu | openmpi | gfortran | +--------------+------------+-----------+-------------------+ | compy | intel | impi | intel-mpi | -| +------------+-----------+-------------------+ -| | gnu | openmpi | gfortran | +--------------+------------+-----------+-------------------+ | pm-cpu | gnu | mpich | gnu-cray | +--------------+------------+-----------+-------------------+ @@ -117,3 +113,285 @@ Under OSX, you can build the MPAS model with .. code-block:: bash make gfortran-clang + +.. _dev_add_supported_machine: + +Adding a New Supported Machine +------------------------------ + +If you want to add a new supported machine, you need to add both a config file +and a yaml file describing your machine, as detailed below. + +Adding a Machine Config File +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The first step in adding a new supported machine to add a config file in +``compass/machines``. The config file needs to describe the parallel +environment and some paths where shared Spack environments will be installed +and shared data will be downloaded. The easiest place to start is one of the +examples provided (machines ``morpheus`` and ``eligos`` for now, but more +will be added soon.) + +.. code-block:: cfg + + # The parallel section describes options related to running jobs in parallel + [parallel] + + # parallel system of execution: slurm, cobalt or single_node + system = single_node + + # whether to use mpirun or srun to run a task + parallel_executable = mpirun + + # cores per node on the machine + cores_per_node = 8 + + + # Config options related to spack environments + [spack] + + # whether to load modules from the spack yaml file before loading the spack + # environment + modules_before = False + + # whether to load modules from the spack yaml file after loading the spack + # environment + modules_after = False + + + # The paths section describes paths that are used within the ocean core test + # cases. + [paths] + + # A shared root directory where MPAS standalone data can be found + database_root = /home/xylar/data/mpas/mpas_standalonedata + + # the path to the base conda environment where compass environments have + # been created + compass_envs = /home/xylar/data/mpas/compass_envs + + + # Options related to deploying a compass conda environment on supported + # machines + [deploy] + + # the compiler set to use for system libraries and MPAS builds + compiler = gnu + + # the system MPI library to use for gnu compiler + mpi_gnu = openmpi + + # the base path for spack environments used by compass + spack = /home/xylar/data/mpas/spack + + # whether to use the same modules for hdf5, netcdf-c, netcdf-fortran and + # pnetcdf as E3SM (spack modules are used otherwise) + use_e3sm_hdf5_netcdf = False + + + # Options related to machine discovery + [discovery] + + # a substring used to identify this machine from its hostname + hostname_contains = morpheus + + +The ``[parallel]`` section should describe the type of parallel queuing +system (currently only ``slurm`` or ``single_node`` are supported), the number +of cores per node and the command for running an MPI executable (typically +``srun`` for Slurm and ``mpirun`` for a "single node" machine like a laptop or +workstation. + +The ``[spack]`` section has some config options to do with loading system +modules before or after loading a Spack environment. On a "single node" +machine, you typically don't have modules so both ``modules_before`` and +``modules_after`` can be set to ``False``. On a high-performance computing +(HPC) machine, you may find it is safest to load modules after the Spack +environment to ensure that certain paths and environment variables are set the +way the modules have them, rather than the way that Spack would have them. +The recommended starting point would be ``modules_before = False`` and +``modules_after = True``, but could be adjusted as needed if the right shared +libraries aren't being found when you try to build an MPAS component. + +In the ``[paths]`` section, you will first give a path where you would like +to store shared data files used in compass test cases in ``database_root``. +Compass will create this directory if it doesn't exist. Then, you can specify +``compass_envs`` as a path where shared conda environments will be installed +for compass releases. If developers always create their own conda +environments, this path will never be used. + +In ``[deploy]``, you will specify config options used in setting up conda +and Spack environments for developers. The ``compiler`` is the default +compiler to use for your system. You must supply a corresponding +``mpi_`` for each supported compiler (not just the default compiler) +that specifies the default MPI library for that compiler. If you only support +one compiler and MPI library, that's pretty simple: ``compiler`` is the name +of the compiler (e.g. ``intel`` or ``gnu``) and ``mpi_`` is the +MPI library (e.g. ``compiler_gnu = mpich`` or ``compiler_intel = openmpi``). +The ``spack`` option specifies a path where Spack environment will be created. +The option ``use_e3sm_hdf5_netcdf = False`` indicates that you will not use +the E3SM default modules for HDF5 and NetCDF libraries (which are not available +for machines installed in the way described here). + +Finally, ``[discovery]`` allows you to add a ``hostname_contains`` that is used +to automatically identify your machine based on its hostname. If your machine +has multiple login nodes with different hostnames, hopefully, a string common +to all login nodes can be used here. If your machine has a unique hostname, +simply give that. This option saves developers from having to specify +``--machine `` each time they setup compass environments or test +cases. + + +Describing a Spack Environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The next step is to create a template YAML file that can be used to create +`Spack environments `_ +for your machine. Compass uses Spack environments to build packages that need +MPI support or which should be build for some other reason with system +compilers rather than coming from pre-built conda packages. Using a Spack +environment allows these packages to be built together in a consistent way that +is not guaranteed if you try to install dependencies one-by-one. In Spack +parlance, this is known as +`unified concretization `_. + +To do this, you will create a file ``conda/spack/__.yaml`` +similar to the following example for an Ubuntu laptop: + +.. code-block:: + + spack: + specs: + - gcc + - openmpi + {{ specs }} + concretizer: + unify: true + packages: + all: + compiler: [gcc@11.3.0] + providers: + mpi: [openmpi] + curl: + externals: + - spec: curl@7.81.0 + prefix: /usr + buildable: false + gcc: + externals: + - spec: gcc@11.3.0 + prefix: /usr + buildable: false + config: + install_missing_compilers: false + compilers: + - compiler: + spec: gcc@11.3.0 + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + flags: {} + operating_system: ubuntu22.04 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] + + +Typically your system will already have compilers if nothing else, and this is +what we assume here. Give the appropriate path (replace ``/usr`` with the +appropriate path on your system). We have had better luck with ``gcc`` than +other compilers like Intel so far so for new supported machines so that's our +recommendation. Use ``gcc --version`` to determine the version and replace +``11.3.0`` with this number. + +Finally, you might need to update the ``target`` and ``operating_system``. +This is a bit of a "catch 22" in that you can use Spack to find this out but +compass is designed to clone and set up Spack for you so we assume you don't +have it yet. For now, make your best guess using the info on +`this page`_ +and correct it later if necessary. + +You may need to load a system module to get the compilers and potentially other +libraries such as MPI, HDF5, and NetCDF-C if you prefer to use system modules +rather than having Spack build them. If this is the case, the best way to do +this is to add a file +``conda/spack/__.sh`` along these lines: + +.. code-block:: bash + + module purge + module load perl/5.32.0-bsnc6lt + module load gcc/9.2.0-ugetvbp + module load openmpi/4.1.3-sxfyy4k + module load intel-mkl/2020.4.304-n3b5fye + module load hdf5/1.10.7-j3zxncu + module load netcdf-c/4.4.1-7ohuiwq + module load netcdf-fortran/4.4.4-k2zu3y5 + module load parallel-netcdf/1.11.0-mirrcz7 + +These modules will be loaded either before or after the spack environment, +depending on the ``modules_before`` and ``modules_after`` config options above. +You can also add modules in your YAML file but this shouldn't be necessary. + +For examples from various supported machines, compilers and MPI libraries, see the +`mache spack directory `_. + +Building the Spack Environment +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The next step is to try setting up compass and asking it to build the Spack +environment with a command something like: + +.. code-block:: bash + + ./conda/configure_compass_env.py --verbose --update_spack --conda -c gnu -i openmpi ... + +The ``--update_spack`` flag tells compass to create (or update) a Spack +environment. You can specify a directory for testing Spack with the +``--spack`` flag. You can specify a temporary directory for building spack +packages with ``--tmpdir`` (this directory must already exist). This is useful +if your ``/tmp`` space is small (Spack will use several GB of temporary space). + + +Creating the Spack environment may take anywhere from minutes to hours, +depending on your system. + +If dependencies don't build as expected, you may get an error message +suggesting that your ``operating_system`` or ``target`` aren't right. Here's +an example: + +.. code-block:: + + ==> Error: concretization failed for the following reasons: + + 1. zlib compiler '%gcc@9.4.0' incompatible with 'os=ubuntu20.04' + 2. readline compiler '%gcc@9.4.0' incompatible with 'os=ubuntu20.04' + 3. pkgconf compiler '%gcc@9.4.0' incompatible with 'os=ubuntu20.04' + +In this example, I had specified ``operating_system: ubuntu22.04`` in the YAML +file but in fact my operating system is ``ubuntu20.04`` as shown in the error +message. + +You can run: + +.. code-block:: bash + + source $SPACKDIR/share/spack/setup-env.sh + spack arch -o + spack arch -g + +where ``$SPACKDIR`` is the directory where the Spack repository was cloned +by compass (you should see ``Cloning into <$SPACKDIR>`` in the terminal, which +will hopefully help you find the right directory). This should hopefully give +you something close to what Spack wants. If you get something like +``x86_64_v4`` for the target, use ``x86_64`` instead. + +If you are getting other error messages, do your best to debug them but also +feel free to get in touch with the compass development team and we'll help if +we can. + +If you get everything working well, please feel free to make a pull request +into the compass main repo to add your supported machine. diff --git a/docs/developers_guide/quick_start.rst b/docs/developers_guide/quick_start.rst index 5bd20ef586..c7cded691e 100644 --- a/docs/developers_guide/quick_start.rst +++ b/docs/developers_guide/quick_start.rst @@ -89,7 +89,7 @@ If you are on one of the :ref:`dev_supported_machines`, run: ./conda/configure_compass_env.py --conda \ -c [--mpi ] [-m ] [--with_albany] \ - [--with_netlib_lapack] [--with_petsc] + [--with_petsc] The ```` is typically ``~/mambaforge``. This is the location where you would like to install Mambaforge or where it is @@ -125,8 +125,8 @@ will be linked in to MPAS-Ocean. Environments with PETSc and Netlib-LAPACK ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -If you are working with MPAS-Ocean test cases that need PETSC and -Netlib-LAPACK, you should specify ``--with_petsc --with_netlib_lapack`` to +If you are working with MPAS-Ocean test cases that need PETSc and +Netlib-LAPACK, you should specify ``--with_petsc`` to point to Spack environments where these libraries are included. Appropriate environment variables for pointing to these libraries will be build into the resulting load script (see below). @@ -181,8 +181,8 @@ you, this script will also: ``--spack`` flag can be used to point to a location for the spack repo to be checked out. Without this flag, a default location is used. Spack is used to build several libraries with system compilers and MPI library, including: - `SCORPIO `_ (parallel i/o for MPAS - components) `ESMF `_ (making mapping files + `ParallelIO `_ (parallel i/o in NetCDF + format) `ESMF `_ (making mapping files in parallel), `Trilinos `_, `Albany `_, `Netlib-LAPACK `_ and @@ -192,7 +192,7 @@ you, this script will also: * with the ``--with_albany`` flag, creates or uses an existing Spack environment that includes Albany and Trilinos. -* with the ``--with_petsc --with_netlib_lapack`` flags, creates or uses an +* with the ``--with_petsc`` flag, creates or uses an existing Spack environment that includes PETSc and Netlib-LAPACK. * make an activation script called ``load_*.sh``, where the details of the @@ -385,8 +385,8 @@ Creating/updating only the compass environment For some workflows (e.g. for MALI development with the Albany library when the MALI build environment has been created outside of ``compass``, for example, -on an unsupported machine), you may -only want to create the conda environment and not build SCORPIO, ESMF or +on an unsupported machine), you may only want to create the conda environment +and not build PIO, ESMF or other packages with system compilers, or include any system modules or environment variables in your activation script. In such cases, run with the ``--env_only`` flag: @@ -529,7 +529,7 @@ Code style for compass guidelines, the easiest way to stay in compliance as a developer writing new code or modifying existing code is to use a PEP8 style checker. One option is to use an IDE with a PEP8 style checker built in, such as -`PyCharm `_. See +`PyCharm `_. See `this tutorial `_ for some tips on checking code style in PyCharm.