diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f68d931d8..2305081d39 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,8 +8,6 @@ repos: hooks: - id: trailing-whitespace - id: end-of-file-fixer - - id: check-yaml - exclude: conda/recipe # Can run individually with `pre-commit run isort --all-files` - repo: https://github.com/PyCQA/isort diff --git a/compass/machines/chicoma-cpu.cfg b/compass/machines/chicoma-cpu.cfg index 600a768e5e..70a2256f6d 100644 --- a/compass/machines/chicoma-cpu.cfg +++ b/compass/machines/chicoma-cpu.cfg @@ -4,11 +4,11 @@ [paths] # A shared root directory where MPAS standalone data can be found -database_root = /usr/projects/regionalclimate/COMMON_MPAS/mpas_standalonedata/ +database_root = /usr/projects/e3sm/mpas_standalonedata/ # the path to the base conda environment where compass environments have # been created -compass_envs = /usr/projects/climate/SHARED_CLIMATE/compass/chicoma-cpu/base +compass_envs = /usr/projects/e3sm/compass/chicoma-cpu/base # Options related to deploying a compass conda environment on supported @@ -22,7 +22,7 @@ compiler = gnu mpi_gnu = mpich # the base path for spack environments used by compass -spack = /usr/projects/climate/SHARED_CLIMATE/compass/chicoma-cpu/spack +spack = /usr/projects/e3sm/compass/chicoma-cpu/spack # whether to use the same modules for hdf5, netcdf-c, netcdf-fortran and # pnetcdf as E3SM (spack modules are used otherwise) @@ -51,4 +51,3 @@ partition = standard # The job quality of service (QOS) to use qos = standard - diff --git a/compass/machines/cori-haswell.cfg b/compass/machines/cori-haswell.cfg deleted file mode 100644 index 14a955a737..0000000000 --- a/compass/machines/cori-haswell.cfg +++ /dev/null @@ -1,43 +0,0 @@ - -# The paths section describes paths that are used within the ocean core test -# cases. -[paths] - -# A shared root directory where MPAS standalone data can be found -database_root = /global/cfs/cdirs/e3sm/mpas_standalonedata - -# the path to the base conda environment where compass environments have -# been created -compass_envs = /global/common/software/e3sm/compass/cori-haswell/base - - -# Options related to deploying a compass conda environment on supported -# machines -[deploy] - -# the compiler set to use for system libraries and MPAS builds -compiler = intel - -# the system MPI library to use for intel compiler -mpi_intel = mpt - -# the system MPI library to use for gnu compiler -mpi_gnu = mpt - -# the base path for spack environments used by compass -spack = /global/cfs/cdirs/e3sm/software/compass/cori-haswell/spack - -# whether to use the same modules for hdf5, netcdf-c, netcdf-fortran and -# pnetcdf as E3SM (spack modules are used otherwise) -use_e3sm_hdf5_netcdf = True - -# the version of ESMF to build if using system compilers and MPI (don't build) -esmf = None - - -# Config options related to creating a job script -[job] - -# The job constraint to use, by default, taken from the first constraint (if -# any) provided for the machine by mache -constraint = haswell diff --git a/compass/version.py b/compass/version.py index 84f88e39e7..cbbfb1a865 100644 --- a/compass/version.py +++ b/compass/version.py @@ -1 +1 @@ -__version__ = '1.2.0-alpha.4' +__version__ = '1.2.0-alpha.5' diff --git a/conda/albany_supported.txt b/conda/albany_supported.txt index b77f98d25f..bd9ff79467 100644 --- a/conda/albany_supported.txt +++ b/conda/albany_supported.txt @@ -1,8 +1,5 @@ # a list of supported machine, compiler and mpi combinations for Albany -anvil, gnu, openmpi chicoma-cpu, gnu, mpich chrysalis, gnu, openmpi -compy, gnu, openmpi -cori-haswell, gnu, mpt pm-cpu, gnu, mpich diff --git a/conda/bootstrap.py b/conda/bootstrap.py index 9812f29190..57bc1e3fa2 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -21,6 +21,7 @@ get_logger, get_spack_base, install_miniconda, + log_message, parse_args, ) @@ -190,11 +191,9 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path, if args.with_petsc: lib_suffix = f'{lib_suffix}_petsc' - message = 'Turning off OpenMP because it doesn\'t work well with PETSc' - if logger is None: - print(message) - else: - logger.info(message) + log_message( + logger, + 'Turning off OpenMP because it doesn\'t work well with PETSc') args.without_openmp = True else: config.set('deploy', 'petsc', 'None') @@ -389,6 +388,7 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, spack_base, spack_template_path, env_vars, tmpdir, logger): albany = config.get('deploy', 'albany') + cmake = config.get('deploy', 'cmake') esmf = config.get('deploy', 'esmf') lapack = config.get('deploy', 'lapack') petsc = config.get('deploy', 'petsc') @@ -398,6 +398,9 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, specs = list() + if cmake != 'None': + specs.append(f'cmake "@{cmake}"') + e3sm_hdf5_netcdf = config.getboolean('deploy', 'use_e3sm_hdf5_netcdf') if not e3sm_hdf5_netcdf: hdf5 = config.get('deploy', 'hdf5') @@ -405,32 +408,36 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, netcdf_fortran = config.get('deploy', 'netcdf_fortran') pnetcdf = config.get('deploy', 'pnetcdf') specs.extend([ - f'hdf5@{hdf5}+cxx+fortran+hl+mpi+shared', - f'netcdf-c@{netcdf_c}+mpi~parallel-netcdf', - f'netcdf-fortran@{netcdf_fortran}', - f'parallel-netcdf@{pnetcdf}+cxx+fortran']) + f'hdf5 "@{hdf5}+cxx+fortran+hl+mpi+shared"', + f'netcdf-c "@{netcdf_c}+mpi~parallel-netcdf"', + f'netcdf-fortran "@{netcdf_fortran}"', + f'parallel-netcdf "@{pnetcdf}+cxx+fortran"']) if esmf != 'None': - specs.append(f'esmf@{esmf}+mpi+netcdf~pio+pnetcdf') + specs.append(f'esmf "@{esmf}+mpi+netcdf~pio+pnetcdf"') if lapack != 'None': - specs.append(f'netlib-lapack@{lapack}') + specs.append(f'netlib-lapack "@{lapack}"') include_e3sm_lapack = False else: include_e3sm_lapack = True if petsc != 'None': - specs.append(f'petsc@{petsc}+mpi+batch') + specs.append(f'petsc "@{petsc}+mpi+batch"') if scorpio != 'None': specs.append( - f'scorpio@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc') + f'scorpio ' + f'"@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc"') if albany != 'None': - specs.append(f'albany@{albany}+mpas') + specs.append(f'albany "@{albany}+mpas"') yaml_template = f'{spack_template_path}/{machine}_{compiler}_{mpi}.yaml' if not os.path.exists(yaml_template): yaml_template = None if update_spack: + home_dir = os.path.expanduser('~') + log_message(logger, 'Removing ~/.spack for safety') + safe_rmtree(os.path.join(home_dir, '.spack')) make_spack_env(spack_path=spack_branch_base, env_name=spack_env, spack_specs=specs, compiler=compiler, mpi=mpi, machine=machine, @@ -792,6 +799,13 @@ def check_supported(library, machine, compiler, mpi, source_path): f'on {machine}') +def safe_rmtree(path): + try: + shutil.rmtree(path) + except OSError: + pass + + def main(): # noqa: C901 args = parse_args(bootstrap=True) @@ -871,10 +885,7 @@ def main(): # noqa: C901 build_dir = f'conda/build{activ_suffix}' - try: - shutil.rmtree(build_dir) - except OSError: - pass + safe_rmtree(build_dir) try: os.makedirs(build_dir) except FileExistsError: diff --git a/conda/compass_env/spec-file.template b/conda/compass_env/spec-file.template index feb6240717..a45d3890e5 100644 --- a/conda/compass_env/spec-file.template +++ b/conda/compass_env/spec-file.template @@ -16,7 +16,7 @@ jigsawpy=0.3.3 jupyter lxml {% if include_mache %} -mache=1.10.0 +mache=1.14.0 {% endif %} matplotlib-base metis diff --git a/conda/configure_compass_env.py b/conda/configure_compass_env.py index 7e955c785d..cff86bbbeb 100755 --- a/conda/configure_compass_env.py +++ b/conda/configure_compass_env.py @@ -101,7 +101,7 @@ def main(): if local_mache: mache = '' else: - mache = '"mache=1.10.0"' + mache = '"mache=1.14.0"' setup_install_env(env_name, activate_base, args.use_local, logger, args.recreate, conda_base, mache) diff --git a/conda/default.cfg b/conda/default.cfg index a5cde0183a..b8cf19a440 100644 --- a/conda/default.cfg +++ b/conda/default.cfg @@ -21,6 +21,8 @@ mpi = nompi # the version of various packages to include if using spack albany = develop +# cmake newer than 3.23.0 needed for Trilinos +cmake = 3.23.0: esmf = 8.2.0 hdf5 = 1.12.1 lapack = 3.9.1 diff --git a/conda/petsc_supported.txt b/conda/petsc_supported.txt index 98a64b08bc..5bc50ab6ab 100644 --- a/conda/petsc_supported.txt +++ b/conda/petsc_supported.txt @@ -7,7 +7,4 @@ chicoma-cpu, gnu, mpich chrysalis, intel, openmpi chrysalis, gnu, openmpi compy, intel, impi -compy, gnu, openmpi -cori-haswell, intel, mpt -cori-haswell, gnu, mpt pm-cpu, gnu, mpich diff --git a/conda/recipe/meta.yaml b/conda/recipe/meta.yaml index 1c11ddf993..900ac80b38 100644 --- a/conda/recipe/meta.yaml +++ b/conda/recipe/meta.yaml @@ -1,5 +1,5 @@ {% set name = "compass" %} -{% set version = "1.2.0alpha.4" %} +{% set version = "1.2.0alpha.5" %} {% set build = 0 %} {% if mpi == "nompi" %} @@ -52,7 +52,7 @@ requirements: - jigsawpy 0.3.3 - jupyter - lxml - - mache 1.10.0 + - mache 1.14.0 - matplotlib-base - metis - mpas_tools 0.19.0 diff --git a/conda/shared.py b/conda/shared.py index ffd40561fc..c2c5358ed4 100644 --- a/conda/shared.py +++ b/conda/shared.py @@ -220,6 +220,13 @@ def restore_bashrc(): shutil.move(src, dst) +def log_message(logger, message): + if logger is None: + print(message) + else: + logger.info(message) + + def get_logger(name, log_filename): print(f'Logging to: {log_filename}\n') try: diff --git a/conda/unsupported.txt b/conda/unsupported.txt index 4e72cffc0a..b5b897d69f 100644 --- a/conda/unsupported.txt +++ b/conda/unsupported.txt @@ -9,6 +9,7 @@ chrysalis, gnu, impi chrysalis, oneapi-ifx, openmpi chrysalis, oneapi-ifx, impi compy, intel, mvapich2 +compy, gnu, openmpi compy, pgi, impi compy, pgi, mvapich2 pm-cpu, nvidia, mpich diff --git a/docs/design_docs/compass_package.rst b/docs/design_docs/compass_package.rst index a352811735..330e4dc9ba 100644 --- a/docs/design_docs/compass_package.rst +++ b/docs/design_docs/compass_package.rst @@ -2208,7 +2208,6 @@ The available machines are listed with: Machines: anvil default - cori-haswell chrysalis compy diff --git a/docs/developers_guide/machines/cori.rst b/docs/developers_guide/machines/cori.rst deleted file mode 100644 index df3a440c96..0000000000 --- a/docs/developers_guide/machines/cori.rst +++ /dev/null @@ -1,34 +0,0 @@ -Cori -==== - -cori-haswell, intel -------------------- - -This is the default ``compass`` architecture and compiler on Cori. If the -environment has been set up properly (see :ref:`dev_conda_env`), you should be -able to source: - -.. code-block:: bash - - source load_dev_compass_1.0.0_cori-haswell_intel_mpt.sh - -Then, you can build the MPAS model with - -.. code-block:: bash - - make [DEBUG=true] [OPENMP=true] intel-cray - -cori-haswell, gnu ------------------ - -If you've set things up for this compiler, you should be able to: - -.. code-block:: bash - - source load_dev_compass_1.0.0_cori-haswell_gnu_mpt.sh - -Then, you can build the MPAS model with - -.. code-block:: bash - - make [DEBUG=true] [OPENMP=true] [ALBANY=true] gnu-cray diff --git a/docs/developers_guide/machines/index.rst b/docs/developers_guide/machines/index.rst index 1f050d36f2..7cb96e1c1f 100644 --- a/docs/developers_guide/machines/index.rst +++ b/docs/developers_guide/machines/index.rst @@ -63,10 +63,6 @@ the MPAS model. | +------------+-----------+-------------------+ | | gnu | openmpi | gfortran | +--------------+------------+-----------+-------------------+ -| cori-haswell | intel | mpt | intel-cray | -| +------------+-----------+-------------------+ -| | gnu | mpt | gnu-cray | -+--------------+------------+-----------+-------------------+ | pm-cpu | gnu | mpich | gnu-cray | +--------------+------------+-----------+-------------------+ @@ -79,7 +75,6 @@ Below are specifics for each supported machine chicoma chrysalis compy - cori perlmutter diff --git a/docs/users_guide/machines/cori.rst b/docs/users_guide/machines/cori.rst deleted file mode 100644 index 6f6bd598a7..0000000000 --- a/docs/users_guide/machines/cori.rst +++ /dev/null @@ -1,215 +0,0 @@ -Cori -==== - -login: ``ssh $my_username@cori.nersc.gov`` - -interactive login: - -.. code-block:: bash - - # for Haswell: - salloc --partition=debug --nodes=1 --time=30:00 -C haswell - - # for KNL: - salloc --partition=debug --nodes=1 --time=30:00 -C knl - -Compute time: - -* Check hours of compute usage at https://nim.nersc.gov/ - -File system: - -* Overview: https://docs.nersc.gov/filesystems/ - -* home directory: ``/global/homes/$my_username`` - -* scratch directory: ``/global/cscratch1/sd/$my_username`` - -* Check your individual disk usage with ``myquota`` - -* Check the group disk usage with ``prjquota projectID``, i.e. - ``prjquota m2833`` or ``prjquota acme`` - -Archive: - -* NERSC uses HPSS with the commands ``hsi`` and ``htar`` - -* overview: https://docs.nersc.gov/filesystems/archive/ - -* E3SM uses `zstash `_ - - -Cori-Haswell ------------- - -Cori's Haswell and KNL nodes have different configuration options and -compilers. We only support Cori-Haswell at this time. - -config options -~~~~~~~~~~~~~~ - -Here are the default -config options added when you choose ``-m cori-haswell`` when setting up test -cases or a test suite: - -.. code-block:: cfg - - # The paths section describes paths that are used within the ocean core test - # cases. - [paths] - - # A shared root directory where MPAS standalone data can be found - database_root = /global/cfs/cdirs/e3sm/mpas_standalonedata - - # the path to the base conda environment where compass environments have - # been created - compass_envs = /global/common/software/e3sm/compass/cori-haswell/base - - - # Options related to deploying a compass conda environment on supported - # machines - [deploy] - - # the compiler set to use for system libraries and MPAS builds - compiler = intel - - # the system MPI library to use for intel compiler - mpi_intel = mpt - - # the system MPI library to use for gnu compiler - mpi_gnu = mpt - - # the base path for spack environments used by compass - spack = /global/cfs/cdirs/e3sm/software/compass/cori-haswell/spack - - # whether to use the same modules for hdf5, netcdf-c, netcdf-fortran and - # pnetcdf as E3SM (spack modules are used otherwise) - use_e3sm_hdf5_netcdf = True - - # the version of ESMF to build if using system compilers and MPI (don't build) - esmf = None - - - # Config options related to creating a job script - [job] - - # The job constraint to use, by default, taken from the first constraint (if - # any) provided for the machine by mache - constraint = haswell - -Additionally, some relevant config options come from the -`mache `_ package: - -.. code-block:: cfg - - # The parallel section describes options related to running jobs in parallel - [parallel] - - # parallel system of execution: slurm, cobalt or single_node - system = slurm - - # whether to use mpirun or srun to run a task - parallel_executable = srun - - # cores per node on the machine - cores_per_node = 32 - - # account for running diagnostics jobs - account = e3sm - - # available configurations(s) (default is the first) - configurations = haswell - - # quality of service (default is the first) - qos = regular, premium, debug - - -Intel on Cori-Haswell -~~~~~~~~~~~~~~~~~~~~~ - -To load the compass environment and modules, and set appropriate environment -variables: - -.. code-block:: bash - - source /global/cfs/cdirs/e3sm/software/compass/cori-haswell/load_latest_compass_intel_mpt.sh - -To build the MPAS model with - -.. code-block:: bash - - make [DEBUG=true] [OPENMP=true] intel-cray - - -Gnu on Cori-Haswell -~~~~~~~~~~~~~~~~~~~ - -To load the compass environment and modules, and set appropriate environment -variables: - -.. code-block:: bash - - source /global/cfs/cdirs/e3sm/software/compass/cori-haswell/load_latest_compass_gnu_mpt.sh - -To build the MPAS model with - -.. code-block:: bash - - make [DEBUG=true] [OPENMP=true] [ALBANY=true] gnu-cray - - -Jupyter notebook on remote data -------------------------------- - -You can run Jupyter notebooks on NERSC with direct access to scratch data as -follows: - -.. code-block:: bash - - ssh -Y -L 8844:localhost:8844 MONIKER@cori.nersc.gov - jupyter notebook --no-browser --port 8844 - # in local browser, go to: - http://localhost:8844/ - -Note that on NERSC, you can also use their -`Jupyter server `_, -it’s really nice and grabs a compute node for you automatically on logon. -You’ll need to create a python kernel from e3sm-unified following these steps -(taken from https://docs.nersc.gov/connect/jupyter/). After creating the -kernel, you just go to “Change Kernel” in the Jupyter notebook and you’re ready -to go. - -You can use one of our default Python 2, Python 3, or R kernels. If you have a -Conda environment, depending on how it is installed, it may just show up in the -list of kernels you can use. If not, use the following procedure to enable a -custom kernel based on a Conda environment. Let's start by assuming you are a -user with username ``user`` who wants to create a Conda environment on Cori and use -it from Jupyter. - -.. code-block:: bash - - - module load python - conda create -n myenv python=3.7 ipykernel - <... installation messages ...> - source activate myenv - python -m ipykernel install --user --name myenv --display-name MyEnv - Installed kernelspec myenv in /global/u1/u/user/.local/share/jupyter/kernels/myenv - -Be sure to specify what version of Python interpreter you want installed. This -will create and install a JSON file called a "kernel spec" in ``kernel.json`` at -the path described in the install command output. - -.. code-block:: json - - { - "argv": [ - "/global/homes/u/user/.conda/envs/myenv/bin/python", - "-m", - "ipykernel_launcher", - "-f", - "{connection_file}" - ], - "display_name": "MyEnv", - "language": "python" - } diff --git a/docs/users_guide/machines/index.rst b/docs/users_guide/machines/index.rst index 8b169fe0fa..32187189ed 100644 --- a/docs/users_guide/machines/index.rst +++ b/docs/users_guide/machines/index.rst @@ -136,7 +136,6 @@ to use the default MPI variant for each compiler on each machine. chicoma chrysalis compy - cori perlmutter diff --git a/docs/users_guide/quick_start.rst b/docs/users_guide/quick_start.rst index 53a5eaf0e4..e2943565e6 100644 --- a/docs/users_guide/quick_start.rst +++ b/docs/users_guide/quick_start.rst @@ -16,7 +16,7 @@ For each ``compass`` release, we maintain a ``compass`` package as well as all of its dependencies and some libraries (currently `ESMF `_ and `SCORPIO `_) built with system -MPI on our standard machines (Anvil, Chrysalis, Compy, and Cori). +MPI on our standard machines (Anvil, Chicoma, Chrysalis, Compy, and Perlmutter). Here are the commands to load the the environment for the latest ``compass`` release with the default compiler and MPI library on each machine: @@ -44,12 +44,6 @@ Here are the commands to load the the environment for the latest source /share/apps/E3SM/conda_envs/compass/load_latest_compass.sh -* Cori-Haswell: - -.. code-block:: bash - - source /global/cfs/cdirs/e3sm/software/compass/cori-haswell/load_latest_compass.sh - * Perlmutter-CPU (coming soon): .. code-block:: bash diff --git a/utils/matrix/setup_matrix.py b/utils/matrix/setup_matrix.py index 735d584175..df452626d6 100755 --- a/utils/matrix/setup_matrix.py +++ b/utils/matrix/setup_matrix.py @@ -8,16 +8,15 @@ try: from configparser import ConfigParser except ImportError: - from six.moves import configparser import six + from six.moves import configparser if six.PY2: ConfigParser = configparser.SafeConfigParser else: ConfigParser = configparser.ConfigParser -from shared import get_logger, check_call - +from shared import check_call, get_logger # build targets from # https://mpas-dev.github.io/compass/latest/developers_guide/machines/index.html#supported-machines @@ -37,9 +36,6 @@ 'compy': { ('intel', 'impi'): 'intel-mpi', ('gnu', 'openmpi'): 'gfortran'}, - 'cori-haswell': { - ('intel', 'mpt'): 'intel-cray', - ('gnu', 'mpt'): 'gnu-cray'}, 'pm-cpu': { ('gnu', 'mpich'): 'gnu-cray'}, 'conda-linux': { @@ -311,8 +307,8 @@ def compass_setup(script_name, setup_command, mpas_path, mpas_model, work_base, if index == -1: index = parts.index('--test_suite') - if index != -1 and len(parts) > index+1: - suite = parts[index+1] + if index != -1 and len(parts) > index + 1: + suite = parts[index + 1] elif setup_command.startswith('compass setup'): suite = 'custom'