From fa6970ce1fc954aaeb889476953eda1f675a4f8a Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 28 Feb 2023 06:19:44 +0100 Subject: [PATCH 1/3] Update deployment to fix mamba init issue We need to update mamba before installing boa to prevent update failure. --- conda/bootstrap.py | 6 +++--- conda/configure_compass_env.py | 15 ++++++++++----- conda/shared.py | 18 +++++++++--------- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index f71da15152..a1a4b51aea 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -232,7 +232,7 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path, f'source {conda_base}/etc/profile.d/conda.sh; ' \ f'source {conda_base}/etc/profile.d/mamba.sh' - activate_env = f'{source_activation_scripts}; conda activate {env_name}' + activate_env = f'{source_activation_scripts}; mamba activate {env_name}' return python, recreate, conda_mpi, activ_suffix, env_suffix, \ activ_path, env_path, env_name, activate_env, spack_env @@ -268,7 +268,7 @@ def build_conda_env(env_type, recreate, machine, mpi, conda_mpi, version, f'{conda_base}/etc/profile.d/conda.sh') activate_env = \ - f'source {base_activation_script}; conda activate {env_name}' + f'source {base_activation_script}; mamba activate {env_name}' with open(f'{conda_template_path}/spec-file.template', 'r') as f: template = Template(f.read()) @@ -830,7 +830,7 @@ def main(): # noqa: C901 f'source {conda_base}/etc/profile.d/conda.sh; ' \ f'source {conda_base}/etc/profile.d/mamba.sh' - activate_base = f'{source_activation_scripts}; conda activate' + activate_base = f'{source_activation_scripts}; mamba activate' compilers, mpis = get_compilers_mpis(config, machine, args.compilers, args.mpis, source_path) diff --git a/conda/configure_compass_env.py b/conda/configure_compass_env.py index a7fd4938c7..91810f27e9 100755 --- a/conda/configure_compass_env.py +++ b/conda/configure_compass_env.py @@ -8,16 +8,21 @@ try: from configparser import ConfigParser except ImportError: - from six.moves import configparser import six + from six.moves import configparser if six.PY2: ConfigParser = configparser.SafeConfigParser else: ConfigParser = configparser.ConfigParser -from shared import parse_args, get_conda_base, check_call, install_miniconda, \ - get_logger +from shared import ( + check_call, + get_conda_base, + get_logger, + install_miniconda, + parse_args, +) def get_config(config_file): @@ -85,11 +90,11 @@ def main(): 'source {}/etc/profile.d/conda.sh; ' \ 'source {}/etc/profile.d/mamba.sh'.format(conda_base, conda_base) - activate_base = '{}; conda activate'.format(source_activation_scripts) + activate_base = '{}; mamba activate'.format(source_activation_scripts) activate_install_env = \ '{}; ' \ - 'conda activate {}'.format(source_activation_scripts, env_name) + 'mamba activate {}'.format(source_activation_scripts, env_name) try: os.makedirs('conda/logs') except OSError: diff --git a/conda/shared.py b/conda/shared.py index 0ec9498d79..8291cd50dc 100644 --- a/conda/shared.py +++ b/conda/shared.py @@ -1,17 +1,17 @@ from __future__ import print_function -import os -import sys import argparse -import subprocess -import platform import logging +import os +import platform import shutil +import subprocess +import sys try: - from urllib.request import urlopen, Request + from urllib.request import Request, urlopen except ImportError: - from urllib2 import urlopen, Request + from urllib2 import Request, urlopen def parse_args(bootstrap): @@ -155,7 +155,7 @@ def install_miniconda(conda_base, activate_base, logger): else: system = 'Linux' miniconda = 'Mambaforge-{}-x86_64.sh'.format(system) - url = 'https://github.com/conda-forge/miniforge/releases/latest/download/{}'.format(miniconda) + url = 'https://github.com/conda-forge/miniforge/releases/latest/download/{}'.format(miniconda) # noqa: E501 print(url) req = Request(url, headers={'User-Agent': 'Mozilla/5.0'}) f = urlopen(req) @@ -174,8 +174,8 @@ def install_miniconda(conda_base, activate_base, logger): commands = '{}; ' \ 'conda config --add channels conda-forge; ' \ 'conda config --set channel_priority strict; ' \ - 'conda install -y boa; ' \ - 'conda update -y --all; ' \ + 'mamba update -y --all; ' \ + 'mamba install -y boa; ' \ 'mamba init'.format(activate_base) check_call(commands, logger=logger) From d1cb72c9deb1fd61439e3365a913e25f7030475e Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 28 Feb 2023 07:00:02 +0100 Subject: [PATCH 2/3] Switch from to using `&&` between deploy commands This should stop things when there are errors, whereas previously deployment was plowing on even after things had gone wrong. --- conda/bootstrap.py | 41 +++++++++++++++++----------------- conda/configure_compass_env.py | 12 +++++----- conda/shared.py | 11 +++++---- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/conda/bootstrap.py b/conda/bootstrap.py index a1a4b51aea..73a29f5118 100755 --- a/conda/bootstrap.py +++ b/conda/bootstrap.py @@ -229,10 +229,10 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path, env_path = os.path.join(conda_base, 'envs', env_name) source_activation_scripts = \ - f'source {conda_base}/etc/profile.d/conda.sh; ' \ + f'source {conda_base}/etc/profile.d/conda.sh && ' \ f'source {conda_base}/etc/profile.d/mamba.sh' - activate_env = f'{source_activation_scripts}; mamba activate {env_name}' + activate_env = f'{source_activation_scripts} && mamba activate {env_name}' return python, recreate, conda_mpi, activ_suffix, env_suffix, \ activ_path, env_path, env_name, activate_env, spack_env @@ -264,11 +264,12 @@ def build_conda_env(env_type, recreate, machine, mpi, conda_mpi, version, channels = f'--override-channels {" ".join(channels)}' packages = f'python={python}' - base_activation_script = os.path.abspath( - f'{conda_base}/etc/profile.d/conda.sh') + conda_base = os.path.abspath(conda_base) activate_env = \ - f'source {base_activation_script}; mamba activate {env_name}' + f'source {conda_base}/etc/profile.d/conda.sh &&' \ + f'source {conda_base}/etc/profile.d/mamba.sh &&' \ + f'mamba activate {env_name}' with open(f'{conda_template_path}/spec-file.template', 'r') as f: template = Template(f.read()) @@ -296,14 +297,14 @@ def build_conda_env(env_type, recreate, machine, mpi, conda_mpi, version, if env_type == 'dev': # install dev dependencies and compass itself commands = \ - f'{activate_base}; ' \ + f'{activate_base} && ' \ f'mamba create -y -n {env_name} {channels} ' \ f'--file {spec_filename} {packages}' check_call(commands, logger=logger) commands = \ - f'{activate_env}; ' \ - f'cd {source_path}; ' \ + f'{activate_env} && ' \ + f'cd {source_path} && ' \ f'python -m pip install -e .' check_call(commands, logger=logger) @@ -311,7 +312,7 @@ def build_conda_env(env_type, recreate, machine, mpi, conda_mpi, version, # conda packages don't like dashes version_conda = version.replace('-', '') packages = f'{packages} "compass={version_conda}={mpi_prefix}_*"' - commands = f'{activate_base}; ' \ + commands = f'{activate_base} && ' \ f'mamba create -y -n {env_name} {channels} {packages}' check_call(commands, logger=logger) else: @@ -319,14 +320,14 @@ def build_conda_env(env_type, recreate, machine, mpi, conda_mpi, version, print(f'Updating {env_name}\n') # install dev dependencies and compass itself commands = \ - f'{activate_base}; ' \ + f'{activate_base} && ' \ f'mamba install -y -n {env_name} {channels} ' \ f'--file {spec_filename} {packages}' check_call(commands, logger=logger) commands = \ - f'{activate_env}; ' \ - f'cd {source_path}; ' \ + f'{activate_env} && ' \ + f'cd {source_path} && ' \ f'python -m pip install -e .' check_call(commands, logger=logger) else: @@ -493,9 +494,9 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env, def set_ld_library_path(spack_branch_base, spack_env, logger): commands = \ - f'source {spack_branch_base}/share/spack/setup-env.sh; ' \ - f'spack env activate {spack_env}; ' \ - f'spack config add modules:prefix_inspections:lib:[LD_LIBRARY_PATH]; ' \ + f'source {spack_branch_base}/share/spack/setup-env.sh && ' \ + f'spack env activate {spack_env} && ' \ + f'spack config add modules:prefix_inspections:lib:[LD_LIBRARY_PATH] && ' \ f'spack config add modules:prefix_inspections:lib64:[LD_LIBRARY_PATH]' # noqa: E501 check_call(commands, logger=logger) @@ -597,12 +598,12 @@ def check_env(script_filename, env_name, logger): ['compass', 'clean', '--help']] for import_name in imports: - command = '{}; python -c "import {}"'.format(activate, import_name) + command = '{} && python -c "import {}"'.format(activate, import_name) test_command(command, os.environ, import_name, logger) for command in commands: package = command[0] - command = '{}; {}'.format(activate, ' '.join(command)) + command = '{} && {}'.format(activate, ' '.join(command)) test_command(command, os.environ, package, logger) @@ -827,10 +828,10 @@ def main(): # noqa: C901 conda_base = os.path.abspath(conda_base) source_activation_scripts = \ - f'source {conda_base}/etc/profile.d/conda.sh; ' \ + f'source {conda_base}/etc/profile.d/conda.sh && ' \ f'source {conda_base}/etc/profile.d/mamba.sh' - activate_base = f'{source_activation_scripts}; mamba activate' + activate_base = f'{source_activation_scripts} && mamba activate' compilers, mpis = get_compilers_mpis(config, machine, args.compilers, args.mpis, source_path) @@ -952,7 +953,7 @@ def main(): # noqa: C901 check_call(f'ln -sfn {script_filename} {link}') os.chdir(source_path) - commands = '{}; conda clean -y -p -t'.format(activate_base) + commands = '{} && conda clean -y -p -t'.format(activate_base) check_call(commands, logger=logger) if args.update_spack or env_type != 'dev': diff --git a/conda/configure_compass_env.py b/conda/configure_compass_env.py index 91810f27e9..8e782cba89 100755 --- a/conda/configure_compass_env.py +++ b/conda/configure_compass_env.py @@ -42,7 +42,7 @@ def bootstrap(activate_install_env, source_path, local_conda_build): print('Creating the compass conda environment\n') bootstrap_command = '{}/conda/bootstrap.py'.format(source_path) - command = '{}; ' \ + command = '{} && ' \ '{} {}'.format(activate_install_env, bootstrap_command, ' '.join(sys.argv[1:])) if local_conda_build is not None: @@ -61,13 +61,13 @@ def setup_install_env(env_name, activate_base, use_local, logger, recreate, packages = 'progressbar2 jinja2 "mache=1.10.0"' if recreate or not os.path.exists(env_path): print('Setting up a conda environment for installing compass\n') - commands = '{}; ' \ + commands = '{} && ' \ 'mamba create -y -n {} {} {}'.format(activate_base, env_name, channels, packages) else: print('Updating conda environment for installing compass\n') - commands = '{}; ' \ + commands = '{} && ' \ 'mamba install -y -n {} {} {}'.format(activate_base, env_name, channels, packages) @@ -87,13 +87,13 @@ def main(): env_name = 'compass_bootstrap' source_activation_scripts = \ - 'source {}/etc/profile.d/conda.sh; ' \ + 'source {}/etc/profile.d/conda.sh && ' \ 'source {}/etc/profile.d/mamba.sh'.format(conda_base, conda_base) - activate_base = '{}; mamba activate'.format(source_activation_scripts) + activate_base = '{} && mamba activate'.format(source_activation_scripts) activate_install_env = \ - '{}; ' \ + '{} && ' \ 'mamba activate {}'.format(source_activation_scripts, env_name) try: os.makedirs('conda/logs') diff --git a/conda/shared.py b/conda/shared.py index 8291cd50dc..d06f6d85ed 100644 --- a/conda/shared.py +++ b/conda/shared.py @@ -116,7 +116,7 @@ def get_spack_base(spack_base, config): def check_call(commands, env=None, logger=None): - print_command = '\n '.join(commands.split('; ')) + print_command = '\n '.join(commands.split(' && ')) if logger is None: print('\n Running:\n {}\n'.format(print_command)) else: @@ -171,11 +171,10 @@ def install_miniconda(conda_base, activate_base, logger): backup_bashrc() print('Doing initial setup\n') - commands = '{}; ' \ - 'conda config --add channels conda-forge; ' \ - 'conda config --set channel_priority strict; ' \ - 'mamba update -y --all; ' \ - 'mamba install -y boa; ' \ + commands = '{} && ' \ + 'conda config --add channels conda-forge && ' \ + 'conda config --set channel_priority strict && ' \ + 'mamba update -y --all && ' \ 'mamba init'.format(activate_base) check_call(commands, logger=logger) From 95beb59adeda06224452a76360ccd5e57cf14993 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Tue, 28 Feb 2023 07:11:00 +0100 Subject: [PATCH 3/3] Try removing boa for now (very slow) --- conda/shared.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/conda/shared.py b/conda/shared.py index d06f6d85ed..a20cfd8476 100644 --- a/conda/shared.py +++ b/conda/shared.py @@ -171,9 +171,22 @@ def install_miniconda(conda_base, activate_base, logger): backup_bashrc() print('Doing initial setup\n') + commands = '{} && ' \ 'conda config --add channels conda-forge && ' \ - 'conda config --set channel_priority strict && ' \ + 'conda config --set channel_priority strict' \ + ''.format(activate_base) + + check_call(commands, logger=logger) + + commands = '{} && ' \ + 'conda remove -y boa'.format(activate_base) + try: + check_call(commands, logger=logger) + except subprocess.CalledProcessError: + pass + + commands = '{} && ' \ 'mamba update -y --all && ' \ 'mamba init'.format(activate_base)