Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix deployment with latest conda (23.1.0) #545

Merged
merged 3 commits into from
Feb 28, 2023
Merged

Fix deployment with latest conda (23.1.0) #545

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fa6970c
Update deployment to fix mamba init issue
xylar Feb 28, 2023
d1cb72c
Switch from to using `&&` between deploy commands
xylar Feb 28, 2023
95beb59
Try removing boa for now (very slow)
xylar Feb 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 21 additions & 20 deletions conda/bootstrap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,10 +229,10 @@ def get_env_setup(args, config, machine, compiler, mpi, env_type, source_path,
env_path = os.path.join(conda_base, 'envs', env_name)

source_activation_scripts = \
f'source {conda_base}/etc/profile.d/conda.sh; ' \
f'source {conda_base}/etc/profile.d/conda.sh && ' \
f'source {conda_base}/etc/profile.d/mamba.sh'

activate_env = f'{source_activation_scripts}; conda activate {env_name}'
activate_env = f'{source_activation_scripts} && mamba activate {env_name}'

return python, recreate, conda_mpi, activ_suffix, env_suffix, \
activ_path, env_path, env_name, activate_env, spack_env
Expand Down Expand Up @@ -264,11 +264,12 @@ def build_conda_env(env_type, recreate, machine, mpi, conda_mpi, version,
channels = f'--override-channels {" ".join(channels)}'
packages = f'python={python}'

base_activation_script = os.path.abspath(
f'{conda_base}/etc/profile.d/conda.sh')
conda_base = os.path.abspath(conda_base)

activate_env = \
f'source {base_activation_script}; conda activate {env_name}'
f'source {conda_base}/etc/profile.d/conda.sh &&' \
f'source {conda_base}/etc/profile.d/mamba.sh &&' \
f'mamba activate {env_name}'

with open(f'{conda_template_path}/spec-file.template', 'r') as f:
template = Template(f.read())
Expand Down Expand Up @@ -296,37 +297,37 @@ def build_conda_env(env_type, recreate, machine, mpi, conda_mpi, version,
if env_type == 'dev':
# install dev dependencies and compass itself
commands = \
f'{activate_base}; ' \
f'{activate_base} && ' \
f'mamba create -y -n {env_name} {channels} ' \
f'--file {spec_filename} {packages}'
check_call(commands, logger=logger)

commands = \
f'{activate_env}; ' \
f'cd {source_path}; ' \
f'{activate_env} && ' \
f'cd {source_path} && ' \
f'python -m pip install -e .'
check_call(commands, logger=logger)

else:
# conda packages don't like dashes
version_conda = version.replace('-', '')
packages = f'{packages} "compass={version_conda}={mpi_prefix}_*"'
commands = f'{activate_base}; ' \
commands = f'{activate_base} && ' \
f'mamba create -y -n {env_name} {channels} {packages}'
check_call(commands, logger=logger)
else:
if env_type == 'dev':
print(f'Updating {env_name}\n')
# install dev dependencies and compass itself
commands = \
f'{activate_base}; ' \
f'{activate_base} && ' \
f'mamba install -y -n {env_name} {channels} ' \
f'--file {spec_filename} {packages}'
check_call(commands, logger=logger)

commands = \
f'{activate_env}; ' \
f'cd {source_path}; ' \
f'{activate_env} && ' \
f'cd {source_path} && ' \
f'python -m pip install -e .'
check_call(commands, logger=logger)
else:
Expand Down Expand Up @@ -493,9 +494,9 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, spack_env,

def set_ld_library_path(spack_branch_base, spack_env, logger):
commands = \
f'source {spack_branch_base}/share/spack/setup-env.sh; ' \
f'spack env activate {spack_env}; ' \
f'spack config add modules:prefix_inspections:lib:[LD_LIBRARY_PATH]; ' \
f'source {spack_branch_base}/share/spack/setup-env.sh && ' \
f'spack env activate {spack_env} && ' \
f'spack config add modules:prefix_inspections:lib:[LD_LIBRARY_PATH] && ' \
f'spack config add modules:prefix_inspections:lib64:[LD_LIBRARY_PATH]' # noqa: E501
check_call(commands, logger=logger)

Expand Down Expand Up @@ -597,12 +598,12 @@ def check_env(script_filename, env_name, logger):
['compass', 'clean', '--help']]

for import_name in imports:
command = '{}; python -c "import {}"'.format(activate, import_name)
command = '{} && python -c "import {}"'.format(activate, import_name)
test_command(command, os.environ, import_name, logger)

for command in commands:
package = command[0]
command = '{}; {}'.format(activate, ' '.join(command))
command = '{} && {}'.format(activate, ' '.join(command))
test_command(command, os.environ, package, logger)


Expand Down Expand Up @@ -827,10 +828,10 @@ def main(): # noqa: C901
conda_base = os.path.abspath(conda_base)

source_activation_scripts = \
f'source {conda_base}/etc/profile.d/conda.sh; ' \
f'source {conda_base}/etc/profile.d/conda.sh && ' \
f'source {conda_base}/etc/profile.d/mamba.sh'

activate_base = f'{source_activation_scripts}; conda activate'
activate_base = f'{source_activation_scripts} && mamba activate'

compilers, mpis = get_compilers_mpis(config, machine, args.compilers,
args.mpis, source_path)
Expand Down Expand Up @@ -952,7 +953,7 @@ def main(): # noqa: C901
check_call(f'ln -sfn {script_filename} {link}')
os.chdir(source_path)

commands = '{}; conda clean -y -p -t'.format(activate_base)
commands = '{} && conda clean -y -p -t'.format(activate_base)
check_call(commands, logger=logger)

if args.update_spack or env_type != 'dev':
Expand Down
25 changes: 15 additions & 10 deletions conda/configure_compass_env.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,21 @@
try:
from configparser import ConfigParser
except ImportError:
from six.moves import configparser
import six
from six.moves import configparser

if six.PY2:
ConfigParser = configparser.SafeConfigParser
else:
ConfigParser = configparser.ConfigParser

from shared import parse_args, get_conda_base, check_call, install_miniconda, \
get_logger
from shared import (
check_call,
get_conda_base,
get_logger,
install_miniconda,
parse_args,
)


def get_config(config_file):
Expand All @@ -37,7 +42,7 @@ def bootstrap(activate_install_env, source_path, local_conda_build):

print('Creating the compass conda environment\n')
bootstrap_command = '{}/conda/bootstrap.py'.format(source_path)
command = '{}; ' \
command = '{} && ' \
'{} {}'.format(activate_install_env, bootstrap_command,
' '.join(sys.argv[1:]))
if local_conda_build is not None:
Expand All @@ -56,13 +61,13 @@ def setup_install_env(env_name, activate_base, use_local, logger, recreate,
packages = 'progressbar2 jinja2 "mache=1.10.0"'
if recreate or not os.path.exists(env_path):
print('Setting up a conda environment for installing compass\n')
commands = '{}; ' \
commands = '{} && ' \
'mamba create -y -n {} {} {}'.format(activate_base,
env_name, channels,
packages)
else:
print('Updating conda environment for installing compass\n')
commands = '{}; ' \
commands = '{} && ' \
'mamba install -y -n {} {} {}'.format(activate_base,
env_name, channels,
packages)
Expand All @@ -82,14 +87,14 @@ def main():
env_name = 'compass_bootstrap'

source_activation_scripts = \
'source {}/etc/profile.d/conda.sh; ' \
'source {}/etc/profile.d/conda.sh && ' \
'source {}/etc/profile.d/mamba.sh'.format(conda_base, conda_base)

activate_base = '{}; conda activate'.format(source_activation_scripts)
activate_base = '{} && mamba activate'.format(source_activation_scripts)

activate_install_env = \
'{}; ' \
'conda activate {}'.format(source_activation_scripts, env_name)
'{} && ' \
'mamba activate {}'.format(source_activation_scripts, env_name)
try:
os.makedirs('conda/logs')
except OSError:
Expand Down
38 changes: 25 additions & 13 deletions conda/shared.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
from __future__ import print_function

import os
import sys
import argparse
import subprocess
import platform
import logging
import os
import platform
import shutil
import subprocess
import sys

try:
from urllib.request import urlopen, Request
from urllib.request import Request, urlopen
except ImportError:
from urllib2 import urlopen, Request
from urllib2 import Request, urlopen


def parse_args(bootstrap):
Expand Down Expand Up @@ -116,7 +116,7 @@ def get_spack_base(spack_base, config):


def check_call(commands, env=None, logger=None):
print_command = '\n '.join(commands.split('; '))
print_command = '\n '.join(commands.split(' && '))
if logger is None:
print('\n Running:\n {}\n'.format(print_command))
else:
Expand Down Expand Up @@ -155,7 +155,7 @@ def install_miniconda(conda_base, activate_base, logger):
else:
system = 'Linux'
miniconda = 'Mambaforge-{}-x86_64.sh'.format(system)
url = 'https://github.com/conda-forge/miniforge/releases/latest/download/{}'.format(miniconda)
url = 'https://github.com/conda-forge/miniforge/releases/latest/download/{}'.format(miniconda) # noqa: E501
print(url)
req = Request(url, headers={'User-Agent': 'Mozilla/5.0'})
f = urlopen(req)
Expand All @@ -171,11 +171,23 @@ def install_miniconda(conda_base, activate_base, logger):
backup_bashrc()

print('Doing initial setup\n')
commands = '{}; ' \
'conda config --add channels conda-forge; ' \
'conda config --set channel_priority strict; ' \
'conda install -y boa; ' \
'conda update -y --all; ' \

commands = '{} && ' \
'conda config --add channels conda-forge && ' \
'conda config --set channel_priority strict' \
''.format(activate_base)

check_call(commands, logger=logger)

commands = '{} && ' \
'conda remove -y boa'.format(activate_base)
try:
check_call(commands, logger=logger)
except subprocess.CalledProcessError:
pass

commands = '{} && ' \
'mamba update -y --all && ' \
'mamba init'.format(activate_base)

check_call(commands, logger=logger)
Expand Down