Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add additional flags to pip install command #860

Merged
merged 3 commits into from
Sep 23, 2024
Merged

Add additional flags to pip install command #860

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b112395
Add flags to `pip install` in quick start
altheaden Sep 17, 2024
3d67afb
Update `pip install` flags in conda files
altheaden Sep 23, 2024
d32d67e
Satisfy linter
altheaden Sep 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 67 additions & 50 deletions conda/bootstrap.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,8 +312,9 @@ def build_conda_env(env_type, recreate, mpi, conda_mpi, version,
# conda packages don't like dashes
version_conda = version.replace('-', '')
packages = f'{packages} "compass={version_conda}={mpi_prefix}_*"'
commands = f'{activate_base} && ' \
f'conda create -y -n {env_name} {channels} {packages}'
commands = \
f'{activate_base} && ' \
f'conda create -y -n {env_name} {channels} {packages}'
check_call(commands, logger=logger)
else:
if env_type == 'dev':
Expand All @@ -336,7 +337,7 @@ def build_conda_env(env_type, recreate, mpi, conda_mpi, version,
f'git submodule update --init alphaBetaLab &&' \
f'cd {source_path}/alphaBetaLab&& ' \
f'conda install -y --file dev-spec.txt && ' \
f'python -m pip install --no-deps -e .'
f'python -m pip install --no-deps --no-build-isolation -e .'
check_call(commands, logger=logger)

if recreate or update_jigsaw:
Expand All @@ -348,7 +349,7 @@ def build_conda_env(env_type, recreate, mpi, conda_mpi, version,
f'{activate_env} && ' \
f'cd {source_path} && ' \
f'rm -rf compass.egg-info && ' \
f'python -m pip install --no-deps -e .'
f'python -m pip install --no-deps --no-build-isolation -e .'
check_call(commands, logger=logger)

print('Installing pre-commit\n')
Expand Down Expand Up @@ -402,13 +403,13 @@ def build_jigsaw(activate_env, source_path, env_path, logger):
commands = \
f'{activate_env} && ' \
f'cd {source_path}/jigsaw-python && ' \
f'python -m pip install --no-deps -e . && ' \
f'python -m pip install --no-deps --no-build-isolation -e . && ' \
f'cp jigsawpy/_bin/* ${{CONDA_PREFIX}}/bin'
check_call(commands, logger=logger)

t1 = time.time()
total = int(t1 - t0 + 0.5)
message = f'JIGSAW install took {total:.1f} s.'
message = f'JIGSAW install took {total:.1f} s.' # noqa: 231
if logger is None:
print(message)
else:
Expand All @@ -424,11 +425,12 @@ def get_env_vars(machine, compiler, mpilib):
env_vars = 'export MPAS_EXTERNAL_LIBS=""\n'

if 'intel' in compiler and machine == 'anvil':
env_vars = f'{env_vars}' \
f'export I_MPI_CC=icc\n' \
f'export I_MPI_CXX=icpc\n' \
f'export I_MPI_F77=ifort\n' \
f'export I_MPI_F90=ifort\n'
env_vars = \
f'{env_vars}' \
f'export I_MPI_CC=icc\n' \
f'export I_MPI_CXX=icpc\n' \
f'export I_MPI_F77=ifort\n' \
f'export I_MPI_F90=ifort\n'

if machine.startswith('conda'):
# we're using parallelio so we don't have ADIOS support
Expand All @@ -442,9 +444,10 @@ def get_env_vars(machine, compiler, mpilib):
f'export MPAS_EXTERNAL_LIBS="${{MPAS_EXTERNAL_LIBS}} -lgomp"\n'

if mpilib == 'mvapich':
env_vars = f'{env_vars}' \
f'export MV2_ENABLE_AFFINITY=0\n' \
f'export MV2_SHOW_CPU_BINDING=1\n'
env_vars = \
f'{env_vars}' \
f'export MV2_ENABLE_AFFINITY=0\n' \
f'export MV2_SHOW_CPU_BINDING=1\n'

if machine.startswith('chicoma') or machine.startswith('pm'):
env_vars = \
Expand Down Expand Up @@ -525,8 +528,9 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901
f'"scorpio'
f'@{scorpio}+pnetcdf~timing+internal-timing~tools+malloc"')
# make sure scorpio, not esmf, libraries are linked
lib_path = f'{spack_branch_base}/var/spack/environments/' \
f'{spack_env}/.spack-env/view/lib'
lib_path = \
f'{spack_branch_base}/var/spack/environments/' \
f'{spack_env}/.spack-env/view/lib'
scorpio_lib_path = '$(spack find --format "{prefix}" scorpio)'
custom_spack = \
f'{custom_spack}' \
Expand Down Expand Up @@ -572,8 +576,9 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901
custom_spack=custom_spack, spack_mirror=spack_mirror)

# remove ESMC/ESMF include files that interfere with MPAS time keeping
include_path = f'{spack_branch_base}/var/spack/environments/' \
f'{spack_env}/.spack-env/view/include'
include_path = \
f'{spack_branch_base}/var/spack/environments/' \
f'{spack_env}/.spack-env/view/include'
for prefix in ['ESMC', 'esmf']:
files = glob.glob(os.path.join(include_path, f'{prefix}*'))
for filename in files:
Expand All @@ -587,10 +592,12 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901
include_e3sm_hdf5_netcdf=e3sm_hdf5_netcdf,
yaml_template=yaml_template)

spack_view = f'{spack_branch_base}/var/spack/environments/' \
f'{spack_env}/.spack-env/view'
env_vars = f'{env_vars}' \
f'export PIO={spack_view}\n'
spack_view = \
f'{spack_branch_base}/var/spack/environments/' \
f'{spack_env}/.spack-env/view'
env_vars = \
f'{env_vars}' \
f'export PIO={spack_view}\n'
if parallelio != 'None':
env_vars = \
f'{env_vars}' \
Expand Down Expand Up @@ -619,14 +626,16 @@ def build_spack_env(config, update_spack, machine, compiler, mpi, # noqa: C901
f'${{ALBANY_LINK_LIBS}} {stdcxx} {mpicxx}"\n'

if lapack != 'None':
env_vars = f'{env_vars}' \
f'export LAPACK={spack_view}\n' \
f'export USE_LAPACK=true\n'
env_vars = \
f'{env_vars}' \
f'export LAPACK={spack_view}\n' \
f'export USE_LAPACK=true\n'

if petsc != 'None':
env_vars = f'{env_vars}' \
f'export PETSC={spack_view}\n' \
f'export USE_PETSC=true\n'
env_vars = \
f'{env_vars}' \
f'export PETSC={spack_view}\n' \
f'export USE_PETSC=true\n'

return spack_branch_base, spack_script, env_vars

Expand All @@ -636,7 +645,7 @@ def set_ld_library_path(spack_branch_base, spack_env, logger):
f'source {spack_branch_base}/share/spack/setup-env.sh && ' \
f'spack env activate {spack_env} && ' \
f'spack config add modules:prefix_inspections:lib:[LD_LIBRARY_PATH] && ' \
f'spack config add modules:prefix_inspections:lib64:[LD_LIBRARY_PATH]' # noqa: E501
f'spack config add modules:prefix_inspections:lib64:[LD_LIBRARY_PATH]' # noqa: E501,E231
check_call(commands, logger=logger)


Expand All @@ -658,21 +667,26 @@ def write_load_compass(template_path, activ_path, conda_base, env_type,
script_filename = '{}/{}{}.sh'.format(activ_path, prefix, activ_suffix)

if not env_only:
env_vars = f'{env_vars}\n' \
f'export USE_PIO2=true'
env_vars = \
f'{env_vars}\n' \
f'export USE_PIO2=true'
if without_openmp:
env_vars = f'{env_vars}\n' \
f'export OPENMP=false'
env_vars = \
f'{env_vars}\n' \
f'export OPENMP=false'
else:
env_vars = f'{env_vars}\n' \
f'export OPENMP=true'
env_vars = \
f'{env_vars}\n' \
f'export OPENMP=true'

env_vars = f'{env_vars}\n' \
f'export HDF5_USE_FILE_LOCKING=FALSE\n' \
f'export LOAD_COMPASS_ENV={script_filename}'
env_vars = \
f'{env_vars}\n' \
f'export HDF5_USE_FILE_LOCKING=FALSE\n' \
f'export LOAD_COMPASS_ENV={script_filename}'
if machine is not None and not machine.startswith('conda'):
env_vars = f'{env_vars}\n' \
f'export COMPASS_MACHINE={machine}'
env_vars = \
f'{env_vars}\n' \
f'export COMPASS_MACHINE={machine}'

filename = f'{template_path}/load_compass.template'
with open(filename, 'r') as f:
Expand All @@ -688,7 +702,8 @@ def write_load_compass(template_path, activ_path, conda_base, env_type,
mkdir -p conda/logs
echo Reinstalling compass package in edit mode...
rm -rf compass.egg-info
python -m pip install --no-deps -e . &> conda/logs/install_compass.log
python -m pip install --no-deps --no-build-isolation -e . \\
&> conda/logs/install_compass.log
echo Done.
echo
fi
Expand Down Expand Up @@ -1089,11 +1104,12 @@ def main(): # noqa: C901

if local_mache:
print('Install local mache\n')
commands = f'source {conda_base}/etc/profile.d/conda.sh && ' \
f'conda activate {conda_env_name} && ' \
f'cd ../build_mache/mache && ' \
f'conda install -y --file spec-file.txt && ' \
f'python -m pip install --no-deps .'
commands = \
f'source {conda_base}/etc/profile.d/conda.sh && ' \
f'conda activate {conda_env_name} && ' \
f'cd ../build_mache/mache && ' \
f'conda install -y --file spec-file.txt && ' \
f'python -m pip install --no-deps --no-build-isolation .'
check_call(commands, logger=logger)

previous_conda_env = conda_env_name
Expand All @@ -1109,10 +1125,11 @@ def main(): # noqa: C901
config, args.update_spack, machine, compiler, mpi,
spack_env, spack_base, spack_template_path, env_vars,
args.tmpdir, logger)
spack_script = f'echo Loading Spack environment...\n' \
f'{spack_script}\n' \
f'echo Done.\n' \
f'echo\n'
spack_script = \
f'echo Loading Spack environment...\n' \
f'{spack_script}\n' \
f'echo Done.\n' \
f'echo\n'
else:
env_vars = \
f'{env_vars}' \
Expand Down
29 changes: 16 additions & 13 deletions conda/configure_compass_env.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,9 @@ def bootstrap(activate_install_env, source_path, local_conda_build):

print('Creating the compass conda environment\n')
bootstrap_command = f'{source_path}/conda/bootstrap.py'
command = f'{activate_install_env} && ' \
f'{bootstrap_command} {" ".join(sys.argv[1:])}'
command = \
f'{activate_install_env} && ' \
f'{bootstrap_command} {" ".join(sys.argv[1:])}'
if local_conda_build is not None:
command = f'{command} --local_conda_build {local_conda_build}'
check_call(command)
Expand All @@ -51,8 +52,9 @@ def setup_install_env(env_name, activate_base, use_local, logger, recreate,
f'conda create -y -n {env_name} {channels} {packages}'
else:
print('Updating conda environment for installing compass\n')
commands = f'{activate_base} && ' \
f'conda install -y -n {env_name} {channels} {packages}'
commands = \
f'{activate_base} && ' \
f'conda install -y -n {env_name} {channels} {packages}'

check_call(commands, logger=logger)

Expand Down Expand Up @@ -107,15 +109,16 @@ def main():

if local_mache:
print('Clone and install local mache\n')
commands = f'{activate_install_env} && ' \
f'rm -rf conda/build_mache && ' \
f'mkdir -p conda/build_mache && ' \
f'cd conda/build_mache && ' \
f'git clone -b {args.mache_branch} ' \
f'[email protected]:{args.mache_fork}.git mache && ' \
f'cd mache && ' \
f'conda install -y --file spec-file.txt && ' \
f'python -m pip install --no-deps .'
commands = \
f'{activate_install_env} && ' \
f'rm -rf conda/build_mache && ' \
f'mkdir -p conda/build_mache && ' \
f'cd conda/build_mache && ' \
f'git clone -b {args.mache_branch} ' \
f'[email protected]:{args.mache_fork}.git mache && ' \
f'cd mache && ' \
f'conda install -y --file spec-file.txt && ' \
f'python -m pip install --no-deps --no-build-isolation .' # noqa: E231,E501

check_call(commands, logger=logger)

Expand Down
4 changes: 2 additions & 2 deletions docs/developers_guide/quick_start.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -382,7 +382,7 @@ switch branches.

.. code-block:: bash

python -m pip install -e .
python -m pip install --no-deps --no-build-isolation -e .

The activation script will do this automatically when you source it in
the root directory of your compass branch. The activation script will also
Expand Down Expand Up @@ -446,7 +446,7 @@ to the current directory.

.. code-block:: bash

python -m pip install -e .
python -m pip install --no-deps --no-build-isolation -e .

This will be substantially faster than rerunning
``./conda/configure_compass_env.py ...`` but at the risk that dependencies are
Expand Down
Loading