Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make builddependencies iterable. #2741

Merged
merged 33 commits into from
Mar 15, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
08adff3
Add iterate_builddependencies easyconfig option.
bartoldeman Jan 30, 2019
e723d4b
Make docstring raw to make Hound-ci happy.
bartoldeman Jan 30, 2019
fc0b990
Fix dump of exts_list by special casing iterate_builddependencies.
bartoldeman Jan 31, 2019
26259c7
Modify toy-0.0-iter dep to have both easyconfig and module available.
bartoldeman Jan 31, 2019
66ce0c7
Restore initial environment when resetting the environment.
bartoldeman Jan 31, 2019
8c91557
In reset environment for iterate reset to the env first encountered h…
bartoldeman Jan 31, 2019
8dd69f3
Avoid temporary modification of builddeps for _finalize_dependencies()
bartoldeman Jan 31, 2019
5c3beb2
Merge iterate_builddependencies into DEPENDENCY_PARAMETERS.
bartoldeman Jan 31, 2019
a22dfb2
Fix typo in parced.
bartoldeman Jan 31, 2019
f976245
Fix check for iterate_* keys in validate_iterate_opts_lists
bartoldeman Jan 31, 2019
f074e51
Keep iterate_builddependencies completely separate from builddependen…
bartoldeman Jan 31, 2019
f5def9e
Use flatten function instead of list comprehension to flatten.
bartoldeman Jan 31, 2019
3bf5873
Properly unload modules for iterate instead of messing with the envir…
bartoldeman Jan 31, 2019
641421b
Make Hound happy about comment.
bartoldeman Jan 31, 2019
d9f06e3
Merge remote-tracking branch 'github_hpcugent/develop' into iterable-…
bartoldeman Feb 11, 2019
ed6f73f
Iterate builddependencies instead of iterate_builddependencies.
bartoldeman Feb 14, 2019
8d6f3bb
Hound-ci changes.
bartoldeman Feb 14, 2019
b51ad11
yaml easyconfigs builddeps already a list of dicts, fixing that.
bartoldeman Feb 15, 2019
fcdc90b
add & use EasyConfig.get_ref method + code cleanup & minor style fixes
boegel Feb 20, 2019
96b0df0
Merge pull request #14 from boegel/iterable-builddep
bartoldeman Feb 20, 2019
4ff5722
Restore environment when iterating instead of use modules.
bartoldeman Mar 1, 2019
aa29163
Introduce "iterating" easyconfig field.
bartoldeman Mar 14, 2019
72c48cd
Correct boolean check for iterating.
bartoldeman Mar 15, 2019
e8b393b
Remove duplicates in flattened lists of builddependencies.
bartoldeman Mar 15, 2019
b0579ad
Fix typo in parameters spotted by @boegel.
bartoldeman Mar 15, 2019
248dc25
Add comment about reset_environ.
bartoldeman Mar 15, 2019
bddaa0d
rename 'EasyBlock.reset_changes' to 'EasyBlock.reset_env'
boegel Mar 15, 2019
7c34a92
use 'nub' to filter out duplicates from flattened list of build depen…
boegel Mar 15, 2019
8e4299b
Merge pull request #15 from boegel/iterable-builddep
bartoldeman Mar 15, 2019
96c253b
can't use nub to filter out duplicates in flattened list of build dep…
boegel Mar 15, 2019
e9bcc8f
Merge pull request #16 from boegel/iterable-builddep
bartoldeman Mar 15, 2019
4945275
fix filtering of duplicates in builddependencies method (@boegel shou…
boegel Mar 15, 2019
33c46df
Merge pull request #17 from boegel/iterable-builddep
bartoldeman Mar 15, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 50 additions & 38 deletions easybuild/framework/easyblock.py
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,7 @@ def __init__(self, ec):

# keep track of initial environment we start in, so we can restore it if needed
self.initial_environ = copy.deepcopy(os.environ)
self.reset_environ = None
self.tweaked_env_vars = {}

# should we keep quiet?
Expand Down Expand Up @@ -457,9 +458,7 @@ def fetch_extension_sources(self, skip_checksums=False):
Find source file for extensions.
"""
exts_sources = []
self.cfg.enable_templating = False
exts_list = self.cfg['exts_list']
self.cfg.enable_templating = True
exts_list = self.cfg.get_ref('exts_list')

if self.dry_run:
self.dry_run_msg("\nList of sources/patches for extensions:")
Expand All @@ -477,9 +476,7 @@ def fetch_extension_sources(self, skip_checksums=False):

# make sure we grab *raw* dict of default options for extension,
# since it may use template values like %(name)s & %(version)s
self.cfg.enable_templating = False
ext_options = copy.deepcopy(self.cfg['exts_default_options'])
self.cfg.enable_templating = True
ext_options = copy.deepcopy(self.cfg.get_ref('exts_default_options'))

def_src_tmpl = "%(name)s-%(version)s.tar.gz"

Expand Down Expand Up @@ -863,10 +860,25 @@ def make_builddir(self):
# otherwise we wipe the already partially populated installation directory,
# see https://github.com/easybuilders/easybuild-framework/issues/2556
if not (self.build_in_installdir and self.iter_idx > 0):
# make sure we no longer sit in the build directory before cleaning it.
change_dir(self.orig_workdir)
self.make_dir(self.builddir, self.cfg['cleanupoldbuild'])

trace_msg("build dir: %s" % self.builddir)

def reset_env(self):
"""
Reset environment.
When iterating over builddependencies, every time we start a new iteration
we need to restore the environment to where it was before the relevant modules
were loaded.
"""
env.reset_changes()
if self.reset_environ is None:
self.reset_environ = copy.deepcopy(os.environ)
bartoldeman marked this conversation as resolved.
Show resolved Hide resolved
else:
restore_env(self.reset_environ)

def gen_installdir(self):
"""
Generate the name of the installation directory.
Expand Down Expand Up @@ -1371,10 +1383,8 @@ def skip_extensions(self):
- use this to detect existing extensions and to remove them from self.exts
- based on initial R version
"""
# disabling templating is required here to support legacy string templates like name/version
self.cfg.enable_templating = False
exts_filter = self.cfg['exts_filter']
self.cfg.enable_templating = True
# obtaining untemplated reference value is required here to support legacy string templates like name/version
exts_filter = self.cfg.get_ref('exts_filter')

if not exts_filter or len(exts_filter) == 0:
raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig")
Expand Down Expand Up @@ -1471,21 +1481,25 @@ def handle_iterate_opts(self):
"""Handle options relevant during iterated part of build/install procedure."""

# disable templating in this function, since we're messing about with values in self.cfg
prev_enable_templating = self.cfg.enable_templating
self.cfg.enable_templating = False

# handle configure/build/install options that are specified as lists
# tell easyconfig we are iterating (used by dependencies() and builddependencies())
self.cfg.iterating = True

# handle configure/build/install options that are specified as lists (+ perhaps builddependencies)
# set first element to be used, keep track of list in self.iter_opts
# this will only be done during first iteration, since after that the options won't be lists anymore
for opt in ITERATE_OPTIONS:
# only needs to be done during first iteration, since after that the options won't be lists anymore
if self.iter_idx == 0:
# keep track of list, supply first element as first option to handle
if isinstance(self.cfg[opt], (list, tuple)):
for opt in self.cfg.iterate_options:
self.iter_opts[opt] = self.cfg[opt] # copy
self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt])

if self.iter_opts:
self.log.info("Current iteration index: %s", self.iter_idx)

# pop first element from all *_list options as next value to use
# pop first element from all iterative easyconfig parameters as next value to use
for opt in self.iter_opts:
if len(self.iter_opts[opt]) > self.iter_idx:
self.cfg[opt] = self.iter_opts[opt][self.iter_idx]
Expand All @@ -1494,22 +1508,26 @@ def handle_iterate_opts(self):
self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt])))

# re-enable templating before self.cfg values are used
self.cfg.enable_templating = True
self.cfg.enable_templating = prev_enable_templating

# prepare for next iteration (if any)
self.iter_idx += 1

def restore_iterate_opts(self):
"""Restore options that were iterated over"""
# disable templating, since we're messing about with values in self.cfg
prev_enable_templating = self.cfg.enable_templating
self.cfg.enable_templating = False

for opt in self.iter_opts:
self.cfg[opt] = self.iter_opts[opt]
self.log.debug("Restored value of '%s' that was iterated over: %s", opt, self.cfg[opt])

# tell easyconfig we are no longer iterating (used by dependencies() and builddependencies())
self.cfg.iterating = False

# re-enable templating before self.cfg values are used
self.cfg.enable_templating = True
self.cfg.enable_templating = prev_enable_templating

def det_iter_cnt(self):
"""Determine iteration count based on configure/build/install options that may be lists."""
Expand Down Expand Up @@ -1822,25 +1840,28 @@ def prepare_step(self, start_dir=True):

# list of paths to include in RPATH filter;
# only include builddir if we're not building in installation directory
self.rpath_filter_dirs.append(tempfile.gettempdir())
self.rpath_filter_dirs = [tempfile.gettempdir()]
if not self.build_in_installdir:
self.rpath_filter_dirs.append(self.builddir)

# always include '<installdir>/lib', '<installdir>/lib64', $ORIGIN, $ORIGIN/../lib and $ORIGIN/../lib64
# $ORIGIN will be resolved by the loader to be the full path to the executable or shared object
# see also https://linux.die.net/man/8/ld-linux;
self.rpath_include_dirs.append(os.path.join(self.installdir, 'lib'))
self.rpath_include_dirs.append(os.path.join(self.installdir, 'lib64'))
self.rpath_include_dirs.append('$ORIGIN')
self.rpath_include_dirs.append('$ORIGIN/../lib')
self.rpath_include_dirs.append('$ORIGIN/../lib64')
self.rpath_include_dirs = [
os.path.join(self.installdir, 'lib'),
os.path.join(self.installdir, 'lib64'),
'$ORIGIN',
'$ORIGIN/../lib',
'$ORIGIN/../lib64',
]

# prepare toolchain: load toolchain module and dependencies, set up build environment
self.toolchain.prepare(self.cfg['onlytcmod'], deps=self.cfg.dependencies(), silent=self.silent,
rpath_filter_dirs=self.rpath_filter_dirs, rpath_include_dirs=self.rpath_include_dirs)

# keep track of environment variables that were tweaked and need to be restored after environment got reset
# $TMPDIR may be tweaked for OpenMPI 2.x, which doesn't like long $TMPDIR paths...
self.tweaked_env_vars = {}
for var in ['TMPDIR']:
if os.environ.get(var) != self.initial_environ.get(var):
self.tweaked_env_vars[var] = os.environ.get(var)
Expand Down Expand Up @@ -2660,7 +2681,7 @@ def get_step(tag, descr, substeps, skippable, initial=True):
ready_substeps = [
(False, lambda x: x.check_readiness_step),
(True, lambda x: x.make_builddir),
(True, lambda x: env.reset_changes),
(True, lambda x: x.reset_env),
(True, lambda x: x.handle_iterate_opts),
]

Expand All @@ -2678,14 +2699,6 @@ def source_step_spec(initial):
"""Return source step specified."""
return get_step(SOURCE_STEP, "unpacking", source_substeps, True, initial=initial)

def prepare_step_spec(initial):
"""Return prepare step specification."""
if initial:
substeps = [lambda x: x.prepare_step]
else:
substeps = [lambda x: x.guess_start_dir]
return (PREPARE_STEP, 'preparing', substeps, False)

install_substeps = [
(False, lambda x: x.stage_install_step),
(False, lambda x: x.make_installdir),
Expand All @@ -2696,10 +2709,11 @@ def install_step_spec(initial):
"""Return install step specification."""
return get_step(INSTALL_STEP, "installing", install_substeps, True, initial=initial)

# format for step specifications: (stop_name: (description, list of functions, skippable))
# format for step specifications: (step_name, description, list of functions, skippable)

# core steps that are part of the iterated loop
patch_step_spec = (PATCH_STEP, 'patching', [lambda x: x.patch_step], True)
prepare_step_spec = (PREPARE_STEP, 'preparing', [lambda x: x.prepare_step], False)
configure_step_spec = (CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step], True)
build_step_spec = (BUILD_STEP, 'building', [lambda x: x.build_step], True)
test_step_spec = (TEST_STEP, 'testing', [lambda x: x.test_step], True)
Expand All @@ -2710,7 +2724,7 @@ def install_step_spec(initial):
ready_step_spec(True),
source_step_spec(True),
patch_step_spec,
prepare_step_spec(True),
prepare_step_spec,
configure_step_spec,
build_step_spec,
test_step_spec,
Expand All @@ -2723,7 +2737,7 @@ def install_step_spec(initial):
ready_step_spec(False),
source_step_spec(False),
patch_step_spec,
prepare_step_spec(False),
prepare_step_spec,
configure_step_spec,
build_step_spec,
test_step_spec,
Expand Down Expand Up @@ -3270,9 +3284,7 @@ def make_checksum_lines(checksums, indent_level):

# make sure we grab *raw* dict of default options for extension,
# since it may use template values like %(name)s & %(version)s
app.cfg.enable_templating = False
exts_default_options = app.cfg['exts_default_options']
app.cfg.enable_templating = True
exts_default_options = app.cfg.get_ref('exts_default_options')

for key, val in sorted(ext_options.items()):
if key != 'checksums' and val != exts_default_options.get(key):
Expand Down
Loading