Make builddependencies iterable.

easybuild/framework/easyblock.py

@@ -219,6 +219,7 @@ def __init__(self, ec):
 
         # keep track of initial environment we start in, so we can restore it if needed
         self.initial_environ = copy.deepcopy(os.environ)
+        self.reset_environ = None
         self.tweaked_env_vars = {}
 
         # should we keep quiet?
@@ -863,10 +864,22 @@ def make_builddir(self):
         # otherwise we wipe the already partially populated installation directory,
         # see https://github.com/easybuilders/easybuild-framework/issues/2556
         if not (self.build_in_installdir and self.iter_idx > 0):
+            # make sure we no longer sit in the build directory before cleaning it.
+            change_dir(self.orig_workdir)
             self.make_dir(self.builddir, self.cfg['cleanupoldbuild'])
 
         trace_msg("build dir: %s" % self.builddir)
 
+    def reset_changes(self):
+        """
+        Reset environment
+        """
+        env.reset_changes()
+        if self.reset_environ is None:
+            self.reset_environ = copy.deepcopy(os.environ)
+        else:
+            restore_env(self.reset_environ)
+
     def gen_installdir(self):
         """
         Generate the name of the installation directory.
@@ -1478,7 +1491,11 @@ def handle_iterate_opts(self):
         # this will only be done during first iteration, since after that the options won't be lists anymore
         for opt in ITERATE_OPTIONS:
             # keep track of list, supply first element as first option to handle
-            if isinstance(self.cfg[opt], (list, tuple)):
+            if isinstance(self.cfg[opt], (list, tuple)) and self.cfg[opt] != [[]]:
+                if opt.startswith('iterate_') and self.iter_idx == 0:
+                    # save original common value (e.g. builddependencies)
+                    commonopt = opt[len('iterate_'):]
+                    self.iter_opts[commonopt] = self.cfg[commonopt]
                 self.iter_opts[opt] = self.cfg[opt]  # copy
                 self.log.debug("Found list for %s: %s", opt, self.iter_opts[opt])
 
@@ -1487,11 +1504,15 @@ def handle_iterate_opts(self):
 
         # pop first element from all *_list options as next value to use
         for opt in self.iter_opts:
-            if len(self.iter_opts[opt]) > self.iter_idx:
-                self.cfg[opt] = self.iter_opts[opt][self.iter_idx]
-            else:
-                self.cfg[opt] = ''  # empty list => empty option as next value
-            self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt])))
+            if opt in ITERATE_OPTIONS:
+                if opt.startswith('iterate_'):
+                    commonopt = opt[len('iterate_'):]
+                    self.cfg[commonopt] = self.iter_opts[opt][self.iter_idx] + self.iter_opts[commonopt]
+                elif len(self.iter_opts[opt]) > self.iter_idx:
+                    self.cfg[opt] = self.iter_opts[opt][self.iter_idx]
+                else:
+                    self.cfg[opt] = ''  # empty list => empty option as next value
+                self.log.debug("Next value for %s: %s" % (opt, str(self.cfg[opt])))
 
         # re-enable templating before self.cfg values are used
         self.cfg.enable_templating = True
@@ -1822,13 +1843,15 @@ def prepare_step(self, start_dir=True):
 
         # list of paths to include in RPATH filter;
         # only include builddir if we're not building in installation directory
+        self.rpath_filter_dirs = []
         self.rpath_filter_dirs.append(tempfile.gettempdir())
         if not self.build_in_installdir:
             self.rpath_filter_dirs.append(self.builddir)
 
         # always include '<installdir>/lib', '<installdir>/lib64', $ORIGIN, $ORIGIN/../lib and $ORIGIN/../lib64
         # $ORIGIN will be resolved by the loader to be the full path to the executable or shared object
         # see also https://linux.die.net/man/8/ld-linux;
+        self.rpath_include_dirs = []
         self.rpath_include_dirs.append(os.path.join(self.installdir, 'lib'))
         self.rpath_include_dirs.append(os.path.join(self.installdir, 'lib64'))
         self.rpath_include_dirs.append('$ORIGIN')
@@ -2660,7 +2683,7 @@ def get_step(tag, descr, substeps, skippable, initial=True):
         ready_substeps = [
             (False, lambda x: x.check_readiness_step),
             (True, lambda x: x.make_builddir),
-            (True, lambda x: env.reset_changes),
+            (True, lambda x: x.reset_changes),
             (True, lambda x: x.handle_iterate_opts),
         ]
 
@@ -2678,14 +2701,6 @@ def source_step_spec(initial):
             """Return source step specified."""
             return get_step(SOURCE_STEP, "unpacking", source_substeps, True, initial=initial)
 
-        def prepare_step_spec(initial):
-            """Return prepare step specification."""
-            if initial:
-                substeps = [lambda x: x.prepare_step]
-            else:
-                substeps = [lambda x: x.guess_start_dir]
-            return (PREPARE_STEP, 'preparing', substeps, False)
-
         install_substeps = [
             (False, lambda x: x.stage_install_step),
             (False, lambda x: x.make_installdir),
@@ -2696,10 +2711,11 @@ def install_step_spec(initial):
             """Return install step specification."""
             return get_step(INSTALL_STEP, "installing", install_substeps, True, initial=initial)
 
-        # format for step specifications: (stop_name: (description, list of functions, skippable))
+        # format for step specifications: (step_name, description, list of functions, skippable)
 
         # core steps that are part of the iterated loop
         patch_step_spec = (PATCH_STEP, 'patching', [lambda x: x.patch_step], True)
+        prepare_step_spec = (PREPARE_STEP, 'preparing', [lambda x: x.prepare_step], False)
         configure_step_spec = (CONFIGURE_STEP, 'configuring', [lambda x: x.configure_step], True)
         build_step_spec = (BUILD_STEP, 'building', [lambda x: x.build_step], True)
         test_step_spec = (TEST_STEP, 'testing', [lambda x: x.test_step], True)
@@ -2710,7 +2726,7 @@ def install_step_spec(initial):
             ready_step_spec(True),
             source_step_spec(True),
             patch_step_spec,
-            prepare_step_spec(True),
+            prepare_step_spec,
             configure_step_spec,
             build_step_spec,
             test_step_spec,
@@ -2723,7 +2739,7 @@ def install_step_spec(initial):
             ready_step_spec(False),
             source_step_spec(False),
             patch_step_spec,
-            prepare_step_spec(False),
+            prepare_step_spec,
             configure_step_spec,
             build_step_spec,
             test_step_spec,

easybuild/framework/easyconfig/default.py

@@ -142,6 +142,7 @@
     # DEPENDENCIES easyconfig parameters
     'allow_system_deps': [[], "Allow listed system dependencies (format: (<name>, <version>))", DEPENDENCIES],
     'builddependencies': [[], "List of build dependencies", DEPENDENCIES],
+    'iterate_builddependencies': [[[]], "Iterated list of list of build dependencies", DEPENDENCIES],
     'dependencies': [[], "List of dependencies", DEPENDENCIES],
     'hiddendependencies': [[], "List of dependencies available as hidden modules", DEPENDENCIES],
     'osdependencies': [[], "OS dependencies that should be present on the system", DEPENDENCIES],

easybuild/framework/easyconfig/easyconfig.py

@@ -83,7 +83,8 @@
 MANDATORY_PARAMS = ['name', 'version', 'homepage', 'description', 'toolchain']
 
 # set of configure/build/install options that can be provided as lists for an iterated build
-ITERATE_OPTIONS = ['preconfigopts', 'configopts', 'prebuildopts', 'buildopts', 'preinstallopts', 'installopts']
+ITERATE_OPTIONS = ['iterate_builddependencies',
+                   'preconfigopts', 'configopts', 'prebuildopts', 'buildopts', 'preinstallopts', 'installopts']
 
 # name of easyconfigs archive subdirectory
 EASYCONFIGS_ARCHIVE_DIR = '__archive__'
@@ -174,7 +175,7 @@ def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains,
 
 @toolchain_hierarchy_cache
 def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False):
-    """
+    r"""
     Determine list of subtoolchains for specified parent toolchain.
     Result starts with the most minimal subtoolchains first, ends with specified toolchain.
 
@@ -543,6 +544,8 @@ def parse(self):
         # it's important that templating is still disabled at this stage!
         self.log.info("Parsing dependency specifications...")
         self['builddependencies'] = [self._parse_dependency(dep, build_only=True) for dep in self['builddependencies']]
+        self['iterate_builddependencies'] = [[self._parse_dependency(dep, build_only=True) for dep in x]
+                                             for x in self['iterate_builddependencies']]
         self['dependencies'] = [self._parse_dependency(dep) for dep in self['dependencies']]
         self['hiddendependencies'] = [self._parse_dependency(dep, hidden=True) for dep in self['hiddendependencies']]
 
@@ -665,7 +668,7 @@ def validate_iterate_opts_lists(self):
                 raise EasyBuildError("%s not available in self.cfg (anymore)?!", opt)
 
             # keep track of list, supply first element as first option to handle
-            if isinstance(self[opt], (list, tuple)):
+            if isinstance(self[opt], (list, tuple)) or opt.endswith('_iteropt'):
                 opt_counts.append((opt, len(self[opt])))
 
         # make sure that options that specify lists have the same length
@@ -1096,10 +1099,15 @@ def _finalize_dependencies(self):
         for key in DEPENDENCY_PARAMETERS:
             # loop over a *copy* of dependency dicts (with resolved templates);
             # to update the original dep dict, we need to index with idx into self._config[key][0]...
-            for idx, dep in enumerate(self[key]):
+            val = self[key]
+            orig_val = self._config[key][0]
+            if key == 'iterate_builddependencies':
+                val = [dep for deps in val for dep in deps]
+                orig_val = [dep for deps in orig_val for dep in deps]
+            for idx, dep in enumerate(val):
 
                 # reference to original dep dict, this is the one we should be updating
-                orig_dep = self._config[key][0][idx]
+                orig_dep = orig_val[idx]
 
                 if filter_deps and orig_dep['name'] in filter_deps:
                     self.log.debug("Skipping filtered dependency %s when finalising dependencies", orig_dep['name'])

easybuild/framework/easyconfig/format/format.py

@@ -50,7 +50,7 @@
 FORMAT_VERSION_REGEXP = re.compile(r'^#\s+%s\s*(?P<major>\d+)\.(?P<minor>\d+)\s*$' % FORMAT_VERSION_KEYWORD, re.M)
 FORMAT_DEFAULT_VERSION = EasyVersion('1.0')
 
-DEPENDENCY_PARAMETERS = ['builddependencies', 'dependencies', 'hiddendependencies']
+DEPENDENCY_PARAMETERS = ['iterate_builddependencies', 'builddependencies', 'dependencies', 'hiddendependencies']
 
 # values for these keys will not be templated in dump()
 EXCLUDED_KEYS_REPLACE_TEMPLATES = ['description', 'easyblock', 'exts_list', 'homepage', 'name', 'toolchain',

easybuild/framework/easyconfig/format/one.py

@@ -52,6 +52,7 @@
 # dependency parameters always need to be reformatted, to correctly deal with dumping parsed dependencies
 REFORMAT_FORCED_PARAMS = ['sanity_check_paths'] + DEPENDENCY_PARAMETERS
 REFORMAT_SKIPPED_PARAMS = ['toolchain', 'toolchainopts']
+REFORMAT_LIST_OF_LISTS_OF_TUPLES = ['iterate_builddependencies']
 REFORMAT_THRESHOLD_LENGTH = 100  # only reformat lines that would be longer than this amount of characters
 REFORMAT_ORDERED_ITEM_KEYS = {
     'sanity_check_paths': ['files', 'dirs'],
@@ -140,6 +141,7 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
         # note: this does not take into account the parameter name + '=', only the value
         line_too_long = len(param_strval) + addlen > REFORMAT_THRESHOLD_LENGTH
         forced = param_name in REFORMAT_FORCED_PARAMS
+        list_of_lists_of_tuples = param_name in REFORMAT_LIST_OF_LISTS_OF_TUPLES
 
         if param_name in REFORMAT_SKIPPED_PARAMS:
             self.log.info("Skipping reformatting value for parameter '%s'", param_name)
@@ -170,9 +172,15 @@ def _reformat_line(self, param_name, param_val, outer=False, addlen=0):
                     for item in param_val:
                         comment = self._get_item_comments(param_name, item).get(str(item), '')
                         addlen = addlen + len(INDENT_4SPACES) + len(comment)
+                        if isinstance(item, list) and list_of_lists_of_tuples:
+                            itemstr = '[' + (",\n " + INDENT_4SPACES).join([
+                                self._reformat_line(param_name, subitem, outer=True, addlen=addlen)
+                                for subitem in item]) + ']'
+                        else:
+                            itemstr = self._reformat_line(param_name, item, addlen=addlen)
                         res += item_tmpl % {
                             'comment': comment,
-                            'item': self._reformat_line(param_name, item, addlen=addlen)
+                            'item': itemstr
                         }
 
                 # end with closing character: ], ), }
@@ -234,7 +242,9 @@ def _find_defined_params(self, ecfg, keyset, default_values, templ_const, templ_
 
                 if val != default_values[key]:
                     # dependency easyconfig parameters were parsed, so these need special care to 'unparse' them
-                    if key in DEPENDENCY_PARAMETERS:
+                    if key == 'iterate_builddependencies':
+                        valstr = [[dump_dependency(d, ecfg['toolchain']) for d in dep] for dep in val]
+                    elif key in DEPENDENCY_PARAMETERS:
                         valstr = [dump_dependency(d, ecfg['toolchain']) for d in val]
                     elif key == 'toolchain':
                         valstr = "{'name': '%(name)s', 'version': '%(version)s'}" % ecfg[key]

easybuild/framework/easyconfig/tweak.py

@@ -832,9 +832,14 @@ def map_easyconfig_to_target_tc_hierarchy(ec_spec, toolchain_mapping, targetdir=
     for key in DEPENDENCY_PARAMETERS:
         # loop over a *copy* of dependency dicts (with resolved templates);
         # to update the original dep dict, we need to index with idx into self._config[key][0]...
-        for idx, dep in enumerate(parsed_ec['ec'][key]):
+        val = parced_ec['ec'][key]
+        orig_val = parced_ec['ec']._config[key][0]
+        if key == 'iterate_builddependencies':
+            val = [dep for deps in val for dep in deps]
+            orig_val = [dep for deps in orig_val for dep in deps]
+        for idx, dep in enumerate(val):
             # reference to original dep dict, this is the one we should be updating
-            orig_dep = parsed_ec['ec']._config[key][0][idx]
+            orig_dep = orig_val[idx]
             # skip dependencies that are marked as external modules
             if dep['external_module']:
                 continue

test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-iter.eb

@@ -5,19 +5,26 @@ versionsuffix = '-iter'
 homepage = 'https://easybuilders.github.io/easybuild'
 description = "Toy C program, 100% toy."
 
-toolchain = {'name': 'dummy', 'version': 'dummy'}
+toolchain = {'name': 'dummy', 'version': ''}
 
 sources = [SOURCE_TAR_GZ]
 patches = ['toy-0.0_fix-silly-typo-in-printf-statement.patch']
 
+iterate_builddependencies = [
+    [('GCC', '6.4.0-2.28'),
+     ('OpenMPI', '2.1.2', '', ('GCC', '6.4.0-2.28'))],
+    [('GCC', '6.4.0-2.28')],
+    [('GCC', '7.3.0-2.30')],
+]
+
 buildopts = [
     '',
-    "-O2; mv %(name)s toy_O2",
-    "-O1; mv %(name)s toy_O1",
+    "-O2; mv %(name)s toy_O2_$EBVERSIONGCC",
+    "-O1; mv %(name)s toy_O1_$EBVERSIONGCC",
 ]
 
 sanity_check_paths = {
-    'files': [('bin/yot', 'bin/toy'), 'bin/toy_O1', 'bin/toy_O2'],
+    'files': [('bin/yot', 'bin/toy'), 'bin/toy_O1_7.3.0-2.30', 'bin/toy_O2_6.4.0-2.28'],
     'dirs': ['bin'],
 }
 

test/framework/toy_build.py

@@ -1796,7 +1796,7 @@ def test_toy_iter(self):
         topdir = os.path.abspath(os.path.dirname(__file__))
         toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-iter.eb')
 
-        expected_buildopts = ['', '-O2; mv %(name)s toy_O2', '-O1; mv %(name)s toy_O1']
+        expected_buildopts = ['', '-O2; mv %(name)s toy_O2_$EBVERSIONGCC', '-O1; mv %(name)s toy_O1_$EBVERSIONGCC']
 
         for extra_args in [None, ['--minimal-toolchains']]:
             # sanity check will make sure all entries in buildopts list were taken into account