Allow multiple subtoolchains and optional toolchains.

This PR replaces easybuilders#2234.
subtoolchains are now searched using a breadth-first-search, which
allows multiple subtoolchains per toolchain.

Some subtoolchains such as golf and golfc are now marked OPTIONAL
since they do not have to exist as modules, and can be skipped.

Since those optional subtoolchains cannot usually be found via a
dependency search (as toolchain easyconfigs do not typically have
subtoolchains as dependencies)
try to search for subtoolchain/version with the same version as
the parent. E.g. goolfc/2.6.10 searches for golfc/2.6.10, and
golfc/2.6.10 searches for golf/2.6.10 and gcccuda/2.6.10.

easybuild/framework/easyconfig/easyconfig.py

@@ -148,14 +148,22 @@ def get_toolchain_hierarchy(parent_toolchain):
     # obtain list of all possible subtoolchains
     _, all_tc_classes = search_toolchain('')
     subtoolchains = dict((tc_class.NAME, getattr(tc_class, 'SUBTOOLCHAIN', None)) for tc_class in all_tc_classes)
-
-    current_tc_name, current_tc_version = parent_toolchain['name'], parent_toolchain['version']
-    subtoolchain_name, subtoolchain_version = subtoolchains[current_tc_name], None
+    optional_toolchains = set(tc_class.NAME for tc_class in all_tc_classes if getattr(tc_class, 'OPTIONAL', False))
 
     # the parent toolchain is at the top of the hierarchy
     toolchain_hierarchy = [parent_toolchain]
-
-    while subtoolchain_name:
+    # use a queue to handle a breadth-first-search of the hierarchy
+    bfs_queue = [parent_toolchain]
+    visited = set()
+
+    while bfs_queue:
+        current_tc = bfs_queue.pop()
+        current_tc_name, current_tc_version = current_tc['name'], current_tc['version']
+        subtoolchain_names = subtoolchains[current_tc_name]
+        if subtoolchain_names is None:
+            continue
+        if not isinstance(subtoolchain_names, list):
+            subtoolchain_names = [subtoolchain_names]
         # grab the easyconfig of the current toolchain and search the dependencies for a version of the subtoolchain
         path = robot_find_easyconfig(current_tc_name, current_tc_version)
         if path is None:
@@ -195,44 +203,54 @@ def get_toolchain_hierarchy(parent_toolchain):
                 cands.append({'name': depdep['name'], 'version': depdep['version'] + depdep['versionsuffix']})
                 cands.append(depdep['toolchain'])
 
-        # only retain candidates that match subtoolchain name
-        cands = [c for c in cands if c['name'] == subtoolchain_name]
-
-        uniq_subtc_versions = set([subtc['version'] for subtc in cands])
-
-        if len(uniq_subtc_versions) == 1:
-            subtoolchain_version = cands[0]['version']
-
-        elif len(uniq_subtc_versions) == 0:
-            # only retain GCCcore as subtoolchain if version was found
-            if subtoolchain_name == GCCcore.NAME:
-                _log.info("No version found for %s; assuming legacy toolchain and skipping it as subtoolchain.",
-                          subtoolchain_name)
-                subtoolchain_name = GCCcore.SUBTOOLCHAIN
-                subtoolchain_version = ''
-            # dummy toolchain: end of the line
-            elif subtoolchain_name == DUMMY_TOOLCHAIN_NAME:
-                subtoolchain_version = ''
-            else:
-                raise EasyBuildError("No version found for subtoolchain %s in dependencies of %s",
-                                     subtoolchain_name, current_tc_name)
-        else:
-            if subtoolchain_name == DUMMY_TOOLCHAIN_NAME:
-                # Don't care about multiple versions of dummy
-                _log.info("Ignoring multiple versions of %s in toolchain hierarchy", DUMMY_TOOLCHAIN_NAME)
-                subtoolchain_version = ''
+        # only retain candidates that match subtoolchain names
+        cands = [c for c in cands if c['name'] in subtoolchain_names]
+
+        for subtoolchain_name in subtoolchain_names:
+            uniq_subtc_versions = set([subtc['version'] for subtc in cands if subtc['name'] == subtoolchain_name])
+
+            if len(uniq_subtc_versions) == 1:
+                subtoolchain_version = list(uniq_subtc_versions)[0]
+
+            elif len(uniq_subtc_versions) == 0:
+                # only retain GCCcore as subtoolchain if version was found
+                if subtoolchain_name == GCCcore.NAME:
+                    _log.info("No version found for %s; assuming legacy toolchain and skipping it as subtoolchain.",
+                              subtoolchain_name)
+                    subtoolchain_name = GCCcore.SUBTOOLCHAIN
+                    subtoolchain_version = ''
+                # dummy toolchain: end of the line
+                elif subtoolchain_name == DUMMY_TOOLCHAIN_NAME:
+                    subtoolchain_version = ''
+                elif ((subtoolchain_name in optional_toolchains or current_tc_name in optional_toolchains) and
+                      robot_find_easyconfig(subtoolchain_name, current_tc_version)):
+                    # special case: optionally find e.g. golf/1.4.10 for goolf/1.4.10 even if it is not in
+                    # the module dependencies. This is only allowed for and inside optional subtoolchains.
+                    subtoolchain_version = current_tc_version
+                elif subtoolchain_name in optional_toolchains:
+                    continue
+                else:
+                    raise EasyBuildError("No version found for subtoolchain %s in dependencies of %s",
+                                         subtoolchain_name, current_tc_name)
             else:
-                raise EasyBuildError("Multiple versions of %s found in dependencies of toolchain %s: %s",
-                                     subtoolchain_name, current_tc_name, ', '.join(sorted(uniq_subtc_versions)))
+                if subtoolchain_name == DUMMY_TOOLCHAIN_NAME:
+                    # Don't care about multiple versions of dummy
+                    _log.info("Ignoring multiple versions of %s in toolchain hierarchy", DUMMY_TOOLCHAIN_NAME)
+                    subtoolchain_version = ''
+                else:
+                    raise EasyBuildError("Multiple versions of %s found in dependencies of toolchain %s: %s",
+                                         subtoolchain_name, current_tc_name, ', '.join(sorted(uniq_subtc_versions)))
 
-        if subtoolchain_name == DUMMY_TOOLCHAIN_NAME and not build_option('add_dummy_to_minimal_toolchains'):
-            # we're done
-            break
+            if subtoolchain_name == DUMMY_TOOLCHAIN_NAME and not build_option('add_dummy_to_minimal_toolchains'):
+                # we're done
+                continue
 
-        # add to hierarchy and move to next
-        current_tc_name, current_tc_version = subtoolchain_name, subtoolchain_version
-        subtoolchain_name, subtoolchain_version = subtoolchains[current_tc_name], None
-        toolchain_hierarchy.insert(0, {'name': current_tc_name, 'version': current_tc_version})
+            # add to hierarchy and move to next
+            if subtoolchain_name not in visited:
+                tc = {'name': subtoolchain_name, 'version': subtoolchain_version}
+                toolchain_hierarchy.insert(0, tc)
+                bfs_queue.insert(0, tc)
+                visited.add(tc['name'])
 
     _log.info("Found toolchain hierarchy for toolchain %s: %s", parent_toolchain, toolchain_hierarchy)
     return toolchain_hierarchy

easybuild/toolchains/golf.py

@@ -38,3 +38,4 @@ class Golf(GccToolchain, OpenBLAS, Fftw):
     """Compiler toolchain with GCC, OpenBLAS, and FFTW."""
     NAME = 'golf'
     SUBTOOLCHAIN = GccToolchain.NAME
+    OPTIONAL = True

easybuild/toolchains/golfc.py

@@ -0,0 +1,41 @@
+##
+# Copyright 2013-2018 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for golfc compiler toolchain (includes GCC+CUDA, OpenBLAS, LAPACK, and FFTW).
+
+:author: Kenneth Hoste (Ghent University)
+:author: Bart Oldeman (McGill University, Calcul Quebec, Compute Canada)
+"""
+
+from easybuild.toolchains.gcccuda import GccCUDA
+from easybuild.toolchains.golf import Golf
+from easybuild.toolchains.fft.fftw import Fftw
+from easybuild.toolchains.linalg.openblas import OpenBLAS
+
+class Golfc(GccCUDA, Golf, OpenBLAS, Fftw):
+    """Compiler toolchain with GCC+CUDA, OpenMPI, OpenBLAS, and FFTW."""
+    NAME = 'golfc'
+    SUBTOOLCHAIN = [GccCUDA.NAME, Golf.NAME]
+    OPTIONAL = True

easybuild/toolchains/goolf.py

@@ -29,6 +29,7 @@
 """
 
 from easybuild.toolchains.gompi import Gompi
+from easybuild.toolchains.golf import Golf
 from easybuild.toolchains.fft.fftw import Fftw
 from easybuild.toolchains.linalg.openblas import OpenBLAS
 from easybuild.toolchains.linalg.scalapack import ScaLAPACK
@@ -37,4 +38,4 @@
 class Goolf(Gompi, OpenBLAS, ScaLAPACK, Fftw):
     """Compiler toolchain with GCC, OpenMPI, OpenBLAS, ScaLAPACK and FFTW."""
     NAME = 'goolf'
-    SUBTOOLCHAIN = Gompi.NAME
+    SUBTOOLCHAIN = [Gompi.NAME, Golf.NAME]

easybuild/toolchains/goolfc.py

@@ -29,12 +29,12 @@
 """
 
 from easybuild.toolchains.gompic import Gompic
+from easybuild.toolchains.golfc import Golfc
 from easybuild.toolchains.fft.fftw import Fftw
 from easybuild.toolchains.linalg.openblas import OpenBLAS
 from easybuild.toolchains.linalg.scalapack import ScaLAPACK
 
 class Goolfc(Gompic, OpenBLAS, ScaLAPACK, Fftw):
     """Compiler toolchain with GCC+CUDA, OpenMPI, OpenBLAS, ScaLAPACK and FFTW."""
     NAME = 'goolfc'
-    SUBTOOLCHAIN = Gompic.NAME
-
+    SUBTOOLCHAIN = [Gompic.NAME, Golfc.NAME]

test/framework/robot.py

@@ -691,14 +691,17 @@ def test_get_toolchain_hierarchy(self):
         goolfc_hierarchy = get_toolchain_hierarchy({'name': 'goolfc', 'version': '2.6.10'})
         self.assertEqual(goolfc_hierarchy, [
             {'name': 'GCC', 'version': '4.8.2'},
+            {'name': 'golf', 'version': '2.6.10'},
             {'name': 'gcccuda', 'version': '2.6.10'},
+            {'name': 'golfc', 'version': '2.6.10'},
             {'name': 'gompic', 'version': '2.6.10'},
             {'name': 'goolfc', 'version': '2.6.10'},
         ])
 
         goolf_hierarchy = get_toolchain_hierarchy({'name': 'goolf', 'version': '1.4.10'})
         self.assertEqual(goolf_hierarchy, [
             {'name': 'GCC', 'version': '4.7.2'},
+            {'name': 'golf', 'version': '1.4.10'},
             {'name': 'gompi', 'version': '1.4.10'},
             {'name': 'goolf', 'version': '1.4.10'},
         ])