add generic SystemMPI easyblock to generate module for existing MPI library installation

.travis.yml

@@ -20,6 +20,9 @@ addons:
   apt:
     packages:
       - tcl8.5
+before_install:
+    # For testing systemmpi easyblock we need an OpenMPI instance
+    - sudo apt-get update && sudo apt-get install -y libopenmpi-dev openmpi-bin
 install:
     # install easybuild-framework (and dependencies)
     # prefer clone & easy_install over easy_install directly in order to obtain information of what was installed exactly

easybuild/easyblocks/generic/bundle.py

@@ -162,7 +162,7 @@ def install_step(self):
 
     def make_module_extra(self):
         """Set extra stuff in module file, e.g. $EBROOT*, $EBVERSION*, etc."""
-        return super(Bundle, self).make_module_extra(altroot=self.altroot, altversion=self.altversion)
+        return EasyBlock.make_module_extra(self, altroot=self.altroot, altversion=self.altversion)
 
     def sanity_check_step(self, *args, **kwargs):
         """

easybuild/easyblocks/generic/systemcompiler.py

@@ -27,18 +27,64 @@
 
 @author Bernd Mohr (Juelich Supercomputing Centre)
 @author Kenneth Hoste (Ghent University)
+@author Alan O'Cais (Juelich Supercomputing Centre)
 """
 import os
 import re
+from vsc.utils import fancylogger
+from distutils.version import LooseVersion
 
 from easybuild.easyblocks.generic.bundle import Bundle
-from easybuild.tools.filetools import read_file, which
-from easybuild.tools.run import run_cmd
+from easybuild.easyblocks.icc import EB_icc
+from easybuild.easyblocks.ifort import EB_ifort
+from easybuild.easyblocks.gcc import EB_GCC
 from easybuild.framework.easyconfig.easyconfig import ActiveMNS
+from easybuild.framework.easyconfig import CUSTOM
 from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.filetools import read_file, resolve_path, which
+from easybuild.tools.run import run_cmd
+
+_log = fancylogger.getLogger('easyblocks.generic.systemcompiler')
+
+def extract_compiler_version(compiler_name):
+    """Extract compiler version for provided compiler_name."""
+    # look for 3-4 digit version number, surrounded by spaces
+    # examples:
+    # gcc (GCC) 4.4.7 20120313 (Red Hat 4.4.7-11)
+    # Intel(R) C Intel(R) 64 Compiler XE for applications running on Intel(R) 64, Version 15.0.1.133 Build 20141023
+    version_regex = re.compile(r'\s([0-9]+(?:\.[0-9]+){1,3})\s', re.M)
+    if compiler_name == 'gcc':
+        out, _ = run_cmd("gcc --version", simple=False)
+        res = version_regex.search(out)
+        if res is None:
+            raise EasyBuildError("Could not extract GCC version from %s", out)
+        compiler_version = res.group(1)
+    elif compiler_name in ['icc', 'ifort']:
+        # A fully resolved icc/ifort (without symlinks) includes the release version in the path
+        # e.g. .../composer_xe_2015.3.187/bin/intel64/icc
+        # Match the last incidence of _ since we don't know what might be in the path, then split it up on /
+        out = (resolve_path(which(compiler_name)).split("_")[-1]).split("/")
+        compiler_version = out[0]
+        # Check what we have looks like a version number (the regex we use requires spaces around the version number)
+        if version_regex.search(" " + compiler_version + " ") is None:
+            typical_install_path = '.../composer_xe_2015.3.187/bin/intel64/icc'
+            raise EasyBuildError(
+                "Derived Intel compiler version %s doesn't look correct, is compiler installed in a path like %s?",
+                typical_install_path, compiler_version
+            )
+    else:
+        raise EasyBuildError("Unknown compiler %s", compiler_name)
 
+    if compiler_version:
+        _log.debug("Extracted compiler version '%s' for %s from: %s", compiler_version, compiler_name, out)
+    else:
+        raise EasyBuildError("Failed to extract compiler version for %s using regex pattern '%s' from: %s",
+                             compiler_name, version_regex.pattern, out)
 
-class SystemCompiler(Bundle):
+    return compiler_version
+
+# No need to inherit from EB_icc since EB_ifort already inherits from that
+class SystemCompiler(Bundle, EB_GCC, EB_ifort):
     """
     Support for generating a module file for the system compiler with specified name.
 
@@ -48,46 +94,49 @@ class SystemCompiler(Bundle):
     if an actual version is specified, it is checked against the derived version of the system compiler that was found.
     """
 
-    def extract_compiler_version(self, txt):
-        """Extract compiler version from provided string."""
-        # look for 3-4 digit version number, surrounded by spaces
-        # examples:
-        # gcc (GCC) 4.4.7 20120313 (Red Hat 4.4.7-11)
-        # Intel(R) C Intel(R) 64 Compiler XE for applications running on Intel(R) 64, Version 15.0.1.133 Build 20141023
-        version_regex = re.compile(r'\s([0-9]+(?:\.[0-9]+){1,3})\s', re.M)
-        res = version_regex.search(txt)
-        if res:
-            self.compiler_version = res.group(1)
-            self.log.debug("Extracted compiler version '%s' from: %s", self.compiler_version, txt)
-        else:
-            raise EasyBuildError("Failed to extract compiler version using regex pattern '%s' from: %s",
-                                 version_regex.pattern, txt)
+    @staticmethod
+    def extra_options():
+        """Add custom easyconfig parameters for SystemCompiler easyblock."""
+        # Gather extra_vars from inherited classes, order matters here to make this work without problems in __init__
+        extra_vars = EB_GCC.extra_options()
+        extra_vars.update(EB_icc.extra_options())
+        extra_vars.update(EB_ifort.extra_options())
+        extra_vars.update(Bundle.extra_options())
+        # Add an option to add all module path extensions to the resultant easyconfig
+        # This is useful if you are importing a compiler from a non-default path
+        extra_vars.update({
+            'generate_standalone_module': [
+                False,
+                "Add known path/library extensions and environment variables for the compiler to the final module",
+                CUSTOM
+            ],
+        })
+        return extra_vars
 
     def __init__(self, *args, **kwargs):
         """Extra initialization: determine system compiler version and prefix."""
         super(SystemCompiler, self).__init__(*args, **kwargs)
 
         # Determine compiler path (real path, with resolved symlinks)
         compiler_name = self.cfg['name'].lower()
+        if compiler_name == 'gcccore':
+            compiler_name = 'gcc'
         path_to_compiler = which(compiler_name)
         if path_to_compiler:
-            path_to_compiler = os.path.realpath(path_to_compiler)
+            path_to_compiler = resolve_path(path_to_compiler)
             self.log.info("Found path to compiler '%s' (with symlinks resolved): %s", compiler_name, path_to_compiler)
         else:
             raise EasyBuildError("%s not found in $PATH", compiler_name)
 
-        # Determine compiler version and installation prefix
-        if compiler_name == 'gcc':
-            out, _ = run_cmd("gcc --version", simple=False)
-            self.extract_compiler_version(out)
+        # Determine compiler version
+        self.compiler_version = extract_compiler_version(compiler_name)
 
+        # Determine installation prefix
+        if compiler_name == 'gcc':
             # strip off 'bin/gcc'
             self.compiler_prefix = os.path.dirname(os.path.dirname(path_to_compiler))
 
         elif compiler_name in ['icc', 'ifort']:
-            out, _ = run_cmd("%s -V" % compiler_name, simple=False)
-            self.extract_compiler_version(out)
-
             intelvars_fn = path_to_compiler + 'vars.sh'
             if os.path.isfile(intelvars_fn):
                 self.log.debug("Trying to determine compiler install prefix from %s", intelvars_fn)
@@ -103,12 +152,27 @@ def __init__(self, *args, **kwargs):
                 # strip off 'bin/intel*/icc'
                 self.compiler_prefix = os.path.dirname(os.path.dirname(os.path.dirname(path_to_compiler)))
 
+            # For versions 2016+ of Intel compilers they changed the installation path so must shave off 2 more
+            # directories from result of the above
+            if LooseVersion(self.compiler_version) >= LooseVersion('2016'):
+                self.compiler_prefix = os.path.dirname(os.path.dirname(self.compiler_prefix))
+
         else:
             raise EasyBuildError("Unknown system compiler %s" % self.cfg['name'])
 
+        if not os.path.exists(self.compiler_prefix):
+            raise EasyBuildError("Path derived for system compiler (%s) does not exist: %s!",
+                                 compiler_name, self.compiler_prefix)
         self.log.debug("Derived version/install prefix for system compiler %s: %s, %s",
                        compiler_name, self.compiler_version, self.compiler_prefix)
 
+        if self.compiler_prefix in ['/usr', '/usr/local']:
+            if self.cfg['generate_standalone_module']:
+                # Force off adding paths to module since unloading such a module would be a potential shell killer
+                self.cfg['generate_standalone_module'] = False
+                self.log.warning("Disabling option 'generate_standalone_module' since installation prefix is %s",
+                                 self.compiler_prefix)
+
         # If EasyConfig specified "real" version (not 'system' which means 'derive automatically'), check it
         if self.cfg['version'] == 'system':
             self.log.info("Found specified version '%s', going with derived compiler version '%s'",
@@ -127,15 +191,41 @@ def __init__(self, *args, **kwargs):
         self.orig_version = self.cfg['version']
         self.orig_installdir = self.installdir
 
+    def prepare_step(self, *args, **kwargs):
+        """Do the bundle prepare step to ensure any deps are loaded at a minimum."""
+        if self.cfg['generate_standalone_module']:
+            if self.cfg['name'] in ['GCC', 'GCCcore']:
+                EB_GCC.prepare_step(self, *args, **kwargs)
+            elif self.cfg['name'] in ['icc']:
+                EB_icc.prepare_step(self, *args, **kwargs)
+            elif self.cfg['name'] in ['ifort']:
+                EB_ifort.prepare_step(self, *args, **kwargs)
+            else:
+                raise EasyBuildError("I don't know how to do the prepare_step for %s", self.cfg['name'])
+        else:
+            Bundle.prepare_step(self, *args, **kwargs)
+
     def make_installdir(self, dontcreate=None):
         """Custom implementation of make installdir: do nothing, do not touch system compiler directories and files."""
         pass
 
     def make_module_req_guess(self):
         """
-        A dictionary of possible directories to look for.  Return empty dict for a system compiler.
+        A dictionary of possible directories to look for.  Return known dict for the system compiler, or empty dict if
+        generate_standalone_module parameter is False
         """
-        return {}
+        if self.cfg['generate_standalone_module']:
+            if self.cfg['name'] in ['GCC','GCCcore']:
+                guesses = EB_GCC.make_module_req_guess(self)
+            elif self.cfg['name'] in ['icc']:
+                guesses = EB_icc.make_module_req_guess(self)
+            elif self.cfg['name'] in ['ifort']:
+                guesses = EB_ifort.make_module_req_guess(self)
+            else:
+                raise EasyBuildError("I don't know how to generate module var guesses for %s", self.cfg['name'])
+        else:
+            guesses = {}
+        return guesses
 
     def make_module_step(self, fake=False):
         """
@@ -168,3 +258,39 @@ def make_module_extend_modpath(self):
         # Reset to actual compiler version (e.g., "4.8.2")
         self.cfg['version'] = self.compiler_version
         return res
+
+    def make_module_extra(self, *args, **kwargs):
+        """Add any additional module text."""
+        if self.cfg['generate_standalone_module']:
+            if self.cfg['name'] in ['GCC','GCCcore']:
+                extras = EB_GCC.make_module_extra(self, *args, **kwargs)
+            elif self.cfg['name'] in ['icc']:
+                extras = EB_icc.make_module_extra(self, *args, **kwargs)
+            elif self.cfg['name'] in ['ifort']:
+                extras = EB_ifort.make_module_extra(self, *args, **kwargs)
+            else:
+                raise EasyBuildError("I don't know how to generate extra module text for %s", self.cfg['name'])
+        else:
+            extras = super(SystemCompiler, self).make_module_extra(*args, **kwargs)
+        return extras
+
+    def post_install_step(self, *args, **kwargs):
+        """Do nothing."""
+        pass
+
+    def cleanup_step(self):
+        """Do nothing."""
+        pass
+
+    def permissions_step(self):
+        """Do nothing."""
+        pass
+
+    def sanity_check_step(self, *args, **kwargs):
+        """
+        Nothing is being installed, so just being able to load the (fake) module is sufficient
+        """
+        self.log.info("Testing loading of module '%s' by means of sanity check" % self.full_mod_name)
+        fake_mod_data = self.load_fake_module(purge=True)
+        self.log.debug("Cleaning up after testing loading of module")
+        self.clean_up_fake_module(fake_mod_data)