Merge pull request #1018 from boegel/only_module

add support for only (re)generating module file

easybuild/framework/easyblock.py

@@ -68,7 +68,7 @@
 from easybuild.tools.run import run_cmd
 from easybuild.tools.jenkins import write_to_xml
 from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator
-from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
+from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version
 from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX
 from easybuild.tools.modules import get_software_root, modules_tool
 from easybuild.tools.repository.repository import init_repository
@@ -674,9 +674,8 @@ def gen_installdir(self):
         """
         basepath = install_path()
         if basepath:
-            self.install_subdir = ActiveMNS().det_full_module_name(self.cfg, force_visible=True)
-            installdir = os.path.join(basepath, self.install_subdir)
-            self.installdir = os.path.abspath(installdir)
+            self.install_subdir = ActiveMNS().det_install_subdir(self.cfg)
+            self.installdir = os.path.join(os.path.abspath(basepath), self.install_subdir)
             self.log.info("Install dir set to %s" % self.installdir)
         else:
             raise EasyBuildError("Can't set installation directory")
@@ -952,7 +951,7 @@ def make_module_req(self):
         requirements = self.make_module_req_guess()
 
         lines = []
-        if os.path.exists(self.installdir):
+        if os.path.isdir(self.installdir):
             try:
                 os.chdir(self.installdir)
             except OSError, err:
@@ -1580,7 +1579,11 @@ def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=F
                 self.log.warning("Sanity check: %s" % self.sanity_check_fail_msgs[-1])
 
         # chdir to installdir (better environment for running tests)
-        os.chdir(self.installdir)
+        if os.path.isdir(self.installdir):
+            try:
+                os.chdir(self.installdir)
+            except OSError, err:
+                raise EasyBuildError("Failed to move to installdir %s: %s", self.installdir, err)
 
         # run sanity check commands
         commands = self.cfg['sanity_check_commands']
@@ -1724,23 +1727,45 @@ def update_config_template_run_step(self):
             self.cfg.template_values[name[0]] = str(getattr(self, name[0], None))
         self.cfg.generate_template_values()
 
-    def run_step(self, step, methods, skippable=False):
-        """
-        Run step, returns false when execution should be stopped
-        """
+    def _skip_step(self, step, skippable):
+        """Dedice whether or not to skip the specified step."""
+        module_only = build_option('module_only')
+        force = build_option('force')
+        skip = False
+
+        # skip step if specified as individual (skippable) step
         if skippable and (self.skip or step in self.cfg['skipsteps']):
-            self.log.info("Skipping %s step" % step)
+            self.log.info("Skipping %s step (skip: %s, skipsteps: %s)", step, self.skip, self.cfg['skipsteps'])
+            skip = True
+
+        # skip step when only generating module file; still run sanity check without use of force
+        elif module_only and not step in ['sanitycheck', 'module']:
+            self.log.info("Skipping %s step (only generating module)", step)
+            skip = True
+
+        # allow skipping sanity check too when only generating module and force is used
+        elif module_only and step == 'sanitycheck' and force:
+            self.log.info("Skipping %s step because of forced module-only mode", step)
+            skip = True
+
         else:
-            self.log.info("Starting %s step" % step)
-            # update the config templates
-            self.update_config_template_run_step()
+            self.log.debug("Not skipping %s step (skippable: %s, skip: %s, skipsteps: %s, module_only: %s, force: %s",
+                           step, skippable, self.skip, self.cfg['skipsteps'], module_only, force)
 
-            for m in methods:
-                self.log.info("Running method %s part of step %s" % ('_'.join(m.func_code.co_names), step))
-                m(self)
+        return skip
+
+    def run_step(self, step, methods):
+        """
+        Run step, returns false when execution should be stopped
+        """
+        self.log.info("Starting %s step", step)
+        self.update_config_template_run_step()
+        for m in methods:
+            self.log.info("Running method %s part of step %s" % ('_'.join(m.func_code.co_names), step))
+            m(self)
 
         if self.cfg['stop'] == step:
-            self.log.info("Stopping after %s step." % step)
+            self.log.info("Stopping after %s step.", step)
             raise StopException(step)
 
     @staticmethod
@@ -1849,9 +1874,12 @@ def run_all_steps(self, run_test_cases):
 
         print_msg("building and installing %s..." % self.full_mod_name, self.log, silent=self.silent)
         try:
-            for (stop_name, descr, step_methods, skippable) in steps:
-                print_msg("%s..." % descr, self.log, silent=self.silent)
-                self.run_step(stop_name, step_methods, skippable=skippable)
+            for (step_name, descr, step_methods, skippable) in steps:
+                if self._skip_step(step_name, skippable):
+                    print_msg("%s [skipped]" % descr, self.log, silent=self.silent)
+                else:
+                    print_msg("%s..." % descr, self.log, silent=self.silent)
+                    self.run_step(step_name, step_methods)
 
         except StopException:
             pass

easybuild/framework/easyconfig/easyconfig.py

@@ -1090,6 +1090,13 @@ def det_full_module_name(self, ec, force_visible=False):
         self.log.debug("Obtained valid full module name %s" % mod_name)
         return mod_name
 
+    def det_install_subdir(self, ec):
+        """Determine name of software installation subdirectory."""
+        self.log.debug("Determining software installation subdir for %s", ec)
+        subdir = self.mns.det_install_subdir(self.check_ec_type(ec))
+        self.log.debug("Obtained subdir %s", subdir)
+        return subdir
+
     def det_devel_module_filename(self, ec, force_visible=False):
         """Determine devel module filename."""
         modname = self.det_full_module_name(ec, force_visible=force_visible)

easybuild/tools/config.py

@@ -91,6 +91,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
         'ignore_dirs',
         'modules_footer',
         'only_blocks',
+        'module_only',
         'optarch',
         'regtest_output_dir',
         'skip',

easybuild/tools/module_naming_scheme/mns.py

@@ -84,6 +84,18 @@ def det_short_module_name(self, ec):
         # by default: full module name doesn't include a $MODULEPATH subdir
         return self.det_full_module_name(ec)
 
+    def det_install_subdir(self, ec):
+        """
+        Determine name of software installation subdirectory of install path.
+
+        @param ec: dict-like object with easyconfig parameter values; for now only the 'name',
+                   'version', 'versionsuffix' and 'toolchain' parameters are guaranteed to be available
+
+        @return: string with name of subdirectory, e.g.: '<compiler>/<mpi_lib>/<name>/<version>'
+        """
+        # by default: use full module name as name for install subdir
+        return self.det_full_module_name(ec)
+
     def det_module_subdir(self, ec):
         """
         Determine subdirectory for module file in $MODULEPATH.

easybuild/tools/options.py

@@ -207,6 +207,7 @@ def override_options(self):
                           "(e.g. --hide-deps=zlib,ncurses)", 'strlist', 'extend', None),
             'oldstyleconfig':   ("Look for and use the oldstyle configuration file.",
                                  None, 'store_true', True),
+            'module-only': ("Only generate module file (and run sanity check)", None, 'store_true', False),
             'optarch': ("Set architecture optimization, overriding native architecture optimizations",
                         None, 'store', None),
             'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"),

test/framework/sandbox/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py

@@ -0,0 +1,37 @@
+##
+# Copyright 2013-2015 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
+# the Hercules foundation (http://www.herculesstichting.be/in_English)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# http://github.com/hpcugent/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+Implementation of a test module naming scheme that can be used to migrate from EasyBuildMNS to HierarchicalMNS.
+
+@author: Kenneth Hoste (Ghent University)
+"""
+from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS
+from easybuild.tools.module_naming_scheme.hierarchical_mns import HierarchicalMNS
+
+class MigrateFromEBToHMNS(HierarchicalMNS, EasyBuildMNS):
+
+    def det_install_subdir(self, ec):
+        """Determine name of software installation subdirectory of install path, using EasyBuild MNS."""
+        return EasyBuildMNS.det_full_module_name(self, ec)

test/framework/toy_build.py

@@ -44,7 +44,7 @@
 from easybuild.framework.easyconfig.easyconfig import EasyConfig
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import get_module_syntax
-from easybuild.tools.filetools import mkdir, read_file, write_file
+from easybuild.tools.filetools import mkdir, read_file, which, write_file
 from easybuild.tools.modules import modules_tool
 
 
@@ -880,6 +880,80 @@ def test_external_dependencies(self):
         outtxt = self.test_toy_build(ec_file=toy_ec, verbose=True, extra_args=['--dry-run'], verify=False)
         self.assertTrue(re.search(r"^ \* \[ \] .* \(module: toy/0.0-external-deps-broken2\)", outtxt, re.M))
 
+    def test_module_only(self):
+        """Test use of --module-only."""
+        ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'toy-0.0.eb')
+        toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')
+
+        # hide all existing modules
+        self.reset_modulepath([os.path.join(self.test_installpath, 'modules', 'all')])
+
+        # sanity check fails without --force if software is not installed yet
+        common_args = [
+            ec_file,
+            '--sourcepath=%s' % self.test_sourcepath,
+            '--buildpath=%s' % self.test_buildpath,
+            '--installpath=%s' % self.test_installpath,
+            '--debug',
+            '--unittest-file=%s' % self.logfile,
+            '--module-syntax=Tcl',
+        ]
+        args = common_args + ['--module-only']
+        err_msg = "Sanity check failed"
+        self.assertErrorRegex(EasyBuildError, err_msg, self.eb_main, args, do_build=True, raise_error=True)
+        self.assertFalse(os.path.exists(toy_mod))
+
+        self.eb_main(args + ['--force'], do_build=True, raise_error=True)
+        self.assertTrue(os.path.exists(toy_mod))
+
+        os.remove(toy_mod)
+
+        # installing another module under a different naming scheme and using Lua module syntax works fine
+
+        # first actually build and install toy software + module
+        prefix = os.path.join(self.test_installpath, 'software', 'toy', '0.0')
+        self.eb_main(common_args + ['--force'], do_build=True, raise_error=True)
+        self.assertTrue(os.path.exists(toy_mod))
+        self.assertTrue(os.path.exists(os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin')))
+        modtxt = read_file(toy_mod)
+        self.assertTrue(re.search("set root %s" % prefix, modtxt))
+        self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 1)
+        self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1)
+
+        # install (only) additional module under a hierarchical MNS
+        args = common_args + [
+            '--module-only',
+            '--module-naming-scheme=MigrateFromEBToHMNS',
+        ]
+        toy_core_mod = os.path.join(self.test_installpath, 'modules', 'all', 'Core', 'toy', '0.0')
+        self.assertFalse(os.path.exists(toy_core_mod))
+        self.eb_main(args, do_build=True, raise_error=True)
+        self.assertTrue(os.path.exists(toy_core_mod))
+        # existing install is reused
+        modtxt2 = read_file(toy_core_mod)
+        self.assertTrue(re.search("set root %s" % prefix, modtxt2))
+        self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 1)
+        self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1)
+
+        os.remove(toy_mod)
+        os.remove(toy_core_mod)
+
+        # test installing (only) additional module in Lua syntax (if Lmod is available)
+        lmod_abspath = which('lmod')
+        if lmod_abspath is not None:
+            args = common_args[:-1] + [
+                '--module-only',
+                '--module-syntax=Lua',
+                '--modules-tool=Lmod',
+            ]
+            self.assertFalse(os.path.exists(toy_mod + '.lua'))
+            self.eb_main(args, do_build=True, raise_error=True)
+            self.assertTrue(os.path.exists(toy_mod + '.lua'))
+            # existing install is reused
+            modtxt3 = read_file(toy_mod + '.lua')
+            self.assertTrue(re.search('local root = "%s"' % prefix, modtxt3))
+            self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 1)
+            self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1)
 
 def suite():
     """ return all the tests in this file """