diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c1e7542d27..74f2396baa 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -68,7 +68,7 @@ from easybuild.tools.run import run_cmd from easybuild.tools.jenkins import write_to_xml from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator -from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version +from easybuild.tools.module_naming_scheme.utilities import avail_module_naming_schemes, det_full_ec_version from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX from easybuild.tools.modules import get_software_root, modules_tool from easybuild.tools.repository.repository import init_repository @@ -674,9 +674,8 @@ def gen_installdir(self): """ basepath = install_path() if basepath: - self.install_subdir = ActiveMNS().det_full_module_name(self.cfg, force_visible=True) - installdir = os.path.join(basepath, self.install_subdir) - self.installdir = os.path.abspath(installdir) + self.install_subdir = ActiveMNS().det_install_subdir(self.cfg) + self.installdir = os.path.join(os.path.abspath(basepath), self.install_subdir) self.log.info("Install dir set to %s" % self.installdir) else: raise EasyBuildError("Can't set installation directory") @@ -952,7 +951,7 @@ def make_module_req(self): requirements = self.make_module_req_guess() lines = [] - if os.path.exists(self.installdir): + if os.path.isdir(self.installdir): try: os.chdir(self.installdir) except OSError, err: @@ -1580,7 +1579,11 @@ def sanity_check_step(self, custom_paths=None, custom_commands=None, extension=F self.log.warning("Sanity check: %s" % self.sanity_check_fail_msgs[-1]) # chdir to installdir (better environment for running tests) - os.chdir(self.installdir) + if os.path.isdir(self.installdir): + try: + os.chdir(self.installdir) + except OSError, err: + raise EasyBuildError("Failed to move to installdir %s: %s", self.installdir, err) # run sanity check commands commands = self.cfg['sanity_check_commands'] @@ -1724,23 +1727,45 @@ def update_config_template_run_step(self): self.cfg.template_values[name[0]] = str(getattr(self, name[0], None)) self.cfg.generate_template_values() - def run_step(self, step, methods, skippable=False): - """ - Run step, returns false when execution should be stopped - """ + def _skip_step(self, step, skippable): + """Dedice whether or not to skip the specified step.""" + module_only = build_option('module_only') + force = build_option('force') + skip = False + + # skip step if specified as individual (skippable) step if skippable and (self.skip or step in self.cfg['skipsteps']): - self.log.info("Skipping %s step" % step) + self.log.info("Skipping %s step (skip: %s, skipsteps: %s)", step, self.skip, self.cfg['skipsteps']) + skip = True + + # skip step when only generating module file; still run sanity check without use of force + elif module_only and not step in ['sanitycheck', 'module']: + self.log.info("Skipping %s step (only generating module)", step) + skip = True + + # allow skipping sanity check too when only generating module and force is used + elif module_only and step == 'sanitycheck' and force: + self.log.info("Skipping %s step because of forced module-only mode", step) + skip = True + else: - self.log.info("Starting %s step" % step) - # update the config templates - self.update_config_template_run_step() + self.log.debug("Not skipping %s step (skippable: %s, skip: %s, skipsteps: %s, module_only: %s, force: %s", + step, skippable, self.skip, self.cfg['skipsteps'], module_only, force) - for m in methods: - self.log.info("Running method %s part of step %s" % ('_'.join(m.func_code.co_names), step)) - m(self) + return skip + + def run_step(self, step, methods): + """ + Run step, returns false when execution should be stopped + """ + self.log.info("Starting %s step", step) + self.update_config_template_run_step() + for m in methods: + self.log.info("Running method %s part of step %s" % ('_'.join(m.func_code.co_names), step)) + m(self) if self.cfg['stop'] == step: - self.log.info("Stopping after %s step." % step) + self.log.info("Stopping after %s step.", step) raise StopException(step) @staticmethod @@ -1849,9 +1874,12 @@ def run_all_steps(self, run_test_cases): print_msg("building and installing %s..." % self.full_mod_name, self.log, silent=self.silent) try: - for (stop_name, descr, step_methods, skippable) in steps: - print_msg("%s..." % descr, self.log, silent=self.silent) - self.run_step(stop_name, step_methods, skippable=skippable) + for (step_name, descr, step_methods, skippable) in steps: + if self._skip_step(step_name, skippable): + print_msg("%s [skipped]" % descr, self.log, silent=self.silent) + else: + print_msg("%s..." % descr, self.log, silent=self.silent) + self.run_step(step_name, step_methods) except StopException: pass diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index bd161ab708..b906186aff 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -1090,6 +1090,13 @@ def det_full_module_name(self, ec, force_visible=False): self.log.debug("Obtained valid full module name %s" % mod_name) return mod_name + def det_install_subdir(self, ec): + """Determine name of software installation subdirectory.""" + self.log.debug("Determining software installation subdir for %s", ec) + subdir = self.mns.det_install_subdir(self.check_ec_type(ec)) + self.log.debug("Obtained subdir %s", subdir) + return subdir + def det_devel_module_filename(self, ec, force_visible=False): """Determine devel module filename.""" modname = self.det_full_module_name(ec, force_visible=force_visible) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 7b398be0aa..85e62878fc 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -91,6 +91,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'ignore_dirs', 'modules_footer', 'only_blocks', + 'module_only', 'optarch', 'regtest_output_dir', 'skip', diff --git a/easybuild/tools/module_naming_scheme/mns.py b/easybuild/tools/module_naming_scheme/mns.py index 29fbb1f3ff..670493fb78 100644 --- a/easybuild/tools/module_naming_scheme/mns.py +++ b/easybuild/tools/module_naming_scheme/mns.py @@ -84,6 +84,18 @@ def det_short_module_name(self, ec): # by default: full module name doesn't include a $MODULEPATH subdir return self.det_full_module_name(ec) + def det_install_subdir(self, ec): + """ + Determine name of software installation subdirectory of install path. + + @param ec: dict-like object with easyconfig parameter values; for now only the 'name', + 'version', 'versionsuffix' and 'toolchain' parameters are guaranteed to be available + + @return: string with name of subdirectory, e.g.: '///' + """ + # by default: use full module name as name for install subdir + return self.det_full_module_name(ec) + def det_module_subdir(self, ec): """ Determine subdirectory for module file in $MODULEPATH. diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 0b62665209..4561f66554 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -207,6 +207,7 @@ def override_options(self): "(e.g. --hide-deps=zlib,ncurses)", 'strlist', 'extend', None), 'oldstyleconfig': ("Look for and use the oldstyle configuration file.", None, 'store_true', True), + 'module-only': ("Only generate module file (and run sanity check)", None, 'store_true', False), 'optarch': ("Set architecture optimization, overriding native architecture optimizations", None, 'store', None), 'pretend': (("Does the build/installation in a test directory located in $HOME/easybuildinstall"), diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py new file mode 100644 index 0000000000..13c1634b3d --- /dev/null +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/migrate_from_eb_to_hmns.py @@ -0,0 +1,37 @@ +## +# Copyright 2013-2015 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +Implementation of a test module naming scheme that can be used to migrate from EasyBuildMNS to HierarchicalMNS. + +@author: Kenneth Hoste (Ghent University) +""" +from easybuild.tools.module_naming_scheme.easybuild_mns import EasyBuildMNS +from easybuild.tools.module_naming_scheme.hierarchical_mns import HierarchicalMNS + +class MigrateFromEBToHMNS(HierarchicalMNS, EasyBuildMNS): + + def det_install_subdir(self, ec): + """Determine name of software installation subdirectory of install path, using EasyBuild MNS.""" + return EasyBuildMNS.det_full_module_name(self, ec) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index ce14cda823..2a577f3b3b 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -44,7 +44,7 @@ from easybuild.framework.easyconfig.easyconfig import EasyConfig from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import get_module_syntax -from easybuild.tools.filetools import mkdir, read_file, write_file +from easybuild.tools.filetools import mkdir, read_file, which, write_file from easybuild.tools.modules import modules_tool @@ -880,6 +880,80 @@ def test_external_dependencies(self): outtxt = self.test_toy_build(ec_file=toy_ec, verbose=True, extra_args=['--dry-run'], verify=False) self.assertTrue(re.search(r"^ \* \[ \] .* \(module: toy/0.0-external-deps-broken2\)", outtxt, re.M)) + def test_module_only(self): + """Test use of --module-only.""" + ec_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'toy-0.0.eb') + toy_mod = os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0') + + # hide all existing modules + self.reset_modulepath([os.path.join(self.test_installpath, 'modules', 'all')]) + + # sanity check fails without --force if software is not installed yet + common_args = [ + ec_file, + '--sourcepath=%s' % self.test_sourcepath, + '--buildpath=%s' % self.test_buildpath, + '--installpath=%s' % self.test_installpath, + '--debug', + '--unittest-file=%s' % self.logfile, + '--module-syntax=Tcl', + ] + args = common_args + ['--module-only'] + err_msg = "Sanity check failed" + self.assertErrorRegex(EasyBuildError, err_msg, self.eb_main, args, do_build=True, raise_error=True) + self.assertFalse(os.path.exists(toy_mod)) + + self.eb_main(args + ['--force'], do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_mod)) + + os.remove(toy_mod) + + # installing another module under a different naming scheme and using Lua module syntax works fine + + # first actually build and install toy software + module + prefix = os.path.join(self.test_installpath, 'software', 'toy', '0.0') + self.eb_main(common_args + ['--force'], do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_mod)) + self.assertTrue(os.path.exists(os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin'))) + modtxt = read_file(toy_mod) + self.assertTrue(re.search("set root %s" % prefix, modtxt)) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 1) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) + + # install (only) additional module under a hierarchical MNS + args = common_args + [ + '--module-only', + '--module-naming-scheme=MigrateFromEBToHMNS', + ] + toy_core_mod = os.path.join(self.test_installpath, 'modules', 'all', 'Core', 'toy', '0.0') + self.assertFalse(os.path.exists(toy_core_mod)) + self.eb_main(args, do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_core_mod)) + # existing install is reused + modtxt2 = read_file(toy_core_mod) + self.assertTrue(re.search("set root %s" % prefix, modtxt2)) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 1) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) + + os.remove(toy_mod) + os.remove(toy_core_mod) + + # test installing (only) additional module in Lua syntax (if Lmod is available) + lmod_abspath = which('lmod') + if lmod_abspath is not None: + args = common_args[:-1] + [ + '--module-only', + '--module-syntax=Lua', + '--modules-tool=Lmod', + ] + self.assertFalse(os.path.exists(toy_mod + '.lua')) + self.eb_main(args, do_build=True, raise_error=True) + self.assertTrue(os.path.exists(toy_mod + '.lua')) + # existing install is reused + modtxt3 = read_file(toy_mod + '.lua') + self.assertTrue(re.search('local root = "%s"' % prefix, modtxt3)) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software'))), 1) + self.assertEqual(len(os.listdir(os.path.join(self.test_installpath, 'software', 'toy'))), 1) def suite(): """ return all the tests in this file """