add support for backing up modules with --module-only via --backup-modules

easybuild/framework/easyblock.py

@@ -34,6 +34,7 @@
 :author: Toon Willems (Ghent University)
 :author: Ward Poelmans (Ghent University)
 :author: Fotis Georgatos (Uni.Lu, NTUA)
+:author: Damian Alvarez (Forschungszentrum Juelich GmbH)
 """
 
 import copy
@@ -60,22 +61,23 @@
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP
 from easybuild.tools.build_details import get_build_stats
 from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs
-from easybuild.tools.build_log import print_error, print_msg
+from easybuild.tools.build_log import print_error, print_msg, print_warning
 from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath
 from easybuild.tools.config import install_path, log_path, package_path, source_paths
 from easybuild.tools.environment import restore_env, sanitize_env
 from easybuild.tools.filetools import CHECKSUM_TYPE_MD5, CHECKSUM_TYPE_SHA256
-from easybuild.tools.filetools import adjust_permissions, apply_patch, change_dir, convert_name, compute_checksum
-from easybuild.tools.filetools import copy_file, derive_alt_pypi_url, download_file, encode_class_name, extract_file
-from easybuild.tools.filetools import is_alt_pypi_url, mkdir, move_logs, read_file, remove_file, rmtree2, write_file
+from easybuild.tools.filetools import adjust_permissions, apply_patch, back_up_file, change_dir, convert_name
+from easybuild.tools.filetools import compute_checksum, copy_file, derive_alt_pypi_url, diff_files, download_file
+from easybuild.tools.filetools import encode_class_name, extract_file, is_alt_pypi_url, mkdir, move_logs, read_file
+from easybuild.tools.filetools import remove_file, rmtree2, write_file
 from easybuild.tools.filetools import verify_checksum, weld_paths
 from easybuild.tools.run import run_cmd
 from easybuild.tools.jenkins import write_to_xml
 from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
 from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version
-from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX
+from easybuild.tools.modules import Lmod, ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX
 from easybuild.tools.modules import invalidate_module_caches_for, get_software_root, get_software_root_env_var_name
-from easybuild.tools.modules import get_software_version_env_var_name
+from easybuild.tools.modules import get_software_version_env_var_name, modules_tool
 from easybuild.tools.package.utilities import package
 from easybuild.tools.repository.repository import init_repository
 from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME
@@ -169,6 +171,7 @@ def __init__(self, ec):
         # module generator
         self.module_generator = module_generator(self, fake=True)
         self.mod_filepath = self.module_generator.get_module_filepath()
+        self.mod_file_backup = None
 
         # modules footer/header
         self.modules_footer = None
@@ -1169,12 +1172,10 @@ def make_module_req(self):
         """
         requirements = self.make_module_req_guess()
 
-        lines = []
+        lines = ['\n']
         if os.path.isdir(self.installdir):
             change_dir(self.installdir)
 
-            lines.append('\n')
-
             if self.dry_run:
                 self.dry_run_msg("List of paths that would be searched and added to module file:\n")
                 note = "note: glob patterns are not expanded and existence checks "
@@ -1478,6 +1479,16 @@ def check_readiness_step(self):
         if root:
             raise EasyBuildError("Module is already loaded (%s is set), installation cannot continue.", env_var)
 
+        # create backup of existing module file (if requested)
+        if os.path.exists(self.mod_filepath) and build_option('backup_modules'):
+            # backups of modules in Tcl syntax should be hidden to avoid that they're shown in 'module avail';
+            # backups of modules in Lua syntax do not need to be hidden:
+            # since they don't end in .lua (but in .lua.bck_*) Lmod will not pick them up anymore,
+            # which is better than hiding them (since --show-hidden still reveals them)
+            hidden = isinstance(self.module_generator, ModuleGeneratorTcl)
+            self.mod_file_backup = back_up_file(self.mod_filepath, backup_extension='bck', hidden=hidden)
+            print_msg("backup of existing module file stored at %s" % self.mod_file_backup)
+
         # check if main install needs to be skipped
         # - if a current module can be found, skip is ok
         # -- this is potentially very dangerous
@@ -2176,7 +2187,7 @@ def make_module_step(self, fake=False):
 
         :param fake: generate 'fake' module in temporary location, rather than actual module file
         """
-        modpath = self.module_generator.prepare(fake=fake)
+        modpath = self.module_generator.get_modules_path(fake=fake)
         mod_filepath = self.mod_filepath
         if fake:
             mod_filepath = self.module_generator.get_module_filepath(fake=fake)
@@ -2205,8 +2216,18 @@ def make_module_step(self, fake=False):
             write_file(mod_filepath, txt)
             self.log.info("Module file %s written: %s", mod_filepath, txt)
 
+            # if backup module file is there, print diff with newly generated module file
+            if self.mod_file_backup and not fake:
+                diff_msg = "comparing module file with backup %s; " % self.mod_file_backup
+                mod_diff = diff_files(self.mod_file_backup, mod_filepath)
+                if mod_diff:
+                    diff_msg += 'diff is:\n%s' % mod_diff
+                else:
+                    diff_msg += 'no differences found'
+                self.log.info(diff_msg)
+                print_msg(diff_msg)
+
             # invalidate relevant 'module avail'/'module show' cache entries
-            modpath = self.module_generator.get_modules_path(fake=fake)
             # consider both paths: for short module name, and subdir indicated by long module name
             paths = [modpath]
             if self.mod_subdir:

easybuild/tools/config.py

@@ -32,6 +32,7 @@
 :author: Jens Timmerman (Ghent University)
 :author: Toon Willems (Ghent University)
 :author: Ward Poelmans (Ghent University)
+:author: Damian Alvarez (Forschungszentrum Juelich GmbH)
 """
 import copy
 import glob
@@ -101,6 +102,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
 BUILD_OPTIONS_CMDLINE = {
     None: [
         'aggregate_regtest',
+        'backup_modules',
         'download_timeout',
         'dump_test_report',
         'easyblock',

easybuild/tools/filetools.py

@@ -35,7 +35,10 @@
 :author: Fotis Georgatos (Uni.Lu, NTUA)
 :author: Sotiris Fragkiskos (NTUA, CERN)
 :author: Davide Vanzo (ACCRE, Vanderbilt University)
+:author: Damian Alvarez (Forschungszentrum Juelich GmbH)
 """
+import datetime
+import difflib
 import fileinput
 import glob
 import hashlib
@@ -1163,6 +1166,45 @@ def rmtree2(path, n=3):
         _log.info("Path %s successfully removed." % path)
 
 
+def find_backup_name_candidate(src_file):
+    """Returns a non-existing file to be used as destination for backup files"""
+
+    # e.g. 20170817234510 on Aug 17th 2017 at 23:45:10
+    timestamp = datetime.datetime.now()
+    dst_file = '%s_%s' % (src_file, timestamp.strftime('%Y%m%d%H%M%S'))
+    while os.path.exists(dst_file):
+        _log.debug("Backup of %s at %s already found at %s, trying again in a second...", src_file, timestamp)
+        time.sleep(1)
+        timestamp = datetime.datetime.now()
+        dst_file = '%s_%s' % (src_file, timestamp.strftime('%Y%m%d%H%M%S'))
+
+    return dst_file
+
+
+def back_up_file(src_file, backup_extension='', hidden=False):
+    """
+    Backs up a file appending a backup extension and a number to it (if there is already an existing backup). Returns
+    the name of the backup
+
+    :param src_file: file to be back up
+    :param backup_extension: optional extension to use for the backup file
+    :param hidden: make backup hidden (leading dot in filename)
+    """
+    fn_prefix, fn_suffix = '', ''
+    if hidden:
+        fn_prefix = '.'
+    if backup_extension:
+        fn_suffix = '.%s' % backup_extension
+
+    src_dir, src_fn = os.path.split(src_file)
+    backup_fp = find_backup_name_candidate(os.path.join(src_dir, fn_prefix + src_fn + fn_suffix))
+
+    copy_file(src_file, backup_fp)
+    _log.info("File %s backed up in %s", src_file, backup_fp)
+
+    return backup_fp
+
+
 def move_logs(src_logfile, target_logfile):
     """Move log file(s)."""
 
@@ -1180,16 +1222,10 @@ def move_logs(src_logfile, target_logfile):
 
             # retain old logs
             if os.path.exists(new_log_path):
-                i = 0
-                oldlog_backup = "%s_%d" % (new_log_path, i)
-                while os.path.exists(oldlog_backup):
-                    i += 1
-                    oldlog_backup = "%s_%d" % (new_log_path, i)
-                shutil.move(new_log_path, oldlog_backup)
-                _log.info("Moved existing log file %s to %s" % (new_log_path, oldlog_backup))
+                back_up_file(new_log_path)
 
             # move log to target path
-            shutil.move(app_log, new_log_path)
+            move_file(app_log, new_log_path)
             _log.info("Moved log file %s to %s" % (src_logfile, new_log_path))
 
             if zip_log_cmd:
@@ -1548,3 +1584,33 @@ def copy(paths, target_path, force_in_dry_run=False):
             copy_dir(path, full_target_path, force_in_dry_run=force_in_dry_run)
         else:
             raise EasyBuildError("Specified path to copy is not an existing file or directory: %s", path)
+
+
+def move_file(path, target_path, force_in_dry_run=False):
+    """
+    Move a file from path to target_path
+
+    :param path: the original filepath
+    :param target_path: path to move the file to
+    :param force_in_dry_run: force running the command during dry run
+    """
+    if not force_in_dry_run and build_option('extended_dry_run'):
+        dry_run_msg("moved file %s to %s" % (path, target_path))
+    else:
+        # remove first to ensure portability (shutil.move might fail when overwriting files in some systems)
+        remove_file(target_path)
+        try:
+            mkdir(os.path.dirname(target_path), parents=True)
+            shutil.move(path, target_path)
+            _log.info("%s moved to %s", path, target_path)
+        except (IOError, OSError) as err:
+            raise EasyBuildError("Failed to move %s to %s: %s", path, target_path, err)
+
+
+def diff_files(path1, path2):
+    """
+    Return unified diff between two files
+    """
+    file1_lines = ['%s\n' % l for l in read_file(path1).split('\n')]
+    file2_lines = ['%s\n' % l for l in read_file(path2).split('\n')]
+    return ''.join(difflib.unified_diff(file1_lines, file2_lines, fromfile=path1, tofile=path2))

easybuild/tools/module_generator.py

@@ -31,6 +31,7 @@
 :author: Pieter De Baets (Ghent University)
 :author: Jens Timmerman (Ghent University)
 :author: Fotis Georgatos (Uni.Lu, NTUA)
+:author: Damian Alvarez (Forschungszentrum Juelich GmbH)
 """
 import os
 import re
@@ -173,24 +174,6 @@ def get_modules_path(self, fake=False, mod_path_suffix=None):
 
         return os.path.join(mod_path, mod_path_suffix)
 
-    def prepare(self, fake=False):
-        """
-        Prepare for generating module file: Creates the absolute filename for the module.
-        """
-        mod_path = self.get_modules_path(fake=fake)
-        # module file goes in general moduleclass category
-        # make symlink in moduleclass category
-
-        mod_filepath = self.get_module_filepath(fake=fake)
-        mkdir(os.path.dirname(mod_filepath), parents=True)
-
-        # remove module file if it's there (it'll be recreated), see EasyBlock.make_module
-        if os.path.exists(mod_filepath) and not build_option('extended_dry_run'):
-            self.log.debug("Removing existing module file %s", mod_filepath)
-            os.remove(mod_filepath)
-
-        return mod_path
-
     # From this point on just not implemented methods
 
     def comment(self, msg):

easybuild/tools/options.py

@@ -56,7 +56,7 @@
 from easybuild.framework.easyconfig.format.pyheaderconfigobj import build_easyconfig_constants_dict
 from easybuild.framework.easyconfig.tools import get_paths_for
 from easybuild.tools import build_log, run  # build_log should always stay there, to ensure EasyBuildLog
-from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError, raise_easybuilderror
+from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError, print_warning, raise_easybuilderror
 from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS
 from easybuild.tools.config import DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL, DEFAULT_MODULECLASSES
 from easybuild.tools.config import DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL, DEFAULT_PKG_TYPE
@@ -324,6 +324,8 @@ def override_options(self):
                                             None, 'store_true', False),
             'allow-use-as-root-and-accept-consequences': ("Allow using of EasyBuild as root (NOT RECOMMENDED!)",
                                                           None, 'store_true', False),
+            'backup-modules': ("Back up an existing module file, if any. Only works when using --module-only",
+                               None, 'store_true', None),  # default None to allow auto-enabling if not disabled
             'check-ebroot-env-vars': ("Action to take when defined $EBROOT* environment variables are found "
                                       "for which there is no matching loaded module; "
                                       "supported values: %s" % ', '.join(EBROOT_ENV_VAR_ACTIONS), None, 'store', WARN),
@@ -753,6 +755,11 @@ def postprocess(self):
         if self.options.last_log:
             self.options.terse = True
 
+        # auto-enable --backup-modules with --skip and --module-only, unless it was hard disabled
+        if (self.options.module_only or self.options.skip) and self.options.backup_modules is None:
+            self.log.debug("Auto-enabling --backup-modules because of --module-only or --skip")
+            self.options.backup_modules = True
+
         # make sure --optarch has a valid format, but do it only if we are not going to submit jobs. Otherwise it gets
         # processed twice and fails when trying to parse a dictionary as if it was a string
         if self.options.optarch and not self.options.job: