added support to 'download' sources from git

easybuild/framework/easyblock.py

@@ -35,6 +35,7 @@
 :author: Ward Poelmans (Ghent University)
 :author: Fotis Georgatos (Uni.Lu, NTUA)
 :author: Damian Alvarez (Forschungszentrum Juelich GmbH)
+:author: Maxime Boissonneault (Compute Canada)
 """
 
 import copy
@@ -73,6 +74,7 @@
 from easybuild.tools.filetools import compute_checksum, copy_file, derive_alt_pypi_url, diff_files, download_file
 from easybuild.tools.filetools import encode_class_name, extract_file, is_alt_pypi_url, mkdir, move_logs, read_file
 from easybuild.tools.filetools import remove_file, rmtree2, verify_checksum, weld_paths, write_file
+from easybuild.tools.filetools import get_source_from_git
 from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
 from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTPROC_STEP, PREPARE_STEP
 from easybuild.tools.hooks import READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP, run_hook
@@ -337,17 +339,19 @@ def fetch_sources(self, sources=None, checksums=None):
             checksums = self.cfg['checksums']
 
         for index, source in enumerate(sources):
-            extract_cmd, download_filename, source_urls = None, None, None
+            extract_cmd, download_filename, source_urls, git_config = None, None, None, {}
 
             if isinstance(source, basestring):
                 filename = source
 
             elif isinstance(source, dict):
+                # Making a copy to avoid modifying the object with pops
                 source = source.copy()
                 filename = source.pop('filename', None)
                 extract_cmd = source.pop('extract_cmd', None)
                 download_filename = source.pop('download_filename', None)
                 source_urls = source.pop('source_urls', None)
+                git_config = source.pop('git_config', {})
                 if source:
                     raise EasyBuildError("Found one or more unexpected keys in 'sources' specification: %s", source)
 
@@ -361,7 +365,7 @@ def fetch_sources(self, sources=None, checksums=None):
             # check if the sources can be located
             force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
             path = self.obtain_file(filename, download_filename=download_filename, force_download=force_download,
-                                    urls=source_urls)
+                                    urls=source_urls, git_config=git_config)
             if path:
                 self.log.debug('File %s found for source %s' % (path, filename))
                 self.src.append({
@@ -559,7 +563,8 @@ def fetch_extension_sources(self, skip_checksums=False):
 
         return exts_sources
 
-    def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False):
+    def obtain_file(self, filename, extension=False, urls=None, download_filename=None, force_download=False,
+                    git_config=None):
         """
         Locate the file with the given name
         - searches in different subdirectories of source path
@@ -568,6 +573,7 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
         :param extension: indicates whether locations for extension sources should also be considered
         :param urls: list of source URLs where this file may be available
         :param download_filename: filename with which the file should be downloaded, and then renamed to <filename>
+        :param git_config: dictionary to parametrize how to download the repository
         :force_download: always try to download file, even if it's already available in source path
         """
         srcpaths = source_paths()
@@ -659,10 +665,14 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
 
                     break  # no need to try other source paths
 
+            targetdir = os.path.join(srcpaths[0], self.name.lower()[0], self.name)
+
             if foundfile:
                 if self.dry_run:
                     self.dry_run_msg("  * %s found at %s", filename, foundfile)
                 return foundfile
+            elif git_config:
+                return get_source_from_git(filename, targetdir, git_config)
             else:
                 # try and download source files from specified source URLs
                 if urls:
@@ -671,7 +681,6 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                     source_urls = []
                 source_urls.extend(self.cfg['source_urls'])
 
-                targetdir = os.path.join(srcpaths[0], self.name.lower()[0], self.name)
                 mkdir(targetdir, parents=True)
 
                 for url in source_urls:

easybuild/tools/filetools.py

@@ -36,6 +36,7 @@
 :author: Sotiris Fragkiskos (NTUA, CERN)
 :author: Davide Vanzo (ACCRE, Vanderbilt University)
 :author: Damian Alvarez (Forschungszentrum Juelich GmbH)
+:author: Maxime Boissonneault (Compute Canada)
 """
 import datetime
 import difflib
@@ -1627,6 +1628,63 @@ def copy(paths, target_path, force_in_dry_run=False):
         else:
             raise EasyBuildError("Specified path to copy is not an existing file or directory: %s", path)
 
+def get_source_from_git(filename, targetdir, git_config):
+    """
+    Downloads a git repository, at a specific tag or commit, recursively or not, and make an archive with it
+
+    :param filename: name of the archive to save the code to (must be .tar.gz)
+    :param targetdir: target directory where to save the archive to
+    :param git_config: dictionary containing url, repo_name, recursive, and one of tag or commit
+    """
+    # if a non-empty dictionary was provided, download from git and archive
+    if not isinstance(git_config, dict):
+        raise EasyBuildError("Found a non null git_config in 'sources', but value is not a dictionary")
+
+    # Making a copy to avoid modifying the object with pops
+    git_config = git_config.copy()
+    tag = git_config.pop('tag', None)
+    url = git_config.pop('url', None)
+    repo_name = git_config.pop('repo_name', None)
+    commit = git_config.pop('commit', None)
+    recursive = git_config.pop('recursive', False)
+    if git_config:
+        raise EasyBuildError("Found one or more unexpected keys in 'git_config' specification: %s", git_config)
+    if not repo_name:
+        raise EasyBuildError("repo_name not specified in git_config parameter")
+    if not tag and not commit:
+        raise EasyBuildError("Neither tag nor commit found in git_config parameter")
+    if tag and commit:
+        raise EasyBuildError("Tag and commit are mutually exclusive in git_config parameter")
+    if not url:
+        raise EasyBuildError("url not specified in git_config parameter")
+    if '.tar.gz' not in filename:
+        raise EasyBuildError("git_config only supports filename ending in .tar.gz")
+
+    mkdir(targetdir, parents=True)
+    targetpath = os.path.join(targetdir, filename)
+
+    cwd = change_dir(targetdir)
+    recursive = " --recursive " if recursive else ""
+    if tag:
+        cmd = "git clone --branch %s %s %s/%s.git " % (tag, recursive, url, repo_name)
+    else:
+        cmd = "git clone %s %s/%s.git" % (recursive, url, repo_name)
+    (cmdstdouterr, ec) = run.run_cmd(cmd, log_all=True, log_ok=False, simple=False, regexp=False)
+
+    #if a specific commit is asked for, check it out
+    if commit:
+        change_dir(os.path.join(targetdir, repo_name))
+        recursive = " && git submodule update " if recursive else ""
+        cmd = "git checkout %s %s " % (commit, recursive)
+        (cmdstdouterr, ec) = run.run_cmd(cmd, log_all=True, log_ok=False, simple=False, regexp=False)
+        change_dir(targetdir)
+
+    #create an archive and delete the git repo
+    cmd = "tar cfvz %s --exclude-vcs %s && rm -rf %s" % (targetpath, repo_name, repo_name)
+    (cmdstdouterr, ec) = run.run_cmd(cmd, log_all=True, log_ok=False, simple=False, regexp=False)
+
+    change_dir(cwd)
+    return targetpath
 
 def move_file(path, target_path, force_in_dry_run=False):
     """