Skip to content

Commit

Permalink
Update config.guess for all R packages
Browse files Browse the repository at this point in the history 
Factor out the check and obtain config.guess from ConfigureMake into free functions
to be able to use them inside the rpackage EB.
Change the workflow for R packages to always extract which is more in line with the
regular workflow for other EasyConfigs.
This then allows to search for config.guess files to replace by the downloaded version
contained within EasyBuild.
  • Loading branch information
@Flamefire
Flamefire committed Feb 5, 2020
1 parent 85c2f77 commit 36755a4
Show file tree
Hide file tree
Showing 2 changed files with 114 additions and 93 deletions.
158 changes: 95 additions & 63 deletions easybuild/easyblocks/generic/configuremake.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,99 @@
DEFAULT_BUILD_CMD = 'make'
DEFAULT_INSTALL_CMD = 'make install'

def check_config_guess(config_guess, log=None):
"""Check timestamp & SHA256 checksum of config.guess script.
:param configu_gess: Path to config.guess script to check
:param log: Instance of a logger to use or None to run in quiet mode
:return: Whether the script is valid (matches the version and checksum)
"""
# config.guess includes a "timestamp='...'" indicating the version
config_guess_version = None
version_regex = re.compile("^timestamp='(.*)'", re.M)
res = version_regex.search(read_file(config_guess))
if res:
config_guess_version = res.group(1)

config_guess_checksum = compute_checksum(config_guess, checksum_type=CHECKSUM_TYPE_SHA256)
try:
config_guess_timestamp = datetime.fromtimestamp(os.stat(config_guess).st_mtime).isoformat()
except OSError as err:
if log:
log.warning("Failed to determine timestamp of %s: %s", config_guess, err)
config_guess_timestamp = None

# log version, timestamp & SHA256 checksum of config.guess
if log:
log.info("config.guess version: %s (last updated: %s, SHA256 checksum: %s)",
config_guess_version, config_guess_timestamp, config_guess_checksum)

result = True

if config_guess_version != CONFIG_GUESS_VERSION:
result = False
tup = (config_guess, config_guess_version, CONFIG_GUESS_VERSION)
if log:
log.warning("config.guess version at %s does not match expected version: %s vs %s" % tup)
if config_guess_checksum != CONFIG_GUESS_SHA256:
result = False
tup = (config_guess, config_guess_checksum, CONFIG_GUESS_SHA256)
if log:
log.warning("SHA256 checksum of config.guess at %s does not match expected checksum: %s vs %s" % tup)

return result

def obtain_config_guess(download_source_path=None, search_source_paths=None, log=None):
"""
Locate or download an up-to-date config.guess
:param download_source_path: Path to download config.guess to
:param search_source_paths: Paths to search for config.guess
:param log: Instance of a logger to use or None to run in quiet mode
:return: Path to config.guess or None
"""
eb_source_paths = source_paths()
if download_source_path is None:
download_source_path = eb_source_paths[0]
if search_source_paths is None:
search_source_paths = eb_source_paths

config_guess = 'config.guess'
sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake')

config_guess_path = None

# check if config.guess has already been downloaded to source path
for path in eb_source_paths:
cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess)
if os.path.isfile(cand_config_guess_path):
config_guess_path = cand_config_guess_path
if log:
log.info("Found recent %s at %s, using it if required", config_guess, config_guess_path)
break

# if not found, try to download it
if config_guess_path is None:
cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess)
config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID
downloaded_path = download_file(config_guess, config_guess_url, cand_config_guess_path)
if downloaded_path is not None:
# verify SHA256 checksum of download to avoid using a corrupted download
if verify_checksum(downloaded_path, CONFIG_GUESS_SHA256):
config_guess_path = downloaded_path
# add execute permissions
adjust_permissions(downloaded_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True)
if log:
log.info("Downloaded recent %s to %s, using it if required", config_guess, config_guess_path)
else:
if log:
log.warning("Checksum failed for downloaded file %s, not using it!", downloaded_path)
remove_file(downloaded_path)
elif log:
log.warning("Failed to download recent %s to %s", config_guess, cand_config_guess_path)

return config_guess_path


class ConfigureMake(EasyBlock):
"""
Expand Down Expand Up @@ -106,74 +199,13 @@ def obtain_config_guess(self, download_source_path=None, search_source_paths=Non
:param search_source_paths: Paths to search for config.guess
:return: Path to config.guess or None
"""
eb_source_paths = source_paths()
if download_source_path is None:
download_source_path = eb_source_paths[0]
if search_source_paths is None:
search_source_paths = eb_source_paths

config_guess = 'config.guess'
sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake')

config_guess_path = None

# check if config.guess has already been downloaded to source path
for path in eb_source_paths:
cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess)
if os.path.isfile(cand_config_guess_path):
config_guess_path = cand_config_guess_path
self.log.info("Found recent %s at %s, using it if required", config_guess, config_guess_path)
break

# if not found, try to download it
if config_guess_path is None:
cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess)
config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID
downloaded_path = download_file(config_guess, config_guess_url, cand_config_guess_path)
if downloaded_path is not None:
# verify SHA256 checksum of download to avoid using a corrupted download
if verify_checksum(downloaded_path, CONFIG_GUESS_SHA256):
config_guess_path = downloaded_path
# add execute permissions
adjust_permissions(downloaded_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True)
self.log.info("Downloaded recent %s to %s, using it if required", config_guess, config_guess_path)
else:
self.log.warning("Checksum failed for downloaded file %s, not using it!", downloaded_path)
remove_file(downloaded_path)
else:
self.log.warning("Failed to download recent %s to %s for use with ConfigureMake easyblock (if needed)",
config_guess, cand_config_guess_path)

return config_guess_path
return obtain_config_guess(download_source_path, search_source_paths, log=self.log)

def check_config_guess(self):
"""Check timestamp & SHA256 checksum of config.guess script."""
# log version, timestamp & SHA256 checksum of config.guess that was found (if any)
if self.config_guess:
# config.guess includes a "timestamp='...'" indicating the version
config_guess_version = None
version_regex = re.compile("^timestamp='(.*)'", re.M)
res = version_regex.search(read_file(self.config_guess))
if res:
config_guess_version = res.group(1)

config_guess_checksum = compute_checksum(self.config_guess, checksum_type=CHECKSUM_TYPE_SHA256)
try:
config_guess_timestamp = datetime.fromtimestamp(os.stat(self.config_guess).st_mtime).isoformat()
except OSError as err:
self.log.warning("Failed to determine timestamp of %s: %s", self.config_guess, err)
config_guess_timestamp = None

self.log.info("config.guess version: %s (last updated: %s, SHA256 checksum: %s)",
config_guess_version, config_guess_timestamp, config_guess_checksum)

if config_guess_version != CONFIG_GUESS_VERSION:
tup = (self.config_guess, config_guess_version, CONFIG_GUESS_VERSION)
print_warning("config.guess version at %s does not match expected version: %s vs %s" % tup)

if config_guess_checksum != CONFIG_GUESS_SHA256:
tup = (self.config_guess, config_guess_checksum, CONFIG_GUESS_SHA256)
print_warning("SHA256 checksum of config.guess at %s does not match expected checksum: %s vs %s" % tup)
check_config_guess(self.config_guess, log=self.log)

def fetch_step(self, *args, **kwargs):
"""Custom fetch step for ConfigureMake so we use an updated config.guess."""
Expand Down
49 changes: 19 additions & 30 deletions easybuild/easyblocks/generic/rpackage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,11 @@
import shutil

from easybuild.easyblocks.r import EXTS_FILTER_R_PACKAGES, EB_R
from easybuild.easyblocks.generic.configuremake import check_config_guess, obtain_config_guess
from easybuild.framework.easyconfig import CUSTOM
from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import mkdir
from easybuild.tools.filetools import mkdir, copy_file
from easybuild.tools.run import run_cmd, parse_log_for_error


Expand Down Expand Up @@ -132,12 +133,8 @@ def make_cmdline_cmd(self, prefix=None):
else:
prefix = ''

if self.cfg['unpack_sources']:
loc = self.start_dir
elif self.patches:
loc = self.ext_dir
else:
loc = self.ext_src
loc = self.start_dir or self.ext_dir or self.ext_src

cmd = ' '.join([
self.cfg['preinstallopts'],
"R CMD INSTALL",
Expand All @@ -152,23 +149,6 @@ def make_cmdline_cmd(self, prefix=None):
self.log.debug("make_cmdline_cmd returns %s" % cmd)
return cmd, None

def extract_step(self):
"""Source should not be extracted."""

if self.cfg['unpack_sources']:
super(RPackage, self).extract_step()
elif len(self.src) > 1:
raise EasyBuildError("Don't know how to handle R packages with multiple sources.'")
else:
try:
shutil.copy2(self.src[0]['path'], self.builddir)
except OSError as err:
raise EasyBuildError("Failed to copy source to build dir: %s", err)
self.ext_src = self.src[0]['name']

# set final path since it can't be determined from unpacked sources (used for guessing start_dir)
self.src[0]['finalpath'] = self.builddir

def configure_step(self):
"""No configuration for installing R packages."""
pass
Expand All @@ -195,9 +175,19 @@ def install_R_package(self, cmd, inp=None):
else:
self.log.debug("R package %s installed succesfully" % self.name)

def update_config_guess(self, src_dir):
"""Update any config.guess found in src_dir"""
for config_guess_dir in (root for root, _, files in os.walk(src_dir) if 'config.guess' in files):
config_guess = os.path.join(config_guess_dir, 'config.guess')
if not check_config_guess(config_guess):
updated_config_guess = obtain_config_guess(log=self.log)
copy_file(updated_config_guess, config_guess)

def install_step(self):
"""Install procedure for R packages."""

# Update config.guess if the package was extracted
if self.start_dir:
self.update_config_guess(self.start_dir)
cmd, stdin = self.make_cmdline_cmd(prefix=os.path.join(self.installdir, self.cfg['exts_subdir']))
self.install_R_package(cmd, inp=stdin)

Expand All @@ -215,16 +205,15 @@ def run(self):
lib_install_prefix = os.path.join(self.installdir, self.cfg['exts_subdir'])
mkdir(lib_install_prefix, parents=True)

if self.patches:
super(RPackage, self).run(unpack_src=True)
else:
super(RPackage, self).run()

if self.src:
super(RPackage, self).run(unpack_src=True)
self.ext_src = self.src
self.update_config_guess(self.ext_dir)
self.log.debug("Installing R package %s version %s." % (self.name, self.version))
cmd, stdin = self.make_cmdline_cmd(prefix=lib_install_prefix)
else:
if self.patches:
raise EasyBuildError("Cannot patch R package %s as no explicit source is given!", self.name)
self.log.debug("Installing most recent version of R package %s (source not found)." % self.name)
cmd, stdin = self.make_r_cmd(prefix=lib_install_prefix)

Expand Down

0 comments on commit 36755a4

Please sign in to comment.