Skip to content

Commit

Permalink
Add dynamic search option for package sources (spack#2270)
Browse files Browse the repository at this point in the history
Package provides a 'list_url' attribute which may be searched to find
download links. spack#1822 created a slowdown for all tests by always
searching this URL. This reenables dynamic search only in cases where
all other fetchers fail. This also only enables dynamic search when
'mirror_only' is set to false.
  • Loading branch information
scheibelp authored and Elizabeth Fischer committed Dec 4, 2016
1 parent 7ef48b7 commit 330008b
Show file tree
Hide file tree
Showing 4 changed files with 70 additions and 8 deletions.
5 changes: 4 additions & 1 deletion lib/spack/spack/fetch_strategy.py
Original file line number Diff line number Diff line change
Expand Up @@ -911,7 +911,10 @@ def from_list_url(pkg):
versions = pkg.fetch_remote_versions()
try:
url_from_list = versions[pkg.version]
return URLFetchStrategy(url=url_from_list, digest=None)
digest = None
if pkg.version in pkg.versions:
digest = pkg.versions[pkg.version].get('md5', None)
return URLFetchStrategy(url=url_from_list, digest=digest)
except KeyError:
tty.msg("Can not find version %s in url_list" %
self.version)
Expand Down
8 changes: 7 additions & 1 deletion lib/spack/spack/package.py
Original file line number Diff line number Diff line change
Expand Up @@ -703,7 +703,13 @@ def _make_root_stage(self, fetcher):
# Construct a path where the stage should build..
s = self.spec
stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash())
stage = Stage(fetcher, mirror_path=mp, name=stage_name, path=self.path)

def download_search():
dynamic_fetcher = fs.from_list_url(self)
return [dynamic_fetcher] if dynamic_fetcher else []

stage = Stage(fetcher, mirror_path=mp, name=stage_name, path=self.path,
search_fn=download_search)
return stage

def _make_stage(self):
Expand Down
19 changes: 13 additions & 6 deletions lib/spack/spack/stage.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ class Stage(object):
def __init__(
self, url_or_fetch_strategy,
name=None, mirror_path=None, keep=False, path=None, lock=True,
alternate_fetchers=None):
search_fn=None):
"""Create a stage object.
Parameters:
url_or_fetch_strategy
Expand Down Expand Up @@ -198,7 +198,7 @@ def __init__(
self.fetcher.set_stage(self)
# self.fetcher can change with mirrors.
self.default_fetcher = self.fetcher
self.alternate_fetchers = alternate_fetchers
self.search_fn = search_fn
# used for mirrored archives of repositories.
self.skip_checksum_for_mirror = True

Expand Down Expand Up @@ -416,10 +416,17 @@ def fetch(self, mirror_only=False):
self.mirror_path, digest, expand=expand,
extension=extension))

if self.alternate_fetchers:
fetchers.extend(self.alternate_fetchers)

for fetcher in fetchers:
def generate_fetchers():
for fetcher in fetchers:
yield fetcher
# The search function may be expensive, so wait until now to
# call it so the user can stop if a prior fetcher succeeded
if self.search_fn and not mirror_only:
dynamic_fetchers = self.search_fn()
for fetcher in dynamic_fetchers:
yield fetcher

for fetcher in generate_fetchers():
try:
fetcher.set_stage(self)
self.fetcher = fetcher
Expand Down
46 changes: 46 additions & 0 deletions lib/spack/spack/test/stage.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,26 @@ def use_tmp(use_tmp):
yield


def fail_search_fn():
raise Exception("This should not have been called")


class FailingFetchStrategy(spack.fetch_strategy.FetchStrategy):
def fetch(self):
raise spack.fetch_strategy.FailedDownloadError(
"<non-existent URL>",
"This implementation of FetchStrategy always fails")


class MockSearchFunction(object):
def __init__(self):
self.performed_search = False

def __call__(self):
self.performed_search = True
return []


class StageTest(MockPackagesTest):

def setUp(self):
Expand Down Expand Up @@ -251,6 +271,32 @@ def test_fetch(self):
self.check_fetch(stage, self.stage_name)
self.check_destroy(stage, self.stage_name)

def test_no_search_if_default_succeeds(self):
with Stage(self.archive_url, name=self.stage_name,
search_fn=fail_search_fn) as stage:
stage.fetch()
self.check_destroy(stage, self.stage_name)

def test_no_search_mirror_only(self):
with Stage(FailingFetchStrategy(), name=self.stage_name,
search_fn=fail_search_fn) as stage:
try:
stage.fetch(mirror_only=True)
except spack.fetch_strategy.FetchError:
pass
self.check_destroy(stage, self.stage_name)

def test_search_if_default_fails(self):
test_search = MockSearchFunction()
with Stage(FailingFetchStrategy(), name=self.stage_name,
search_fn=test_search) as stage:
try:
stage.fetch(mirror_only=False)
except spack.fetch_strategy.FetchError:
pass
self.check_destroy(stage, self.stage_name)
self.assertTrue(test_search.performed_search)

def test_expand_archive(self):
with Stage(self.archive_url, name=self.stage_name) as stage:
stage.fetch()
Expand Down

0 comments on commit 330008b

Please sign in to comment.