From 330008bfde9500855d17cae6c300750e156a1953 Mon Sep 17 00:00:00 2001 From: scheibelp Date: Thu, 24 Nov 2016 12:21:05 -0800 Subject: [PATCH] Add dynamic search option for package sources (#2270) Package provides a 'list_url' attribute which may be searched to find download links. #1822 created a slowdown for all tests by always searching this URL. This reenables dynamic search only in cases where all other fetchers fail. This also only enables dynamic search when 'mirror_only' is set to false. --- lib/spack/spack/fetch_strategy.py | 5 +++- lib/spack/spack/package.py | 8 +++++- lib/spack/spack/stage.py | 19 +++++++++---- lib/spack/spack/test/stage.py | 46 +++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 8 deletions(-) diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 5084a68e08566f..23f3b9a41eebb3 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -911,7 +911,10 @@ def from_list_url(pkg): versions = pkg.fetch_remote_versions() try: url_from_list = versions[pkg.version] - return URLFetchStrategy(url=url_from_list, digest=None) + digest = None + if pkg.version in pkg.versions: + digest = pkg.versions[pkg.version].get('md5', None) + return URLFetchStrategy(url=url_from_list, digest=digest) except KeyError: tty.msg("Can not find version %s in url_list" % self.version) diff --git a/lib/spack/spack/package.py b/lib/spack/spack/package.py index 7462a735337f12..8bb19042cc4ecb 100644 --- a/lib/spack/spack/package.py +++ b/lib/spack/spack/package.py @@ -703,7 +703,13 @@ def _make_root_stage(self, fetcher): # Construct a path where the stage should build.. s = self.spec stage_name = "%s-%s-%s" % (s.name, s.version, s.dag_hash()) - stage = Stage(fetcher, mirror_path=mp, name=stage_name, path=self.path) + + def download_search(): + dynamic_fetcher = fs.from_list_url(self) + return [dynamic_fetcher] if dynamic_fetcher else [] + + stage = Stage(fetcher, mirror_path=mp, name=stage_name, path=self.path, + search_fn=download_search) return stage def _make_stage(self): diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index 4157511ce0bdcf..91f77839d8c9b5 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -162,7 +162,7 @@ class Stage(object): def __init__( self, url_or_fetch_strategy, name=None, mirror_path=None, keep=False, path=None, lock=True, - alternate_fetchers=None): + search_fn=None): """Create a stage object. Parameters: url_or_fetch_strategy @@ -198,7 +198,7 @@ def __init__( self.fetcher.set_stage(self) # self.fetcher can change with mirrors. self.default_fetcher = self.fetcher - self.alternate_fetchers = alternate_fetchers + self.search_fn = search_fn # used for mirrored archives of repositories. self.skip_checksum_for_mirror = True @@ -416,10 +416,17 @@ def fetch(self, mirror_only=False): self.mirror_path, digest, expand=expand, extension=extension)) - if self.alternate_fetchers: - fetchers.extend(self.alternate_fetchers) - - for fetcher in fetchers: + def generate_fetchers(): + for fetcher in fetchers: + yield fetcher + # The search function may be expensive, so wait until now to + # call it so the user can stop if a prior fetcher succeeded + if self.search_fn and not mirror_only: + dynamic_fetchers = self.search_fn() + for fetcher in dynamic_fetchers: + yield fetcher + + for fetcher in generate_fetchers(): try: fetcher.set_stage(self) self.fetcher = fetcher diff --git a/lib/spack/spack/test/stage.py b/lib/spack/spack/test/stage.py index 64cfa222db6765..cfeb80dd35706c 100644 --- a/lib/spack/spack/test/stage.py +++ b/lib/spack/spack/test/stage.py @@ -59,6 +59,26 @@ def use_tmp(use_tmp): yield +def fail_search_fn(): + raise Exception("This should not have been called") + + +class FailingFetchStrategy(spack.fetch_strategy.FetchStrategy): + def fetch(self): + raise spack.fetch_strategy.FailedDownloadError( + "", + "This implementation of FetchStrategy always fails") + + +class MockSearchFunction(object): + def __init__(self): + self.performed_search = False + + def __call__(self): + self.performed_search = True + return [] + + class StageTest(MockPackagesTest): def setUp(self): @@ -251,6 +271,32 @@ def test_fetch(self): self.check_fetch(stage, self.stage_name) self.check_destroy(stage, self.stage_name) + def test_no_search_if_default_succeeds(self): + with Stage(self.archive_url, name=self.stage_name, + search_fn=fail_search_fn) as stage: + stage.fetch() + self.check_destroy(stage, self.stage_name) + + def test_no_search_mirror_only(self): + with Stage(FailingFetchStrategy(), name=self.stage_name, + search_fn=fail_search_fn) as stage: + try: + stage.fetch(mirror_only=True) + except spack.fetch_strategy.FetchError: + pass + self.check_destroy(stage, self.stage_name) + + def test_search_if_default_fails(self): + test_search = MockSearchFunction() + with Stage(FailingFetchStrategy(), name=self.stage_name, + search_fn=test_search) as stage: + try: + stage.fetch(mirror_only=False) + except spack.fetch_strategy.FetchError: + pass + self.check_destroy(stage, self.stage_name) + self.assertTrue(test_search.performed_search) + def test_expand_archive(self): with Stage(self.archive_url, name=self.stage_name) as stage: stage.fetch()