fix: adds alternate lookup if base lookup fails (#896)

This is a first pass attempt at fixing #895.

Main changes are updating the GithubRelease hoster to look for alternate
paths via the GitHub API. The API sends a JSON response (example:
https://api.github.com/repos/PacificBiosciences/HiPhase/releases) that
contains each release along with a list of assets. The assets are
searched for files that match the provides pattern. Currently, this
first pass puts this as a backup to the main one (which I think is
always failing now). I also did not update the tests, I think someone
may have a better idea of how to do that. Happy to help push this
further, but wanted to get the ball rolling.

---------

Co-authored-by: Johannes Köster <[email protected]>

bioconda_utils/hosters.py

@@ -326,7 +326,61 @@ class GithubBase(OrderedHTMLHoster):
 class GithubRelease(GithubBase):
     """Matches release artifacts uploaded to Github"""
     link_pattern = r"/{account}/{project}/releases/download/{tag}/{fname}{ext}?"
+    alt_releases_formats = ["https://api.github.com/repos/{account}/{project}/releases"]
 
+    async def get_versions(self, req, orig_version):
+        # first, try the older version when HTML worked
+        matches = await super().get_versions(req, orig_version)
+        if len(matches) > 0:
+            return matches
+
+        # old version found nothing, try with the alternate github API URLs which return JSON
+        self.releases_urls = [
+            template.format_map(self.vals)
+            for template in self.alt_releases_formats
+        ]
+
+        # this is basically copied from a mixture of the base version and the JSON version
+        # need to compile the link regex
+        exclude = set(self.exclude)
+        vals = {key: val
+                for key, val in self.vals.items()
+                if key not in exclude}
+        link_pattern = replace_named_capture_group(self.link_pattern_compiled, vals)
+        link_re = re.compile(link_pattern)
+
+        # now iterate over the alter release URLs
+        matches = []
+        for url in self.releases_urls:
+            text = await req.get_text_from_url(url)
+            data = json.loads(text)
+
+            # structured as an array of tagged releases
+            for tag_dict in data:
+                # each release has an asset dict
+                for asset_dict in tag_dict.get('assets', []):
+                    # there is a direct download link for each asset, which should make the typical pattern for an HTML user
+                    download_url = asset_dict['browser_download_url']
+                    re_match = link_re.search(download_url)
+
+                    if re_match:
+                        # this one matches the pattern
+                        # link - just copy the download_url in full
+                        # version - pull out of the regex match
+                        data = re_match.groupdict()
+                        matches.append({
+                            'link' : download_url,
+                            'version' : data['version']
+                        })
+
+        # now strip down to the version(s) that are more recent than what currently is in bioconda
+        num = None
+        for num, match in enumerate(matches):
+            if match["version"] == self.vals["version"]:
+                break
+        if num is None:
+            return matches
+        return matches[:num + 1]
 
 class GithubTag(GithubBase):
     """Matches GitHub repository archives created automatically from tags"""

test/test_bioconductor_skeleton.py

@@ -98,7 +98,7 @@ def test_meta_contents(tmpdir, bioc_fetch):
 @pytest.mark.skip(reason="Does not currently work inside of the CI (cannot find the release) although it seems to work fine locally.")
 def test_find_best_bioc_version():
 
-    assert bioconductor_skeleton.find_best_bioc_version('DESeq2', '1.36.0') == '3.15'
+    assert bioconductor_skeleton.find_best_bioc_version('DESeq2', '1.40.1') == '3.17'
 
     # Non-existent version:
     with pytest.raises(bioconductor_skeleton.PackageNotFoundError):