feat: Bulk build failure wiki

bioconda_utils/build_failure.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import time
 from typing import Optional, Union
 import subprocess as sp
@@ -13,6 +14,8 @@
 import pandas as pd
 import networkx as nx
 
+from .githandler import BiocondaRepo
+
 from bioconda_utils.recipe import Recipe
 from bioconda_utils import graph, utils
 from bioconda_utils.githandler import GitHandler
@@ -179,7 +182,7 @@ def category(self, value):
         self.inner["category"] = value
 
 
-def collect_build_failure_dataframe(recipe_folder, config, channel, link_fmt="txt", link_prefix=""):
+def collect_build_failure_dataframe(recipe_folder, config, channel, link_fmt="txt", link_prefix="", git_range=None):
     def get_build_failure_records(recipe):
         return filter(
             BuildFailureRecord.exists, 
@@ -190,6 +193,21 @@ def has_build_failure(recipe):
         return any(get_build_failure_records(recipe))
 
     recipes = list(utils.get_recipes(recipe_folder))
+
+    if git_range:
+        if not git_range or len(git_range) > 2:
+            sys.exit("--git-range may have only one or two arguments")
+        other = git_range[0]
+        ref = "HEAD" if len(git_range) == 1 else git_range[1]
+        repo = BiocondaRepo(recipe_folder)
+        changed_recipes = repo.get_recipes_to_build(ref, other)
+        logger.info("Constraining to %s git modified recipes%s.", len(changed_recipes),
+                    utils.ellipsize_recipes(changed_recipes, recipe_folder))
+        recipes = [recipe for recipe in recipes if recipe in set(changed_recipes)]
+        if len(recipes) != len(changed_recipes):
+            logger.info("Overlap was %s recipes%s.", len(recipes),
+                        utils.ellipsize_recipes(recipes, recipe_folder))
+
     dag, _ = graph.build(recipes, config)
 
     def get_data():
@@ -214,10 +232,11 @@ def get_data():
             recs = list(get_build_failure_records(recipe))
 
             failures = ", ".join(utils.format_link(rec.path, link_fmt, prefix=link_prefix, label=rec.platform) for rec in recs)
+            categories = ", ".join(rec.category for rec in recs)
             skiplisted = any(rec.skiplist for rec in recs)
             prs = utils.format_link(f"https://github.com/bioconda/bioconda-recipes/pulls?q=is%3Apr+is%3Aopen+{package}", link_fmt, label="show")
-            yield (recipe, downloads, descendants, skiplisted, failures, prs)
+            yield (recipe, downloads, descendants, skiplisted, categories, failures, prs)
 
-    data = pd.DataFrame(get_data(), columns=["recipe", "downloads", "depending", "skiplisted", "build failures", "pull requests"])
+    data = pd.DataFrame(get_data(), columns=["recipe", "downloads", "depending", "skiplisted", "category", "build failures", "pull requests"])
     data.sort_values(by=["depending", "downloads"], ascending=False, inplace=True)
     return data

bioconda_utils/cli.py

@@ -1075,7 +1075,11 @@ def annotate_build_failures(recipes, skiplist=False, reason=None, category=None,
 @arg('--channel', help="Channel with packages to check", default="bioconda")
 @arg('--output-format', help="Output format", choices=['txt', 'markdown'], default="txt")
 @arg('--link-prefix', help="Prefix for links to build failures", default='')
-def list_build_failures(recipe_folder, config, channel=None, output_format=None, link_prefix=None):
+@arg('--git-range', nargs='+',
+     help='''Git range (e.g. commits or something like
+     "master HEAD" to check commits in HEAD vs master, or just "HEAD" to
+     include uncommitted changes).''')
+def list_build_failures(recipe_folder, config, channel=None, output_format=None, link_prefix=None, git_range=None):
     """List recipes with build failure records"""
 
     df = collect_build_failure_dataframe(
@@ -1084,6 +1088,7 @@ def list_build_failures(recipe_folder, config, channel=None, output_format=None,
         channel,
         link_fmt=output_format,
         link_prefix=link_prefix,
+        git_range=git_range
     )
     if output_format == "markdown":
         fmt_writer = pandas.DataFrame.to_markdown

bioconda_utils/utils.py

@@ -1645,4 +1645,6 @@ def yaml_remove_invalid_chars(text: str, valid_chars_re=re.compile(r"[^ \t\n\w\d
 def get_package_downloads(channel, package):
     """Use anaconda API to obtain download counts."""
     data = requests.get(f"https://api.anaconda.org/package/{channel}/{package}").json()
-    return sum(rec["ndownloads"] for rec in data["files"])
+    if "files" in data:
+        return sum(rec["ndownloads"] for rec in data["files"])
+    return 0