From addce1b73d9e45e9407bd02c5bd2ac589ab1b75c Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Thu, 14 Dec 2023 13:08:16 -0500 Subject: [PATCH 1/4] handle new recipes that haven't been uploaded --- bioconda_utils/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bioconda_utils/utils.py b/bioconda_utils/utils.py index 13855f1a77..2e3277ecb7 100644 --- a/bioconda_utils/utils.py +++ b/bioconda_utils/utils.py @@ -1645,4 +1645,6 @@ def yaml_remove_invalid_chars(text: str, valid_chars_re=re.compile(r"[^ \t\n\w\d def get_package_downloads(channel, package): """Use anaconda API to obtain download counts.""" data = requests.get(f"https://api.anaconda.org/package/{channel}/{package}").json() - return sum(rec["ndownloads"] for rec in data["files"]) + if "files" in data: + return sum(rec["ndownloads"] for rec in data["files"]) + return 0 From 126295885520c350459e555d3041d849184cfaf1 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Thu, 14 Dec 2023 13:09:21 -0500 Subject: [PATCH 2/4] feat: git-range for list-build-failures --- bioconda_utils/build_failure.py | 20 +++++++++++++++++++- bioconda_utils/cli.py | 7 ++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/bioconda_utils/build_failure.py b/bioconda_utils/build_failure.py index bec184754b..9f4bffee9a 100644 --- a/bioconda_utils/build_failure.py +++ b/bioconda_utils/build_failure.py @@ -1,4 +1,5 @@ import os +import sys import time from typing import Optional, Union import subprocess as sp @@ -13,6 +14,8 @@ import pandas as pd import networkx as nx +from .githandler import BiocondaRepo, install_gpg_key + from bioconda_utils.recipe import Recipe from bioconda_utils import graph, utils from bioconda_utils.githandler import GitHandler @@ -179,7 +182,7 @@ def category(self, value): self.inner["category"] = value -def collect_build_failure_dataframe(recipe_folder, config, channel, link_fmt="txt", link_prefix=""): +def collect_build_failure_dataframe(recipe_folder, config, channel, link_fmt="txt", link_prefix="", git_range=None): def get_build_failure_records(recipe): return filter( BuildFailureRecord.exists, @@ -190,6 +193,21 @@ def has_build_failure(recipe): return any(get_build_failure_records(recipe)) recipes = list(utils.get_recipes(recipe_folder)) + + if git_range: + if not git_range or len(git_range) > 2: + sys.exit("--git-range may have only one or two arguments") + other = git_range[0] + ref = "HEAD" if len(git_range) == 1 else git_range[1] + repo = BiocondaRepo(recipe_folder) + changed_recipes = repo.get_recipes_to_build(ref, other) + logger.info("Constraining to %s git modified recipes%s.", len(changed_recipes), + utils.ellipsize_recipes(changed_recipes, recipe_folder)) + recipes = [recipe for recipe in recipes if recipe in set(changed_recipes)] + if len(recipes) != len(changed_recipes): + logger.info("Overlap was %s recipes%s.", len(recipes), + utils.ellipsize_recipes(recipes, recipe_folder)) + dag, _ = graph.build(recipes, config) def get_data(): diff --git a/bioconda_utils/cli.py b/bioconda_utils/cli.py index 1b89e5f6ad..8007421258 100644 --- a/bioconda_utils/cli.py +++ b/bioconda_utils/cli.py @@ -1075,7 +1075,11 @@ def annotate_build_failures(recipes, skiplist=False, reason=None, category=None, @arg('--channel', help="Channel with packages to check", default="bioconda") @arg('--output-format', help="Output format", choices=['txt', 'markdown'], default="txt") @arg('--link-prefix', help="Prefix for links to build failures", default='') -def list_build_failures(recipe_folder, config, channel=None, output_format=None, link_prefix=None): +@arg('--git-range', nargs='+', + help='''Git range (e.g. commits or something like + "master HEAD" to check commits in HEAD vs master, or just "HEAD" to + include uncommitted changes).''') +def list_build_failures(recipe_folder, config, channel=None, output_format=None, link_prefix=None, git_range=None): """List recipes with build failure records""" df = collect_build_failure_dataframe( @@ -1084,6 +1088,7 @@ def list_build_failures(recipe_folder, config, channel=None, output_format=None, channel, link_fmt=output_format, link_prefix=link_prefix, + git_range=git_range ) if output_format == "markdown": fmt_writer = pandas.DataFrame.to_markdown From 8e0140b91c6e597b3fea1fcdbb6b612cd399b136 Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Thu, 14 Dec 2023 13:36:55 -0500 Subject: [PATCH 3/4] remove unused import --- bioconda_utils/build_failure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bioconda_utils/build_failure.py b/bioconda_utils/build_failure.py index 9f4bffee9a..d8187e277a 100644 --- a/bioconda_utils/build_failure.py +++ b/bioconda_utils/build_failure.py @@ -14,7 +14,7 @@ import pandas as pd import networkx as nx -from .githandler import BiocondaRepo, install_gpg_key +from .githandler import BiocondaRepo from bioconda_utils.recipe import Recipe from bioconda_utils import graph, utils From d5a1e531ad7e7f57adb8b95994d7f79e3f0169bf Mon Sep 17 00:00:00 2001 From: aliciaaevans Date: Fri, 15 Dec 2023 15:12:33 -0500 Subject: [PATCH 4/4] category column --- bioconda_utils/build_failure.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bioconda_utils/build_failure.py b/bioconda_utils/build_failure.py index d8187e277a..d2e7da2804 100644 --- a/bioconda_utils/build_failure.py +++ b/bioconda_utils/build_failure.py @@ -232,10 +232,11 @@ def get_data(): recs = list(get_build_failure_records(recipe)) failures = ", ".join(utils.format_link(rec.path, link_fmt, prefix=link_prefix, label=rec.platform) for rec in recs) + categories = ", ".join(rec.category for rec in recs) skiplisted = any(rec.skiplist for rec in recs) prs = utils.format_link(f"https://github.com/bioconda/bioconda-recipes/pulls?q=is%3Apr+is%3Aopen+{package}", link_fmt, label="show") - yield (recipe, downloads, descendants, skiplisted, failures, prs) + yield (recipe, downloads, descendants, skiplisted, categories, failures, prs) - data = pd.DataFrame(get_data(), columns=["recipe", "downloads", "depending", "skiplisted", "build failures", "pull requests"]) + data = pd.DataFrame(get_data(), columns=["recipe", "downloads", "depending", "skiplisted", "category", "build failures", "pull requests"]) data.sort_values(by=["depending", "downloads"], ascending=False, inplace=True) return data