From 47eaadcd4f0da856733e3fd3170d3451ec9c4b8d Mon Sep 17 00:00:00 2001 From: "Alicia A. Evans" <108547992+aliciaaevans@users.noreply.github.com> Date: Thu, 2 Nov 2023 05:15:47 -0400 Subject: [PATCH] feat: Enable Live logs and add option to disable (#930) - Enable live logging except when a new `--disable-live-logs` option is enabled. - This will improve troubleshooting of CI builds while having the option to avoid excessive log messages. --- bioconda_utils/build.py | 17 ++++++++++++----- bioconda_utils/cli.py | 7 +++++-- bioconda_utils/docker_utils.py | 4 ++-- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/bioconda_utils/build.py b/bioconda_utils/build.py index 2dd6b2efd2..ea42812c59 100644 --- a/bioconda_utils/build.py +++ b/bioconda_utils/build.py @@ -61,7 +61,8 @@ def build(recipe: str, pkg_paths: List[str] = None, mulled_conda_image: str = pkg_test.MULLED_CONDA_IMAGE, record_build_failure: bool = False, dag: Optional[nx.DiGraph] = None, - skiplist_leafs: bool = False) -> BuildResult: + skiplist_leafs: bool = False, + live_logs: bool = True) -> BuildResult: """ Build a single recipe for a single env @@ -81,6 +82,7 @@ def build(recipe: str, pkg_paths: List[str] = None, record_build_failure: If True, record build failures in a file next to the meta.yaml dag: optional nx.DiGraph with dependency information skiplist_leafs: If True, blacklist leaf packages that fail to build + live_logs: If True, enable live logging during the build process """ if record_build_failure and not dag: raise ValueError("record_build_failure requires dag to be set") @@ -137,7 +139,8 @@ def build(recipe: str, pkg_paths: List[str] = None, docker_builder.build_recipe(recipe_dir=os.path.abspath(recipe), build_args=' '.join(args), env=whitelisted_env, - noarch=is_noarch) + noarch=is_noarch, + live_logs=live_logs) # Use presence of expected packages to check for success for pkg_path in pkg_paths: if not os.path.exists(pkg_path): @@ -156,7 +159,7 @@ def build(recipe: str, pkg_paths: List[str] = None, cmd += [config_file.arg, config_file.path] cmd += [os.path.join(recipe, 'meta.yaml')] with utils.Progress(): - utils.run(cmd, mask=False) + utils.run(cmd, mask=False, live=live_logs) logger.info('BUILD SUCCESS %s', ' '.join(os.path.basename(p) for p in pkg_paths)) @@ -295,7 +298,8 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str], keep_old_work: bool = False, mulled_conda_image: str = pkg_test.MULLED_CONDA_IMAGE, record_build_failures: bool = False, - skiplist_leafs: bool = False): + skiplist_leafs: bool = False, + live_logs: bool = True): """ Build one or many bioconda packages. @@ -322,6 +326,8 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str], worker_offset: If n_workers is >1, then every worker_offset within a given group of sub-DAGs will be processed. keep_old_work: Do not remove anything from environment, even after successful build and test. + skiplist_leafs: If True, blacklist leaf packages that fail to build + live_logs: If True, enable live logging during the build process """ if not recipes: logger.info("Nothing to be done.") @@ -421,7 +427,8 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str], mulled_conda_image=mulled_conda_image, dag=dag, record_build_failure=record_build_failures, - skiplist_leafs=skiplist_leafs) + skiplist_leafs=skiplist_leafs, + live_logs=live_logs) if not res.success: failed.append(recipe) diff --git a/bioconda_utils/cli.py b/bioconda_utils/cli.py index 24e3ab877d..1b89e5f6ad 100644 --- a/bioconda_utils/cli.py +++ b/bioconda_utils/cli.py @@ -434,6 +434,7 @@ def do_lint(recipe_folder, config, packages="*", cache=None, list_checks=False, Dockerfile template.''') @arg("--record-build-failures", action="store_true", help="Record build failures in build_failure.yaml next to the recipe.") @arg("--skiplist-leafs", action="store_true", help="Skiplist leaf recipes (i.e. ones that are not depended on by any other recipes) that fail to build.") +@arg('--disable-live-logs', action='store_true', help="Disable live logging during the build process") @enable_logging() def build(recipe_folder, config, packages="*", git_range=None, testonly=False, force=False, docker=None, mulled_test=False, build_script_template=None, @@ -443,7 +444,8 @@ def build(recipe_folder, config, packages="*", git_range=None, testonly=False, mulled_conda_image=pkg_test.MULLED_CONDA_IMAGE, docker_base_image=None, record_build_failures=False, - skiplist_leafs=False): + skiplist_leafs=False, + disable_live_logs=False): cfg = utils.load_config(config) setup = cfg.get('setup', None) if setup: @@ -503,7 +505,8 @@ def build(recipe_folder, config, packages="*", git_range=None, testonly=False, keep_old_work=keep_old_work, mulled_conda_image=mulled_conda_image, record_build_failures=record_build_failures, - skiplist_leafs=skiplist_leafs) + skiplist_leafs=skiplist_leafs, + live_logs=(not disable_live_logs)) exit(0 if success else 1) diff --git a/bioconda_utils/docker_utils.py b/bioconda_utils/docker_utils.py index e7522a065f..76c851dd33 100644 --- a/bioconda_utils/docker_utils.py +++ b/bioconda_utils/docker_utils.py @@ -402,7 +402,7 @@ def _build_image(self): shutil.rmtree(build_dir) return p - def build_recipe(self, recipe_dir, build_args, env, noarch=False): + def build_recipe(self, recipe_dir, build_args, env, noarch=False, live_logs=True): """ Build a single recipe. @@ -474,7 +474,7 @@ def build_recipe(self, recipe_dir, build_args, env, noarch=False): logger.debug('DOCKER: cmd: %s', cmd) with utils.Progress(): - p = utils.run(cmd, mask=False) + p = utils.run(cmd, mask=False, live=live_logs) return p def cleanup(self):