feat: Enable Live logs and add option to disable (#930)

- Enable live logging except when a new `--disable-live-logs` option is
enabled.
- This will improve troubleshooting of CI builds while having the option
to avoid excessive log messages.

bioconda_utils/build.py

@@ -61,7 +61,8 @@ def build(recipe: str, pkg_paths: List[str] = None,
           mulled_conda_image: str = pkg_test.MULLED_CONDA_IMAGE,
           record_build_failure: bool = False,
           dag: Optional[nx.DiGraph] = None,
-          skiplist_leafs: bool = False) -> BuildResult:
+          skiplist_leafs: bool = False,
+          live_logs: bool = True) -> BuildResult:
     """
     Build a single recipe for a single env
 
@@ -81,6 +82,7 @@ def build(recipe: str, pkg_paths: List[str] = None,
       record_build_failure: If True, record build failures in a file next to the meta.yaml
       dag: optional nx.DiGraph with dependency information
       skiplist_leafs: If True, blacklist leaf packages that fail to build
+      live_logs: If True, enable live logging during the build process
     """
     if record_build_failure and not dag:
         raise ValueError("record_build_failure requires dag to be set")
@@ -137,7 +139,8 @@ def build(recipe: str, pkg_paths: List[str] = None,
             docker_builder.build_recipe(recipe_dir=os.path.abspath(recipe),
                                         build_args=' '.join(args),
                                         env=whitelisted_env,
-                                        noarch=is_noarch)
+                                        noarch=is_noarch,
+                                        live_logs=live_logs)
             # Use presence of expected packages to check for success
             for pkg_path in pkg_paths:
                 if not os.path.exists(pkg_path):
@@ -156,7 +159,7 @@ def build(recipe: str, pkg_paths: List[str] = None,
                     cmd += [config_file.arg, config_file.path]
                 cmd += [os.path.join(recipe, 'meta.yaml')]
                 with utils.Progress():
-                    utils.run(cmd, mask=False)
+                    utils.run(cmd, mask=False, live=live_logs)
 
         logger.info('BUILD SUCCESS %s',
                     ' '.join(os.path.basename(p) for p in pkg_paths))
@@ -295,7 +298,8 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str],
                   keep_old_work: bool = False,
                   mulled_conda_image: str = pkg_test.MULLED_CONDA_IMAGE,
                   record_build_failures: bool = False,
-                  skiplist_leafs: bool = False):
+                  skiplist_leafs: bool = False,
+                  live_logs: bool = True):
     """
     Build one or many bioconda packages.
 
@@ -322,6 +326,8 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str],
       worker_offset: If n_workers is >1, then every worker_offset within a given group of
         sub-DAGs will be processed.
       keep_old_work: Do not remove anything from environment, even after successful build and test.
+      skiplist_leafs: If True, blacklist leaf packages that fail to build
+      live_logs: If True, enable live logging during the build process
     """
     if not recipes:
         logger.info("Nothing to be done.")
@@ -421,7 +427,8 @@ def build_recipes(recipe_folder: str, config_path: str, recipes: List[str],
                     mulled_conda_image=mulled_conda_image,
                     dag=dag,
                     record_build_failure=record_build_failures,
-                    skiplist_leafs=skiplist_leafs)
+                    skiplist_leafs=skiplist_leafs,
+                    live_logs=live_logs)
 
         if not res.success:
             failed.append(recipe)

bioconda_utils/cli.py

@@ -434,6 +434,7 @@ def do_lint(recipe_folder, config, packages="*", cache=None, list_checks=False,
      Dockerfile template.''')
 @arg("--record-build-failures", action="store_true", help="Record build failures in build_failure.yaml next to the recipe.")
 @arg("--skiplist-leafs", action="store_true", help="Skiplist leaf recipes (i.e. ones that are not depended on by any other recipes) that fail to build.")
+@arg('--disable-live-logs', action='store_true', help="Disable live logging during the build process")
 @enable_logging()
 def build(recipe_folder, config, packages="*", git_range=None, testonly=False,
           force=False, docker=None, mulled_test=False, build_script_template=None,
@@ -443,7 +444,8 @@ def build(recipe_folder, config, packages="*", git_range=None, testonly=False,
           mulled_conda_image=pkg_test.MULLED_CONDA_IMAGE,
           docker_base_image=None,
           record_build_failures=False,
-          skiplist_leafs=False):
+          skiplist_leafs=False,
+          disable_live_logs=False):
     cfg = utils.load_config(config)
     setup = cfg.get('setup', None)
     if setup:
@@ -503,7 +505,8 @@ def build(recipe_folder, config, packages="*", git_range=None, testonly=False,
                             keep_old_work=keep_old_work,
                             mulled_conda_image=mulled_conda_image,
                             record_build_failures=record_build_failures,
-                            skiplist_leafs=skiplist_leafs)
+                            skiplist_leafs=skiplist_leafs,
+                            live_logs=(not disable_live_logs))
     exit(0 if success else 1)
 
 

bioconda_utils/docker_utils.py

@@ -402,7 +402,7 @@ def _build_image(self):
             shutil.rmtree(build_dir)
         return p
 
-    def build_recipe(self, recipe_dir, build_args, env, noarch=False):
+    def build_recipe(self, recipe_dir, build_args, env, noarch=False, live_logs=True):
         """
         Build a single recipe.
 
@@ -474,7 +474,7 @@ def build_recipe(self, recipe_dir, build_args, env, noarch=False):
 
         logger.debug('DOCKER: cmd: %s', cmd)
         with utils.Progress():
-            p = utils.run(cmd, mask=False)
+            p = utils.run(cmd, mask=False, live=live_logs)
         return p
 
     def cleanup(self):