Add a ramble workspace logs command

lib/ramble/ramble/application.py

@@ -106,6 +106,7 @@ class ApplicationBase(metaclass=ApplicationMeta):
         "pushdeployment",
         "pushtocache",
         "execute",
+        "logs",
     ]
     _language_classes = [ApplicationMeta, SharedMeta]
 

lib/ramble/ramble/cmd/workspace.py

@@ -51,6 +51,7 @@
     "info",
     "edit",
     "mirror",
+    "experiment-logs",
     ["list", "ls"],
     ["remove", "rm"],
     "generate-config",
@@ -1293,6 +1294,56 @@ def workspace_generate_config(args):
     workspace_manage_experiments(args)
 
 
+def workspace_experiment_logs_setup_parser(subparser):
+    """print log information for workspace"""
+    default_filters = subparser.add_mutually_exclusive_group()
+    default_filters.add_argument(
+        "--limit-one", action="store_true", help="only print the first log information block"
+    )
+
+    default_filters.add_argument(
+        "--first-failed",
+        action="store_true",
+        help="only print the information for the first failed experiment. "
+        + "Requires `ramble workspace analyze` to have been run previously",
+    )
+
+    default_filters.add_argument(
+        "--failed", action="store_true", help="print only failed experiment logs"
+    )
+
+    arguments.add_common_arguments(
+        subparser,
+        ["where", "exclude_where", "filter_tags"],
+    )
+
+
+def workspace_experiment_logs(args):
+    """Print log information for workspace"""
+
+    current_pipeline = ramble.pipeline.pipelines.logs
+    ws = ramble.cmd.require_active_workspace(cmd_name="workspace concretize")
+
+    first_only = args.limit_one or args.first_failed
+    where_filter = args.where.copy() if args.where else []
+    exclude_filter = args.exclude_where.copy() if args.exclude_where else []
+    only_failed = args.first_failed or args.failed
+
+    if only_failed:
+        exclude_filter.append(["'{experiment_status}' == 'SUCCESS'"])
+
+    filters = ramble.filters.Filters(
+        include_where_filters=where_filter,
+        exclude_where_filters=exclude_filter,
+        tags=args.filter_tags,
+    )
+
+    pipeline_cls = ramble.pipeline.pipeline_class(current_pipeline)
+    pipeline = pipeline_cls(ws, filters, first_only=first_only)
+    with ws.write_transaction():
+        workspace_run_pipeline(args, pipeline)
+
+
 #: Dictionary mapping subcommand names and aliases to functions
 subcommand_functions = {}
 

lib/ramble/ramble/modifier.py

@@ -27,7 +27,7 @@ class ModifierBase(metaclass=ModifierMeta):
     _builtin_name = NS_SEPARATOR.join(("modifier_builtin", "{obj_name}", "{name}"))
     _mod_prefix_builtin = f"modifier_builtin{NS_SEPARATOR}"
     _language_classes = [ModifierMeta, SharedMeta]
-    _pipelines = ["analyze", "archive", "mirror", "setup", "pushtocache", "execute"]
+    _pipelines = ["analyze", "archive", "mirror", "setup", "pushtocache", "execute", "logs"]
 
     modifier_class = "ModifierBase"
 

lib/ramble/ramble/package_manager.py

@@ -36,6 +36,7 @@ class PackageManagerBase(metaclass=PackageManagerMeta):
         "pushdeployment",
         "pushtocache",
         "execute",
+        "logs",
     ]
 
     _spec_groups = [

lib/ramble/ramble/pipeline.py

@@ -12,6 +12,7 @@
 import shutil
 import py.path
 import shlex
+import glob
 
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
@@ -586,6 +587,76 @@ def _execute(self):
             executor(*exec_args)
 
 
+class LogsPipeline(Pipeline):
+    """class for the `logs` pipeline"""
+
+    name = "logs"
+
+    def __init__(
+        self,
+        workspace,
+        filters,
+        first_only=False,
+        suppress_per_experiment_prints=True,
+        suppress_run_header=True,
+    ):
+        super().__init__(workspace, filters)
+        self.action_string = "Getting log information for"
+        self.require_inventory = False
+        self.first_only = first_only
+        self.suppress_per_experiment_prints = suppress_per_experiment_prints
+        self.suppress_run_header = suppress_run_header
+
+    def _execute(self):
+        def print_archive_files(app_inst, pattern_title, patterns):
+            print_header = True
+            if len(patterns) > 0:
+                for pattern in patterns:
+                    exp_pattern = app_inst.expander.expand_var(pattern)
+                    for file in glob.glob(exp_pattern):
+                        # Only print the header if a file matched the glob
+                        if print_header:
+                            logger.all_msg(f"    Archive files from {pattern_title}:")
+                            print_header = False
+                        logger.all_msg(f"    - {file}")
+
+        super()._execute()
+
+        if not self.suppress_run_header:
+            logger.all_msg("Finding log information...")
+
+        for exp, app_inst, idx in self._experiment_set.filtered_experiments(self.filters):
+            if app_inst.is_template:
+                continue
+            if app_inst.repeats.is_repeat_base:
+                continue
+
+            log_file = app_inst.expander.expand_var_name("log_file")
+            logger.all_msg(f"Experiment: {exp}")
+            logger.all_msg(f"    Experiment log file: {log_file}")
+
+            analysis_logs, _, _ = app_inst._analysis_dicts(self.workspace.success_list)
+
+            logger.all_msg("    Auxiliary experiment logs:")
+            for log in analysis_logs:
+                logger.all_msg(f"    - {log}")
+
+            print_archive_files(app_inst, "application", app_inst.archive_patterns.keys())
+            if app_inst.package_manager:
+                pm_name = app_inst.package_manager.name
+                print_archive_files(
+                    app_inst,
+                    f"package manager {pm_name}",
+                    app_inst.package_manager.archive_patterns.keys(),
+                )
+
+            for mod in app_inst._modifier_instances:
+                print_archive_files(app_inst, f"modifier {mod.name}", mod.archive_patterns.keys())
+
+            if self.first_only:
+                break
+
+
 class PushDeploymentPipeline(Pipeline):
     """class for the `prepare-deployment` pipeline"""
 
@@ -711,6 +782,7 @@ def _upload_file(src_file, dest_file):
         PushToCachePipeline.name,
         ExecutePipeline.name,
         PushDeploymentPipeline.name,
+        LogsPipeline.name,
     ],
 )
 
@@ -722,6 +794,7 @@ def _upload_file(src_file, dest_file):
     pipelines.pushtocache: PushToCachePipeline,
     pipelines.execute: ExecutePipeline,
     pipelines.pushdeployment: PushDeploymentPipeline,
+    pipelines.logs: LogsPipeline,
 }
 
 

share/ramble/ramble-completion.bash

@@ -638,7 +638,7 @@ _ramble_workspace() {
     then
         RAMBLE_COMPREPLY="-h --help"
     else
-        RAMBLE_COMPREPLY="activate archive deactivate create concretize setup analyze push-to-cache info edit mirror list ls remove rm generate-config manage"
+        RAMBLE_COMPREPLY="activate archive deactivate create concretize setup analyze push-to-cache info edit mirror experiment-logs list ls remove rm generate-config manage"
     fi
 }
 
@@ -696,6 +696,10 @@ _ramble_workspace_mirror() {
     RAMBLE_COMPREPLY="-h --help -d --dry-run --phases --include-phase-dependencies --where --exclude-where"
 }
 
+_ramble_workspace_experiment_logs() {
+    RAMBLE_COMPREPLY="-h --help --limit-one --first-failed --failed --where --exclude-where --filter-tags"
+}
+
 _ramble_workspace_list() {
     RAMBLE_COMPREPLY="-h --help"
 }