Add durations summary for better auditing (#198)

* Produce a durations summary

* console(record=True)

combine-durations/action.py

@@ -2,15 +2,21 @@
 
 from __future__ import annotations
 
+import sys
 import json
 from argparse import ArgumentParser, ArgumentTypeError, Namespace
 from functools import partial
 from pathlib import Path
+import os
 from statistics import fmean
+from typing import NamedTuple
 
 from rich.console import Console
+from rich import box
+from rich.table import Table
 
-print = Console(color_system="standard", soft_wrap=True).print
+console = Console(color_system="standard", soft_wrap=True, record=True)
+print = console.print
 
 
 def validate_dir(value: str, writable: bool = False) -> Path:
@@ -39,39 +45,111 @@ def parse_args() -> Namespace:
     )
     return parser.parse_args()
 
+class DurationStats(NamedTuple):
+    number_of_tests: int
+    total_run_time: float
+    average_run_time: float
+
+
+def read_durations(path: Path, stats: dict[str, DurationStats]) -> tuple[str, dict[str, float]]:
+    OS = path.stem
+    data = json.loads(path.read_text())
+
+    # new durations stats
+    stats[OS] = DurationStats(
+        number_of_tests=len(data),
+        total_run_time=sum(data.values()),
+        average_run_time=fmean(data.values()),
+    )
+
+    return OS, data
+
+
+def dump_summary():
+    # dump summary to GitHub Actions summary
+    summary = os.getenv("GITHUB_STEP_SUMMARY")
+    output = os.getenv("GITHUB_OUTPUT")
+    if summary or output:
+        html = console.export_text()
+    if summary:
+        Path(summary).write_text(f"### Durations Audit\n{html}")
+    if output:
+        with Path(output).open("a") as fh:
+            fh.write(
+                # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-an-output-parameter
+                # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#multiline-strings
+                f"summary<<GITHUB_OUTPUT_summary\n"
+                f"<details>\n"
+                f"<summary>Durations Audit</summary>\n"
+                f"\n"
+                f"{html}\n"
+                f"\n"
+                f"</details>\n"
+                f"GITHUB_OUTPUT_summary\n"
+            )
+
 
 def main() -> None:
     args = parse_args()
 
-    # aggregate new durations
     combined: dict[str, dict[str, list[float]]] = {}
+
+    # aggregate new durations
+    new_stats: dict[str, DurationStats] = {}
     for path in args.artifacts_dir.glob("**/*.json"):
-        os_combined = combined.setdefault((OS := path.stem), {})
+        # read new durations
+        OS, new_data = read_durations(path, new_stats)
 
-        new_data = json.loads(path.read_text())
+        # insert new durations
+        os_combined = combined.setdefault(OS, {})
         for key, value in new_data.items():
             os_combined.setdefault(key, []).append(value)
 
     # aggregate old durations
+    old_stats: dict[str, DurationStats] = {}
     for path in args.durations_dir.glob("*.json"):
+        # read old durations
+        OS, old_data = read_durations(path, old_stats)
+
         try:
-            os_combined = combined[(OS := path.stem)]
+            os_combined = combined[OS]
         except KeyError:
             # KeyError: OS not present in new durations
             print(f"⚠️ {OS} not present in new durations, removing")
             path.unlink()
             continue
 
-        old_data = json.loads(path.read_text())
-        if missing := set(old_data) - set(combined[OS]):
-            for name in missing:
-                print(f"⚠️ {OS}::{name} not present in new durations, removing")
+        # warn about tests that are no longer present
+        for name in set(old_data) - set(combined[OS]):
+            print(f"⚠️ {OS}::{name} not present in new durations, removing")
 
         # only copy over keys that are still present in new durations
         for key in set(old_data) & set(combined[OS]):
             os_combined[key].append(old_data[key])
 
-    # drop durations no longer present in new durations and write out averages
+    # display stats
+    table = Table(box=box.MARKDOWN)
+    table.add_column("OS")
+    table.add_column("Number of tests")
+    table.add_column("Total run time")
+    table.add_column("Average run time")
+    for OS in sorted({*new_stats, *old_stats}):
+        ncount, ntotal, naverage = new_stats.get(OS, (0, 0.0, 0.0))
+        ocount, ototal, oaverage = old_stats.get(OS, (0, 0.0, 0.0))
+
+        dcount = ncount - ocount
+        dtotal = ntotal - ototal
+        daverage = naverage - oaverage
+
+        table.add_row(
+            OS,
+            f"{ncount} ({dcount:+}) {'🟢' if dcount >= 0 else '🔴'}",
+            f"{ntotal:.2f} ({dtotal:+.2f}) {'🔴' if dtotal >= 0 else '🟢'}",
+            f"{naverage:.2f} ({daverage:+.2f}) {'🔴' if daverage >= 0 else '🟢'}",
+        )
+    print(table)
+
+    # write out averages
     for OS, os_combined in combined.items():
         (args.durations_dir / f"{OS}.json").write_text(
             json.dumps(
@@ -82,6 +160,9 @@ def main() -> None:
             + "\n"  # include trailing newline
         )
 
+    dump_summary()
+    sys.exit(0)
+
 
 if __name__ == "__main__":
     main()

combine-durations/action.yml

@@ -26,11 +26,15 @@ inputs:
   repository:
     description: The repository to search for recent pytest runs.
     default: ${{ github.repository }}
+outputs:
+  summary:
+    description: Summary of the durations that were combined.
+    value: ${{ steps.combine.outputs.summary }}
 
 runs:
   using: composite
   steps:
-    - name: download recent artifacts
+    - name: Download Recent Artifacts
       shell: bash
       run: >
         gh run list
@@ -59,11 +63,19 @@ runs:
       with:
         python-version: '3.11'
 
-    - name: install dependencies
+    - name: Install Dependencies
       shell: bash
       run: pip install --quiet -r ${{ github.action_path }}/requirements.txt
 
-    - name: combine recent durations from artifacts
+    - name: Pip List
+      shell: bash
+      run: |
+        echo ::group::Pip List
+        pip list
+        echo ::endgroup::
+
+    - name: Combine Recent Durations
+      id: combine
       shell: bash
       run: >
         python ${{ github.action_path }}/action.py