test: add Actions annotation output

It's possible to annotate failures in Actions by printing
"::error file={},line={},col={}::{message}". This methos is preferrable
over using a problem matcher because problem matchers only allow
single-line messages, whereas ::error allows multi-line messages.

PR-URL: #34590
Reviewed-By: Rich Trott <[email protected]>
Reviewed-By: Richard Lau <[email protected]>

.github/workflows/test-asan.yml

@@ -36,4 +36,4 @@ jobs:
       - name: Build
         run: make build-ci -j2 V=1
       - name: Test
-        run: make run-ci -j2 V=1 TEST_CI_ARGS="-p dots"
+        run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions"

.github/workflows/test-linux.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Build
         run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn"
       - name: Test
-        run: make run-ci -j2 V=1 TEST_CI_ARGS="-p dots"
+        run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions"
 
   test-linux-with-quic:
     runs-on: ubuntu-latest
@@ -42,4 +42,4 @@ jobs:
       - name: Build
         run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn --experimental-quic"
       - name: Test
-        run: make run-ci -j2 V=1 TEST_CI_ARGS="-p dots"
+        run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions"

.github/workflows/test-macos.yml

@@ -27,4 +27,4 @@ jobs:
       - name: Build
         run: make build-ci -j8 V=1 CONFIG_FLAGS="--error-on-warn --experimental-quic"
       - name: Test
-        run: make run-ci -j8 V=1 TEST_CI_ARGS="-p dots"
+        run: make run-ci -j8 V=1 TEST_CI_ARGS="-p actions"

tools/test.py

@@ -116,6 +116,25 @@ def __init__(self, cases, flaky_tests_mode):
     self.lock = threading.Lock()
     self.shutdown_event = threading.Event()
 
+  def GetFailureOutput(self, failure):
+    output = []
+    if failure.output.stderr:
+      output += ["--- stderr ---" ]
+      output += [failure.output.stderr.strip()]
+    if failure.output.stdout:
+      output += ["--- stdout ---"]
+      output += [failure.output.stdout.strip()]
+    output += ["Command: %s" % EscapeCommand(failure.command)]
+    if failure.HasCrashed():
+      output += ["--- %s ---" % PrintCrashed(failure.output.exit_code)]
+    if failure.HasTimedOut():
+      output += ["--- TIMEOUT ---"]
+    output = "\n".join(output)
+    return output
+
+  def PrintFailureOutput(self, failure):
+    print(self.GetFailureOutput(failure))
+
   def PrintFailureHeader(self, test):
     if test.IsNegative():
       negative_marker = '[negative] '
@@ -224,17 +243,7 @@ def Done(self):
     print()
     for failed in self.failed:
       self.PrintFailureHeader(failed.test)
-      if failed.output.stderr:
-        print("--- stderr ---")
-        print(failed.output.stderr.strip())
-      if failed.output.stdout:
-        print("--- stdout ---")
-        print(failed.output.stdout.strip())
-      print("Command: %s" % EscapeCommand(failed.command))
-      if failed.HasCrashed():
-        print("--- %s ---" % PrintCrashed(failed.output.exit_code))
-      if failed.HasTimedOut():
-        print("--- TIMEOUT ---")
+      self.PrintFailureOutput(failed)
     if len(self.failed) == 0:
       print("===")
       print("=== All tests succeeded")
@@ -288,6 +297,21 @@ def HasRun(self, output):
       sys.stdout.write('.')
       sys.stdout.flush()
 
+class ActionsAnnotationProgressIndicator(DotsProgressIndicator):
+  def GetAnnotationInfo(self, test, output):
+    traceback = output.stdout + output.stderr
+    find_full_path = re.search(r' +at .*\(.*%s:([0-9]+):([0-9]+)' % test.file, traceback)
+    col = line = 0
+    if find_full_path:
+        line, col = map(int, find_full_path.groups())
+    root_path = abspath(join(dirname(__file__), '../')) + os.sep
+    filename = test.file.replace(root_path, "")
+    return filename, line, col
+
+  def PrintFailureOutput(self, failure):
+    output = self.GetFailureOutput(failure)
+    filename, line, column = self.GetAnnotationInfo(failure.test, failure.output)
+    print("::error file=%s,line=%d,col=%d::%s" % (filename, line, column, output.replace('\n', '%0A')))
 
 class TapProgressIndicator(SimpleProgressIndicator):
 
@@ -496,6 +520,7 @@ def ClearLine(self, last_line_length):
 PROGRESS_INDICATORS = {
   'verbose': VerboseProgressIndicator,
   'dots': DotsProgressIndicator,
+  'actions': ActionsAnnotationProgressIndicator,
   'color': ColorProgressIndicator,
   'tap': TapProgressIndicator,
   'mono': MonochromeProgressIndicator,
@@ -1299,7 +1324,7 @@ def BuildOptions():
   result.add_option('--logfile', dest='logfile',
       help='write test output to file. NOTE: this only applies the tap progress indicator')
   result.add_option("-p", "--progress",
-      help="The style of progress indicator (verbose, dots, color, mono, tap)",
+      help="The style of progress indicator (%s)" % ", ".join(PROGRESS_INDICATORS.keys()),
       choices=list(PROGRESS_INDICATORS.keys()), default="mono")
   result.add_option("--report", help="Print a summary of the tests to be run",
       default=False, action="store_true")