Send Slack notification for Test Failures (#60)

* return test failures in json * add send_slack_notification * remove BUILDKITE_TOKEN * update condition when no failures * update slack env var value * add job id to build link, update slack message * update SLACK_BUILD_WEBHOOK to SLACK_WEBHOOK * update slack app name and icon * add condition to only send notification for trunk merges * add second argument to decide if should send slack notification * update text in message * for testing - update to use testing branch * update messaging and add change to changelog * move changelog entry to correct line * update send_slack_notification to accept failures array and parse it * update to use OptionParser for slack argument * remove unnecessary declaration * update comment to remove mention of returning json * check if on develop and master branch too * remove unnecessary parsing and remove branch condition * update and prettify texts --------- Co-authored-by: jos <[email protected]>
Automattic · May 29, 2023 · 87964dd · 87964dd
1 parent 7e40e7a
commit 87964dd
Show file tree

Hide file tree

Showing 2 changed files with 82 additions and 3 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -39,6 +39,7 @@ _None._
 ### New Features
 
 - When `install_cocoapods` fails because `Podfile.lock` changed in CI, it now prints a diff of the changes [#59]
+- Update `annotate_test_failures` to be able to send Slack Notification when there are failures. [#60]
 
 ### Bug Fixes
 

diff --git a/bin/annotate_test_failures b/bin/annotate_test_failures
@@ -1,16 +1,31 @@
 #!/usr/bin/env ruby
 #
 # Usage:
-#   annotate_test_failures [junit-file-path]
+#   annotate_test_failures [options] [junit-report-file-path]
 #
 require 'rexml/document'
 require 'shellwords'
+require 'net/http'
+require 'json'
+require 'optparse'
 
 ###################
 # Parse arguments
 ###################
-
-junit_path = ARGV.first || File.join('build', 'results', 'report.junit')
+slack_channel = nil
+slack_webhook = ENV['SLACK_WEBHOOK'] # default value inferred from env var if not provided explicitly
+args = OptionParser.new do |opts|
+  opts.banner = <<~HELP
+    Usage: annotate_test_failures [junit-report-file-path] [--slack CHANNEL] [--slack-webhook URL] 
+
+    Annotates the Buildkite build with a summary of failed and flaky tests based on a JUnit report file (defaults to using `build/results/report.junit`).
+    Optionally also posts the same info to a Slack channel.
+  HELP
+  opts.on('--slack CHANNEL_NAME', 'The name of the Slack channel to post the failure summary to') { |v| slack_channel = '#' + v.delete_prefix('#') }
+  opts.on('--slack-webhook URL', 'The Slack Webhook URL to use if --slack is used. Defaults to the value of the `SLACK_WEBHOOK` env var') { |v| slack_webhook = v }
+  opts.on_tail("-h", "--help", "Show this help message") { puts opts; exit }
+end.parse!
+junit_path = args.first || File.join('build', 'results', 'report.junit')
 title = ENV.fetch('BUILDKITE_LABEL', 'Tests')
 
 unless File.exist?(junit_path)
@@ -100,6 +115,63 @@ def update_annotation(title, list, style, state)
   end
 end
 
+# Given a list of failures, parse list and send a slack notification with the test names in the payload
+#
+def send_slack_notification(slack_webhook, slack_channel, title, list)
+    failing_tests = list.map { |item| "`#{item.name}` in `#{item.classname}`" }
+    assertion_failures_count = list.count
+    test_text = (assertion_failures_count == 1) ? "Test" : "Tests"
+
+    slack_message_payload = {
+      "channel": "#{slack_channel}",
+      "username": "#{ENV['BUILDKITE_PIPELINE_NAME']} Tests Failures",
+      "icon_emoji": ":fire:",
+      "blocks": [
+        {
+          "type": "section",
+          "text": {
+            "type": "mrkdwn",
+            "text": ":warning: *#{assertion_failures_count} #{test_text} Failed in #{ENV['BUILDKITE_PIPELINE_NAME']} - #{title}*"
+          },
+          "accessory": {
+            "type": "button",
+            "text": {
+              "type": "plain_text",
+              "text": "Build",
+              "emoji": true
+            },
+            "value": "build",
+            "url": "#{ENV['BUILDKITE_BUILD_URL']}##{ENV['BUILDKITE_JOB_ID']}",
+            "action_id": "button-action"
+          }
+        },
+        {
+          "type": "divider"
+        },
+        {
+          "type": "section",
+          "text": {
+            "type": "mrkdwn",
+            "text": "*Failing #{test_text}:*\n#{failing_tests.join("\n")}"
+          }
+        }
+      ]
+    }
+
+    json_payload = JSON.generate(slack_message_payload)
+
+    # Send message to Slack
+    uri = URI(slack_webhook)
+    response = Net::HTTP.post(uri, json_payload, "Content-Type" => "application/json")
+
+    # Check response status
+    if response.code == "200"
+      puts "✅ Notification Sent!"
+    else
+      puts "❌ Failed to send notification. Response code: #{response.code}"
+    end
+end
+
 ###################
 # Main
 ###################
@@ -121,3 +193,9 @@ end
 
 update_annotation(title, failures, 'error', 'have failed')
 update_annotation(title, flakies, 'warning', 'were flaky')
+
+if slack_channel.nil? || slack_webhook.nil?
+  puts '⏩ No `--slack` channel name and/or `--slack-webhook` URL provided; skipping Slack notification.'
+else
+  send_slack_notification(slack_webhook, slack_channel, title, failures) unless failures.empty?
+end