Skip to content

Commit

Permalink
Add support for flaky tests and multiple workflows
Browse files Browse the repository at this point in the history
  • Loading branch information
@gsmet
gsmet committed Jan 30, 2024
1 parent c01e632 commit ef74bbc
Show file tree
Hide file tree
Showing 14 changed files with 412 additions and 76 deletions.
43 changes: 20 additions & 23 deletions ...ithub-actions/src/main/java/io/quarkus/bot/buildreporter/githubactions/BuildReporter.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
package io.quarkus.bot.buildreporter.githubactions;

import static io.quarkus.bot.buildreporter.githubactions.WorkflowUtils.getActiveStatusCommentMarker;

import java.io.IOException;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;

Expand All @@ -17,7 +18,7 @@
import org.kohsuke.github.GHCheckRunBuilder;
import org.kohsuke.github.GHCheckRunBuilder.Annotation;
import org.kohsuke.github.GHCheckRunBuilder.Output;
import org.kohsuke.github.GHWorkflowJob;
import org.kohsuke.github.GHWorkflow;
import org.kohsuke.github.GHWorkflowRun;

import io.quarkus.bot.buildreporter.githubactions.report.WorkflowReport;
Expand All @@ -32,9 +33,6 @@ public class BuildReporter {

private static final int GITHUB_FIELD_LENGTH_HARD_LIMIT = 65000;

@Inject
WorkflowRunAnalyzer workflowRunAnalyzer;

@Inject
WorkflowReportFormatter workflowReportFormatter;

Expand All @@ -44,47 +42,44 @@ public class BuildReporter {
@Inject
StackTraceShortener stackTraceShortener;

public Optional<String> generateReportComment(GHWorkflowRun workflowRun,
public Optional<String> generateReportComment(GHWorkflow workflow,
GHWorkflowRun workflowRun,
BuildReporterConfig buildReporterConfig,
WorkflowContext workflowContext,
Map<String, Optional<BuildReports>> buildReportsMap,
boolean artifactsAvailable) throws IOException {
List<GHWorkflowJob> jobs = workflowRun.listJobs().toList()
.stream()
.sorted(buildReporterConfig.getJobNameComparator())
.collect(Collectors.toList());

Optional<WorkflowReport> workflowReportOptional = workflowRunAnalyzer.getReport(workflowRun, workflowContext, jobs,
buildReportsMap);
if (workflowReportOptional.isEmpty()) {
return Optional.empty();
}

WorkflowReport workflowReport = workflowReportOptional.get();
WorkflowReport workflowReport,
boolean artifactsAvailable,
boolean indicateSuccess,
boolean hasOtherPendingCheckRuns) throws IOException {

Optional<GHCheckRun> checkRunOptional = createCheckRun(workflowRun, buildReporterConfig, workflowContext,
artifactsAvailable, workflowReport);

String workflowRunIdMarker = String.format(WorkflowConstants.WORKFLOW_RUN_ID_MARKER, workflowRun.getId());
String statusCommentMarker = workflow == null ? WorkflowConstants.MESSAGE_ID_ACTIVE
: getActiveStatusCommentMarker(workflow.getName());

String reportComment = workflowReportFormatter.getReportComment(workflowReport,
artifactsAvailable,
checkRunOptional.orElse(null),
WorkflowConstants.MESSAGE_ID_ACTIVE,
statusCommentMarker,
workflowRunIdMarker,
WorkflowConstants.BUILD_SCANS_CHECK_RUN_MARKER,
buildReporterConfig.isDevelocityEnabled(),
indicateSuccess,
hasOtherPendingCheckRuns,
true,
true,
workflowReportJobIncludeStrategy);
if (reportComment.length() > GITHUB_FIELD_LENGTH_HARD_LIMIT) {
reportComment = workflowReportFormatter.getReportComment(workflowReport,
artifactsAvailable,
checkRunOptional.orElse(null),
WorkflowConstants.MESSAGE_ID_ACTIVE,
statusCommentMarker,
workflowRunIdMarker,
WorkflowConstants.BUILD_SCANS_CHECK_RUN_MARKER,
buildReporterConfig.isDevelocityEnabled(),
indicateSuccess,
hasOtherPendingCheckRuns,
false,
true,
workflowReportJobIncludeStrategy);
Expand All @@ -93,10 +88,12 @@ public Optional<String> generateReportComment(GHWorkflowRun workflowRun,
reportComment = workflowReportFormatter.getReportComment(workflowReport,
artifactsAvailable,
checkRunOptional.orElse(null),
WorkflowConstants.MESSAGE_ID_ACTIVE,
statusCommentMarker,
workflowRunIdMarker,
WorkflowConstants.BUILD_SCANS_CHECK_RUN_MARKER,
buildReporterConfig.isDevelocityEnabled(),
indicateSuccess,
hasOtherPendingCheckRuns,
false,
false,
workflowReportJobIncludeStrategy);
Expand Down
35 changes: 32 additions & 3 deletions .../src/main/java/io/quarkus/bot/buildreporter/githubactions/BuildReporterActionHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,31 +4,60 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import jakarta.inject.Inject;
import jakarta.inject.Singleton;

import org.jboss.logging.Logger;
import org.kohsuke.github.GHWorkflow;
import org.kohsuke.github.GHWorkflowJob;
import org.kohsuke.github.GHWorkflowRun;
import org.kohsuke.github.GHWorkflowRun.Conclusion;

import io.quarkus.bot.buildreporter.githubactions.report.WorkflowReport;

@Singleton
public class BuildReporterActionHandler {

private static final Logger LOG = Logger.getLogger(BuildReporterActionHandler.class);

@Inject
WorkflowRunAnalyzer workflowRunAnalyzer;

@Inject
BuildReporter buildReporter;

public Optional<String> generateReport(GHWorkflowRun workflowRun, Path buildReportsArtifactsPath,
BuildReporterConfig buildReporterConfig) throws IOException {
return generateReport(null, workflowRun, buildReportsArtifactsPath, buildReporterConfig);
}

public Optional<String> generateReport(GHWorkflow workflow, GHWorkflowRun workflowRun, Path buildReportsArtifactsPath,
BuildReporterConfig buildReporterConfig) throws IOException {
Map<String, Optional<BuildReports>> buildReportsMap = prepareBuildReportMap(buildReportsArtifactsPath);

return buildReporter.generateReportComment(workflowRun, buildReporterConfig,
new WorkflowContext(workflowRun),
buildReportsMap, true);
WorkflowContext workflowContext = new WorkflowContext(workflowRun);
List<GHWorkflowJob> jobs = workflowRun.listJobs().toList()
.stream()
.filter(j -> j.getConclusion() != Conclusion.UNKNOWN && j.getConclusion() != Conclusion.ACTION_REQUIRED)
.sorted(buildReporterConfig.getJobNameComparator())
.collect(Collectors.toList());

Optional<WorkflowReport> workflowReportOptional = workflowRunAnalyzer.getReport(workflow, workflowRun, workflowContext,
jobs,
buildReportsMap);
if (workflowReportOptional.isEmpty()) {
return Optional.empty();
}

return buildReporter.generateReportComment(workflow, workflowRun, buildReporterConfig,
workflowContext,
workflowReportOptional.get(), true, false, false);
}

private Map<String, Optional<BuildReports>> prepareBuildReportMap(Path buildReportsArtifactsPath) {
Expand Down
101 changes: 70 additions & 31 deletions ...s/src/main/java/io/quarkus/bot/buildreporter/githubactions/BuildReporterEventHandler.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
package io.quarkus.bot.buildreporter.githubactions;

import static io.quarkus.bot.buildreporter.githubactions.WorkflowUtils.getActiveStatusCommentMarker;
import static io.quarkus.bot.buildreporter.githubactions.WorkflowUtils.getHiddenStatusCommentMarker;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
Expand Down Expand Up @@ -32,22 +35,24 @@
import org.kohsuke.github.GHIssueState;
import org.kohsuke.github.GHPullRequest;
import org.kohsuke.github.GHWorkflow;
import org.kohsuke.github.GHWorkflowJob;
import org.kohsuke.github.GHWorkflowRun;
import org.kohsuke.github.GHWorkflowRun.Conclusion;
import org.kohsuke.github.GitHub;

import io.quarkiverse.githubapp.event.Actions;
import io.quarkus.bot.buildreporter.githubactions.report.WorkflowReport;
import io.smallrye.graphql.client.dynamic.api.DynamicGraphQLClient;

@Singleton
public class BuildReporterEventHandler {

private static final Logger LOG = Logger.getLogger(BuildReporterEventHandler.class);

public static final String PULL_REQUEST_COMPLETED_SUCCESSFULLY = """
:heavy_check_mark: The latest workflow run for the pull request has completed successfully.
private static final String LABEL_FLAKY_TEST = "triage/flaky-test";

It should be safe to merge provided you have a look at the other checks in the summary.""";
@Inject
WorkflowRunAnalyzer workflowRunAnalyzer;

@Inject
BuildReporter buildReporter;
Expand All @@ -72,18 +77,19 @@ public void handle(GHEventPayload.WorkflowRun workflowRunPayload,

switch (workflowRunPayload.getAction()) {
case Actions.COMPLETED:
handleCompleted(workflowRun, buildReporterConfig, gitHub, gitHubGraphQLClient);
handleCompleted(workflow, workflowRun, buildReporterConfig, gitHub, gitHubGraphQLClient);
break;
case Actions.REQUESTED:
handleRequested(workflowRun, buildReporterConfig, gitHub, gitHubGraphQLClient);
handleRequested(workflow, workflowRun, buildReporterConfig, gitHub, gitHubGraphQLClient);
break;
default:
// we don't do anything for other actions
break;
}
}

private void handleCompleted(GHWorkflowRun workflowRun,
private void handleCompleted(GHWorkflow workflow,
GHWorkflowRun workflowRun,
BuildReporterConfig buildReporterConfig,
GitHub gitHub,
DynamicGraphQLClient gitHubGraphQLClient) throws IOException {
Expand Down Expand Up @@ -125,31 +131,37 @@ private void handleCompleted(GHWorkflowRun workflowRun,
WorkflowContext workflowContext = new WorkflowContext(pullRequest);

hideOutdatedWorkflowRunResults(buildReporterConfig, workflowContext, pullRequest,
gitHubGraphQLClient);

if (conclusion != Conclusion.FAILURE) {
if (!pullRequest.isDraft() && conclusion == Conclusion.SUCCESS
&& !hasPendingCheckRuns(pullRequest)) {
String successComment = PULL_REQUEST_COMPLETED_SUCCESSFULLY
+ "\n\n" + WorkflowConstants.MESSAGE_ID_ACTIVE
+ "\n" + String.format(WorkflowConstants.WORKFLOW_RUN_ID_MARKER, workflowRun.getId());

if (buildReporterConfig.isDevelocityEnabled()) {
successComment += "\n" + WorkflowConstants.BUILD_SCANS_CHECK_RUN_MARKER;
}
workflow.getName(), gitHubGraphQLClient);

pullRequest.comment(successComment);
}
if (conclusion == Conclusion.SUCCESS && pullRequest.isDraft()) {
return;
}

Map<String, Optional<BuildReports>> buildReportsMap = downloadBuildReports(workflowContext,
allBuildReportsDirectory,
artifacts, artifactsAvailable);
List<GHWorkflowJob> jobs = workflowRun.listJobs().toList()
.stream()
.sorted(buildReporterConfig.getJobNameComparator())
.collect(Collectors.toList());

Optional<WorkflowReport> workflowReportOptional = workflowRunAnalyzer.getReport(workflow, workflowRun,
workflowContext,
jobs,
buildReportsMap);
if (workflowReportOptional.isEmpty()) {
return;
}

WorkflowReport workflowReport = workflowReportOptional.get();

Optional<String> reportCommentOptional = buildReporter.generateReportComment(workflowRun, buildReporterConfig,
Optional<String> reportCommentOptional = buildReporter.generateReportComment(workflow, workflowRun,
buildReporterConfig,
workflowContext,
buildReportsMap, artifactsAvailable);
workflowReport,
artifactsAvailable,
true,
hasOtherPendingCheckRuns(pullRequest));

if (reportCommentOptional.isEmpty()) {
return;
Expand All @@ -162,6 +174,14 @@ private void handleCompleted(GHWorkflowRun workflowRun,
} else {
LOG.info("Pull request #" + pullRequest.getNumber() + " - Add test failures:\n" + reportComment);
}

if (workflowReport.hasFlakyTests()) {
if (!buildReporterConfig.isDryRun()) {
pullRequest.addLabels(LABEL_FLAKY_TEST);
} else {
LOG.info("Pull request #" + pullRequest.getNumber() + " - Add label " + LABEL_FLAKY_TEST);
}
}
} else {
Optional<GHIssue> reportIssueOptional = getAssociatedReportIssue(gitHub, workflowRun, artifacts);

Expand All @@ -173,7 +193,7 @@ private void handleCompleted(GHWorkflowRun workflowRun,
WorkflowContext workflowContext = new WorkflowContext(reportIssue);

hideOutdatedWorkflowRunResults(buildReporterConfig, workflowContext, reportIssue,
gitHubGraphQLClient);
workflow.getName(), gitHubGraphQLClient);

if (conclusion == Conclusion.SUCCESS
&& reportIssue.getState() == GHIssueState.OPEN) {
Expand Down Expand Up @@ -204,9 +224,25 @@ private void handleCompleted(GHWorkflowRun workflowRun,
allBuildReportsDirectory,
artifacts, artifactsAvailable);

Optional<String> reportCommentOptional = buildReporter.generateReportComment(workflowRun, buildReporterConfig,
List<GHWorkflowJob> jobs = workflowRun.listJobs().toList()
.stream()
.sorted(buildReporterConfig.getJobNameComparator())
.collect(Collectors.toList());

Optional<WorkflowReport> workflowReportOptional = workflowRunAnalyzer.getReport(workflow, workflowRun,
workflowContext,
jobs,
buildReportsMap);
if (workflowReportOptional.isEmpty()) {
return;
}

WorkflowReport workflowReport = workflowReportOptional.get();

Optional<String> reportCommentOptional = buildReporter.generateReportComment(workflow, workflowRun,
buildReporterConfig,
workflowContext,
buildReportsMap, artifactsAvailable);
workflowReport, artifactsAvailable, false, false);

if (reportCommentOptional.isEmpty()) {
// not able to generate a proper report but let's post a default comment anyway
Expand Down Expand Up @@ -332,7 +368,7 @@ private Map<String, Optional<BuildReports>> downloadBuildReports(WorkflowContext
return buildReportsMap;
}

private void handleRequested(GHWorkflowRun workflowRun,
private void handleRequested(GHWorkflow workflow, GHWorkflowRun workflowRun,
BuildReporterConfig buildReporterConfig,
GitHub gitHub, DynamicGraphQLClient gitHubGraphQLClient) throws IOException {

Expand All @@ -345,24 +381,27 @@ private void handleRequested(GHWorkflowRun workflowRun,
}

hideOutdatedWorkflowRunResults(buildReporterConfig, new WorkflowContext(pullRequests.get(0)), pullRequests.get(0),
gitHubGraphQLClient);
workflow.getName(), gitHubGraphQLClient);
}

private static void hideOutdatedWorkflowRunResults(BuildReporterConfig buildReporterConfig,
WorkflowContext workflowContext, GHIssue issue,
String workflowName,
DynamicGraphQLClient gitHubGraphQLClient)
throws IOException {
List<GHIssueComment> comments = issue.getComments();

for (GHIssueComment comment : comments) {
if (!comment.getBody().contains(WorkflowConstants.MESSAGE_ID_ACTIVE)) {
if (!comment.getBody().contains(WorkflowConstants.MESSAGE_ID_ACTIVE) &&
!comment.getBody().contains(getActiveStatusCommentMarker(workflowName))) {
continue;
}

StringBuilder updatedComment = new StringBuilder();
updatedComment.append(WorkflowConstants.HIDE_MESSAGE_PREFIX);
updatedComment.append(comment.getBody().replace(WorkflowConstants.MESSAGE_ID_ACTIVE,
WorkflowConstants.MESSAGE_ID_HIDDEN));
updatedComment.append(comment.getBody().replace(getActiveStatusCommentMarker(workflowName),
getHiddenStatusCommentMarker(workflowName))
.replace(WorkflowConstants.MESSAGE_ID_ACTIVE, getHiddenStatusCommentMarker(workflowName)));

if (!buildReporterConfig.isDryRun()) {
try {
Expand Down Expand Up @@ -419,7 +458,7 @@ public List<GHArtifact> getArtifacts() {
}
}

private static boolean hasPendingCheckRuns(GHPullRequest pullRequest) {
private static boolean hasOtherPendingCheckRuns(GHPullRequest pullRequest) {
try {
return pullRequest.getRepository().getCheckRuns(pullRequest.getHead().getSha()).toList().stream()
.anyMatch(cr -> cr.getStatus() == Status.QUEUED || cr.getStatus() == Status.IN_PROGRESS);
Expand Down
3 changes: 2 additions & 1 deletion ...b-actions/src/main/java/io/quarkus/bot/buildreporter/githubactions/WorkflowConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ public class WorkflowConstants {
public static final String BUILD_SUMMARY_CHECK_RUN_PREFIX = "Build summary for ";

public static final String MESSAGE_ID_ACTIVE = "<!-- Quarkus-GitHub-Bot/msg-id:workflow-run-status-active -->";
public static final String MESSAGE_ID_HIDDEN = "<!-- Quarkus-GitHub-Bot/msg-id:workflow-run-status-hidden -->";
public static final String MESSAGE_ID_ACTIVE_FOR_WORKFLOW = "<!-- Quarkus-GitHub-Bot/msg-id:workflow-run-status-active:%1$s -->";
public static final String MESSAGE_ID_HIDDEN_FOR_WORKFLOW = "<!-- Quarkus-GitHub-Bot/msg-id:workflow-run-status-hidden:%1$s -->";
public static final String WORKFLOW_RUN_ID_MARKER = "<!-- Quarkus-GitHub-Bot/workflow-run-id:%1$s -->";
public static final String BUILD_SCANS_CHECK_RUN_MARKER = "<!-- Quarkus-GitHub-Bot/build-scans-check-run -->";
public static final String HIDE_MESSAGE_PREFIX = """
Expand Down
Loading

0 comments on commit ef74bbc

Please sign in to comment.