From 3fd8236ecc212038bfbdac2118aa92e53e4c8a62 Mon Sep 17 00:00:00 2001
From: chaodaiG <45011425+chaodaiG@users.noreply.github.com>
Date: Tue, 5 Mar 2019 10:20:39 -0800
Subject: [PATCH] Flaky test reporting(part #4) Github issues tracking for
flaky tests (#506)
* working on github issues tracking
* Update tools/flaky-test-reporter/github_issue.go
Co-Authored-By: chaodaiG <45011425+chaodaiG@users.noreply.github.com>
* update based on PR comments
---
tools/flaky-test-reporter/config.go | 13 +-
tools/flaky-test-reporter/error.go | 36 ++
tools/flaky-test-reporter/ghutil/ghutil.go | 29 +-
tools/flaky-test-reporter/github_issue.go | 459 +++++++++++++++++++++
tools/flaky-test-reporter/main.go | 7 +-
tools/flaky-test-reporter/result.go | 50 ++-
tools/flaky-test-reporter/run.go | 32 ++
7 files changed, 616 insertions(+), 10 deletions(-)
create mode 100644 tools/flaky-test-reporter/error.go
create mode 100644 tools/flaky-test-reporter/github_issue.go
create mode 100644 tools/flaky-test-reporter/run.go
diff --git a/tools/flaky-test-reporter/config.go b/tools/flaky-test-reporter/config.go
index 3cb3cee003..9e1145af1a 100644
--- a/tools/flaky-test-reporter/config.go
+++ b/tools/flaky-test-reporter/config.go
@@ -24,11 +24,16 @@ import (
const (
// Builds to be analyzed, this is an arbitrary number
- buildsCount = 10
+ buildsCount = 10
// Minimal number of results to be counted as valid results for each testcase, this is an arbitrary number
- requiredCount = 8
- // Don't do anything if found more than 5% tests flaky, this is an arbitrary number
- threshold = 0.05
+ requiredCount = 8
+ // Don't do anything if found more than 1% tests flaky, this is an arbitrary number
+ threshold = 0.01
+
+ org = "knative"
+ // Temporarily creating issues under "test-infra" for better management
+ // TODO(chaodaiG): repo for issue same as the src of the test
+ repoForIssue = "test-infra"
)
// jobConfigs defines which job to be monitored for giving repo
diff --git a/tools/flaky-test-reporter/error.go b/tools/flaky-test-reporter/error.go
new file mode 100644
index 0000000000..48fdf394b1
--- /dev/null
+++ b/tools/flaky-test-reporter/error.go
@@ -0,0 +1,36 @@
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// error.go helps with error handling
+
+package main
+
+import (
+ "fmt"
+ "strings"
+)
+
+// combineErrors combines slice of errors and return a single error
+func combineErrors(errs []error) error {
+ if nil == errs || 0 == len(errs) {
+ return nil
+ }
+ var errStrs []string
+ for _, err := range errs {
+ errStrs = append(errStrs, err.Error())
+ }
+ return fmt.Errorf(strings.Join(errStrs, "\n"))
+}
diff --git a/tools/flaky-test-reporter/ghutil/ghutil.go b/tools/flaky-test-reporter/ghutil/ghutil.go
index fdf28a6471..5b0818ea21 100644
--- a/tools/flaky-test-reporter/ghutil/ghutil.go
+++ b/tools/flaky-test-reporter/ghutil/ghutil.go
@@ -38,7 +38,7 @@ const (
// IssueOpenState is the state of open github issue
IssueOpenState IssueStateEnum = "open"
// IssueCloseState is the state of closed github issue
- IssueCloseState IssueStateEnum = "close"
+ IssueCloseState IssueStateEnum = "closed"
// IssueAllState is the state for all, useful when querying issues
IssueAllState IssueStateEnum = "all"
)
@@ -80,6 +80,31 @@ func GetCurrentUser() (*github.User, error) {
return res, err
}
+// ListRepos lists repos under org
+func ListRepos(org string) ([]string, error) {
+ var res []string
+ options := &github.ListOptions{}
+ genericList, err := depaginate(
+ "listing repos",
+ maxRetryCount,
+ options,
+ func() ([]interface{}, *github.Response, error) {
+ page, resp, err := client.Repositories.List(ctx, org, nil)
+ var interfaceList []interface{}
+ if nil == err {
+ for _, repo := range page {
+ interfaceList = append(interfaceList, repo)
+ }
+ }
+ return interfaceList, resp, err
+ },
+ )
+ for _, repo := range genericList {
+ res = append(res, repo.(*github.Repository).GetName())
+ }
+ return res, err
+}
+
// ListIssuesByRepo lists issues within given repo, filters by labels if provided
func ListIssuesByRepo(org, repo string, labels []string) ([]*github.Issue, error) {
issueListOptions := github.IssueListByRepoOptions{
@@ -92,7 +117,7 @@ func ListIssuesByRepo(org, repo string, labels []string) ([]*github.Issue, error
var res []*github.Issue
options := &github.ListOptions{}
genericList, err := depaginate(
- fmt.Sprintf("listing genericList with label '%v'", labels),
+ fmt.Sprintf("listing issues with label '%v'", labels),
maxRetryCount,
options,
func() ([]interface{}, *github.Response, error) {
diff --git a/tools/flaky-test-reporter/github_issue.go b/tools/flaky-test-reporter/github_issue.go
new file mode 100644
index 0000000000..76bf9dbd8d
--- /dev/null
+++ b/tools/flaky-test-reporter/github_issue.go
@@ -0,0 +1,459 @@
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+ "fmt"
+ "log"
+ "regexp"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/google/go-github/github"
+
+ "github.com/knative/test-infra/shared/junit"
+ "github.com/knative/test-infra/tools/flaky-test-reporter/ghutil"
+)
+
+const (
+ // flakyLabel is the Github issue label used for querying all flaky issues auto-generated.
+ flakyLabel = "auto:flaky"
+ testIdentifierToken = "DONT_MODIFY_TEST_IDENTIFIER"
+ latestStatusToken = "Latest result for this test: "
+ beforeHistoryToken = "------Latest History of Up To 10 runs------"
+ afterHistoryToken = "------End of History------"
+ gubernatorURL = "https://gubernator.knative.dev/build/knative-prow/logs/"
+ daysConsiderOld = 30 // arbitrary number of days for an issue to be considered old
+ maxHistoryEntries = 10 // max count of history runs to show in unicode graph
+
+ passedUnicode = "✔" // Checkmark unicode
+ failedUnicode = "✖" // Cross unicode
+ skippedUnicode = "◻" // Open square unicode
+
+ // collapseTemplate is for creating collapsible format in comment, so that only latest result is expanded.
+ // Github markdown supports collapse, strings between is the title,
+ // contents between is collapsible body, body is hidden unless the title is clicked.
+ collapseTemplate = `
+
+ %s
+
+%s
+
` // The blank line is crucial for collapsible format
+
+ // issueBodyTemplate is a template for issue body
+ issueBodyTemplate = `
+\
+Test name: %s
+
+-------------Text below this line is for issues mapping, Please don't edit-------------
+%s`
+)
+
+var (
+ // testIdentifierPattern is used for formatting test identifier,
+ // expect an argument of test identifier
+ testIdentifierPattern = fmt.Sprintf("[%[1]s]%%s[%[1]s]", testIdentifierToken)
+ // reTestIdentifier is regex matching pattern for capturing testname
+ reTestIdentifier = fmt.Sprintf(`\[%[1]s\](.*?)\[%[1]s\]`, testIdentifierToken)
+
+ // historyPattern is for creating history section in commment,
+ // expect an argument of history Unicode from previous comment
+ historyPattern = fmt.Sprintf("\n%s%%s%s\n%s Passed\t%s Failed\t%s Skipped\n",
+ beforeHistoryToken, afterHistoryToken, passedUnicode, failedUnicode, skippedUnicode)
+ // reHistory is for identifying history from comment
+ reHistory = fmt.Sprintf("(?s)%s(.*?)%s", beforeHistoryToken, afterHistoryToken)
+
+ // latestStatusPattern is for creating latest test result line in comment,
+ // expect an argument of test status as defined in result.go
+ latestStatusPattern = fmt.Sprintf(`%s%%s`, latestStatusToken)
+ // reLastestStatus is for identifying latest test result from comment
+ reLastestStatus = fmt.Sprintf(`%s([a-zA-Z]*)`, latestStatusToken)
+)
+
+// Precompute timeConsiderOld so that the same standard used everywhere
+var timeConsiderOld = time.Now().AddDate(0, 0, -daysConsiderOld)
+
+// currentUser is current authenticated user, used for identifying comments created by bot
+var currentUser *github.User
+
+// flakyIssue is a wrapper of github.Issue, used for storing pre-computed information
+type flakyIssue struct {
+ issue *github.Issue
+ identity *string // identity discovered by matching reTestIdentifier
+ comment *github.IssueComment // The first auto comment, updated for every history
+}
+
+// createCommentForTest summarizes latest status of current test case,
+// and creates text to be added to issue comment
+func createCommentForTest(rd *RepoData, testFullName string) string {
+ ts := rd.TestStats[testFullName]
+ totalCount := len(ts.Passed) + len(ts.Skipped) + len(ts.Failed)
+ lastBuildStartTimeStr := time.Unix(*rd.LastBuildStartTime, 0).String()
+ content := fmt.Sprintf("%s\nLast build start time: %s\nFailed %d times out of %d runs.",
+ fmt.Sprintf(latestStatusPattern, ts.getTestStatus()),
+ lastBuildStartTimeStr, len(ts.Failed), totalCount)
+ if len(ts.Failed) > 0 {
+ content += " Failed runs: "
+ var buildIDContents []string
+ for _, buildID := range ts.Failed {
+ buildIDContents = append(buildIDContents,
+ fmt.Sprintf("[%d](%s%s/%d)", buildID, gubernatorURL, rd.Config.Name, buildID))
+ }
+ content += strings.Join(buildIDContents, ", ")
+ }
+ return content
+}
+
+func createHistoryUnicode(rd *RepoData, comment, testFullName string) string {
+ res := ""
+ currentUnicode := fmt.Sprintf("%s: ", time.Unix(*rd.LastBuildStartTime, 0).String())
+ resultSlice := rd.getResultSliceForTest(testFullName)
+ for i, buildID := range rd.BuildIDs {
+ url := fmt.Sprintf("%s%s/%d", gubernatorURL, rd.Config.Name, buildID)
+ var statusUnicode string
+ switch resultSlice[i] {
+ case junit.Passed:
+ statusUnicode = passedUnicode
+ case junit.Failed:
+ statusUnicode = failedUnicode
+ default:
+ statusUnicode = skippedUnicode
+ }
+ currentUnicode += fmt.Sprintf(" [%s](%s)", statusUnicode, url)
+ }
+
+ oldHistory := regexp.MustCompile(reHistory).FindStringSubmatch(comment)
+ res = fmt.Sprintf("\n%s", currentUnicode)
+ if len(oldHistory) >= 2 {
+ oldHistoryEntries := strings.Split(oldHistory[1], "\n")
+ if len(oldHistoryEntries) >= maxHistoryEntries {
+ oldHistoryEntries = oldHistoryEntries[:(maxHistoryEntries - 1)]
+ }
+ res += strings.Join(oldHistoryEntries, "\n")
+ } else {
+ res += "\n"
+ }
+
+ return fmt.Sprintf(historyPattern, res)
+}
+
+// prependComment hides old comment into a collapsible, and prepend
+// new commment on top
+func prependComment(oldComment, newComment string) string {
+ if "" != oldComment {
+ oldComment = fmt.Sprintf(collapseTemplate, "Click to see older results", oldComment)
+ }
+ return fmt.Sprintf("%s\n\n%s", newComment, oldComment)
+}
+
+// updateIssue adds comments to an existing issue, close an issue if test passed both in previous day and today,
+// reopens the issue if test becomes flaky while issue is closed.
+func updateIssue(fi *flakyIssue, newComment string, ts *TestStat, dryrun *bool) error {
+ issue := fi.issue
+ passedLastTime := false
+ latestStatus := regexp.MustCompile(reLastestStatus).FindStringSubmatch(fi.comment.GetBody())
+ if len(latestStatus) >= 2 {
+ switch latestStatus[1] {
+ case passedStatus:
+ passedLastTime = true
+ case flakyStatus, failedStatus, lackDataStatus:
+
+ default:
+ return fmt.Errorf("invalid test status code found from issue '%s'", *issue.URL)
+ }
+ }
+
+ // Update comment unless test passed and issue closed
+ if !ts.isPassed() || issue.GetState() == string(ghutil.IssueOpenState) {
+ if err := run(
+ "updating comment",
+ func() error {
+ return ghutil.EditComment(org, repoForIssue, *fi.comment.ID, prependComment(*fi.comment.Body, newComment))
+ },
+ dryrun); nil != err {
+ return fmt.Errorf("failed updating comments for issue '%s': '%v'", *issue.URL, err)
+ }
+ }
+
+ if ts.isPassed() { // close open issue if the test passed twice consecutively
+ if issue.GetState() == string(ghutil.IssueOpenState) {
+ if passedLastTime {
+ if err := run(
+ "closing issue",
+ func() error {
+ return ghutil.CloseIssue(org, repoForIssue, *issue.Number)
+ },
+ dryrun); nil != err {
+ return fmt.Errorf("failed closing issue '%s': '%v'", *issue.URL, err)
+ }
+ }
+ }
+ } else if ts.isFlaky() { // reopen closed issue if test found flaky
+ if issue.GetState() == string(ghutil.IssueCloseState) {
+ if err := run(
+ "reopening issue",
+ func() error {
+ return ghutil.ReopenIssue(org, repoForIssue, *issue.Number)
+ },
+ dryrun); nil != err {
+ return fmt.Errorf("failed reopen issue: '%s'", *issue.URL)
+ }
+ }
+ }
+ return nil
+}
+
+// createNewIssue creates an issue, adds flaky label and adds comment.
+func createNewIssue(org, repoForIssue, title, body string, comment string, dryrun *bool) error {
+ var newIssue *github.Issue
+ if err := run(
+ "creating issue",
+ func() error {
+ var err error
+ newIssue, err = ghutil.CreateIssue(org, repoForIssue, title, body)
+ return err
+ },
+ dryrun,
+ ); nil != err {
+ return fmt.Errorf("failed creating issue '%s' in repo '%s'", title, repoForIssue)
+ }
+ if err := run(
+ "adding comment",
+ func() error {
+ return ghutil.CreateComment(org, repoForIssue, *newIssue.Number, comment)
+ },
+ dryrun,
+ ); nil != err {
+ return fmt.Errorf("failed adding comment to issue '%s', '%v'", *newIssue.URL, err)
+ }
+ if err := run(
+ "adding flaky label",
+ func() error {
+ return ghutil.AddLabelsToIssue(org, repoForIssue, *newIssue.Number, []string{flakyLabel})
+ },
+ dryrun,
+ ); nil != err {
+ return fmt.Errorf("failed adding '%s' label to issue '%s', '%v'", flakyLabel, *newIssue.URL, err)
+ }
+ return nil
+}
+
+// findExistingComment identify existing comment by comment author and test identifier,
+// if multiple comments were found return the earliest one.
+func findExistingComment(issue *github.Issue, issueIdentity string) (*github.IssueComment, error) {
+ var targetComment *github.IssueComment
+ comments, err := ghutil.ListComments(org, repoForIssue, *issue.Number)
+ if nil != err {
+ return nil, err
+ }
+ sort.Slice(comments, func(i, j int) bool {
+ return nil != comments[i].CreatedAt && (nil == comments[j].CreatedAt || comments[i].CreatedAt.Before(*comments[j].CreatedAt))
+ })
+
+ for i, comment := range comments {
+ if *comment.User.ID != *currentUser.ID {
+ continue
+ }
+ if testNameFromComment := regexp.MustCompile(reTestIdentifier).FindStringSubmatch(*comment.Body); len(testNameFromComment) >= 2 && issueIdentity == testNameFromComment[1] {
+ targetComment = comments[i]
+ break
+ }
+ }
+ if nil == targetComment {
+ return nil, fmt.Errorf("no comment match")
+ }
+ return targetComment, nil
+}
+
+// getFlakyIssues filters all issues by flakyLabel, and return map {testName: slice of issues}
+// Fail if find any issue with no discoverable identifier(testIdentifierPattern missing),
+// also fail if auto comment not found.
+// In most cases there is only 1 issue for each testName, if multiple issues found open for same test,
+// most likely it's caused by old issues being reopened manually, in this case update both issues.
+func getFlakyIssues() (map[string][]*flakyIssue, error) {
+ issuesMap := make(map[string][]*flakyIssue)
+ reposForIssue, err := ghutil.ListRepos(org)
+ if nil != err {
+ return nil, err
+ }
+ for _, repoForIssue := range reposForIssue {
+ issues, err := ghutil.ListIssuesByRepo(org, repoForIssue, []string{flakyLabel})
+ if nil != err {
+ return nil, err
+ }
+ for _, issue := range issues {
+ if nil != issue.ClosedAt && issue.ClosedAt.Before(timeConsiderOld) {
+ continue // Issue closed long time ago, it might fail with a different reason now.
+ }
+ issueIdentity := regexp.MustCompile(reTestIdentifier).FindStringSubmatch(issue.GetBody())
+ if len(issueIdentity) < 2 { // Malformed issue, all auto flaky issues need to be identifiable.
+ return nil, fmt.Errorf("cannot get test identifier from auto flaky issue '%v'", err)
+ }
+ autoComment, err := findExistingComment(issue, issueIdentity[1])
+ if nil != err {
+ return nil, fmt.Errorf("cannot find auto comment for issue '%s': '%v'", *issue.URL, err)
+ }
+ issuesMap[issueIdentity[1]] = append(issuesMap[issueIdentity[1]], &flakyIssue{
+ issue: issue,
+ identity: &issueIdentity[1],
+ comment: autoComment,
+ })
+ }
+ }
+ // Handle test with multiple issues associated
+ // if all open: update all of them
+ // if all closed: only keep latest one
+ // otherwise(these may have been closed manually): remove closed ones from map
+ for k, v := range issuesMap {
+ hasOpen := false
+ hasClosed := false
+ for _, fi := range v {
+ switch fi.issue.GetState() {
+ case string(ghutil.IssueOpenState):
+ hasOpen = true
+ case string(ghutil.IssueCloseState):
+ hasClosed = true
+ }
+ }
+ if hasOpen && hasClosed {
+ for i, fi := range v {
+ if string(ghutil.IssueCloseState) == fi.issue.GetState() {
+ issuesMap[k] = append(issuesMap[k][:i], issuesMap[k][i+1:]...)
+ }
+ }
+ } else if !hasOpen {
+ sort.Slice(issuesMap[k], func(i, j int) bool {
+ return nil != issuesMap[k][i].issue.CreatedAt &&
+ (nil == issuesMap[k][j].issue.CreatedAt || issuesMap[k][i].issue.CreatedAt.Before(*issuesMap[k][j].issue.CreatedAt))
+ })
+ issuesMap[k] = []*flakyIssue{issuesMap[k][0]}
+ }
+ }
+ return issuesMap, err
+}
+
+// processGithubIssueForRepo reads RepoData and existing issues, and create/close/reopen/comment on issues.
+// The function returns a slice of messages containing performed actions, and a slice of error messages,
+// these can later on be printed as summary at the end of run
+func processGithubIssueForRepo(rd *RepoData, flakyIssuesMap map[string][]*flakyIssue, dryrun *bool) ([]string, error) {
+ var messages []string
+ var errs []error
+
+ // If there are too many failures, create a single issue tracking it.
+ flakyRate, err := getFlakyRate(rd.TestStats)
+ if nil != err {
+ return nil, err
+ }
+ if flakyRate > threshold {
+ log.Printf("flaky rate above '%f', creating a single issue", threshold)
+ identity := fmt.Sprintf("%.2f%% tests failed in repo %s on %s",
+ flakyRate*100, rd.Config.Repo, time.Unix(*rd.LastBuildStartTime, 0).String())
+ if _, ok := flakyIssuesMap[identity]; ok {
+ log.Printf("issue already exist, skip creating")
+ return nil, nil
+ }
+ title := fmt.Sprintf("[flaky] %s", identity)
+ comment := fmt.Sprintf("Bulk issue tracking: %s\n%s", identity, fmt.Sprintf(testIdentifierPattern, identity))
+ message := fmt.Sprintf("Creating issue '%s' in repo '%s'", title, repoForIssue)
+ log.Println(message)
+ return []string{message}, createNewIssue(org, repoForIssue, title,
+ fmt.Sprintf(issueBodyTemplate, identity, fmt.Sprintf(testIdentifierPattern, identity)),
+ comment, dryrun)
+ }
+
+ // Update/Create issues for flaky/used-to-be-flaky tests
+ for testFullName, ts := range rd.TestStats {
+ if !ts.hasEnoughRuns() || (!ts.isFlaky() && !ts.isPassed()) {
+ continue
+ }
+ identity := fmt.Sprintf("'%s' in repo '%s'", testFullName, rd.Config.Repo)
+ comment := createCommentForTest(rd, testFullName)
+ if existIssues, ok := flakyIssuesMap[identity]; ok { // update issue with current result
+ for _, existIssue := range existIssues {
+ if strings.Contains(existIssue.comment.GetBody(), comment) {
+ log.Printf("skip updating issue '%s', as it already contains data for run '%d'\n",
+ *existIssue.issue.URL, *rd.LastBuildStartTime)
+ continue
+ }
+ comment += createHistoryUnicode(rd, existIssue.comment.GetBody(), testFullName)
+ message := fmt.Sprintf("Updating issue '%s' for '%s'", *existIssue.issue.URL, *existIssue.identity)
+ log.Println(message)
+ messages = append(messages, message)
+ if err := updateIssue(existIssue, comment, ts, dryrun); nil != err {
+ log.Println(err)
+ errs = append(errs, err)
+ }
+ }
+ } else if ts.isFlaky() {
+ title := fmt.Sprintf("[flaky] %s", identity)
+ comment += createHistoryUnicode(rd, "", testFullName)
+ comment += "\n" + fmt.Sprintf(testIdentifierPattern, identity)
+ message := fmt.Sprintf("Creating issue '%s' in repo '%s'", title, repoForIssue)
+ log.Println(message)
+ messages = append(messages, message)
+ if err := createNewIssue(org, repoForIssue, title,
+ fmt.Sprintf(issueBodyTemplate, identity, fmt.Sprintf(testIdentifierPattern, identity)),
+ comment, dryrun); nil != err {
+ log.Println(err)
+ errs = append(errs, err)
+ }
+ }
+ }
+ return messages, combineErrors(errs)
+}
+
+// analyze all results, figure out flaky tests and processing existing auto:flaky issues
+func processGithubIssues(repoDataAll []*RepoData, githubToken *string, dryrun *bool) error {
+ messagesMap := make(map[string][]string)
+ errMap := make(map[string][]error)
+ var err error
+ ghutil.Authenticate(githubToken)
+ currentUser, err = ghutil.GetCurrentUser()
+ if nil != err {
+ log.Fatalf("cannot get current authenticated username '%v'", err)
+ }
+
+ // Collect all flaky test issues from all knative repos, in case issues are moved around
+ // Fail this job if data collection failed
+ flakyGHIssuesMap, err := getFlakyIssues()
+ if nil != err {
+ log.Fatalf("%v", err)
+ }
+
+ for _, rd := range repoDataAll {
+ messages, err := processGithubIssueForRepo(rd, flakyGHIssuesMap, dryrun)
+ messagesMap[rd.Config.Repo] = messages
+ if nil != err {
+ errMap[rd.Config.Repo] = append(errMap[rd.Config.Repo], err)
+ }
+ }
+
+ // Print summaries
+ summary := "Summary:\n"
+ for _, rd := range repoDataAll {
+ if messages, ok := messagesMap[rd.Config.Repo]; ok {
+ summary += fmt.Sprintf("Summary of repo '%s':\n", rd.Config.Repo)
+ summary += strings.Join(messages, "\n")
+ }
+ if errs, ok := errMap[rd.Config.Repo]; ok {
+ summary += fmt.Sprintf("Errors in repo '%s':\n%v", rd.Config.Repo, combineErrors(errs))
+ }
+ }
+ log.Println(summary)
+ return nil
+}
diff --git a/tools/flaky-test-reporter/main.go b/tools/flaky-test-reporter/main.go
index 008fc57dc7..679d77ceb6 100644
--- a/tools/flaky-test-reporter/main.go
+++ b/tools/flaky-test-reporter/main.go
@@ -30,6 +30,7 @@ import (
func main() {
serviceAccount := flag.String("service-account", os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"), "JSON key file for service account to use")
+ githubToken := flag.String("github-token", "", "Token file for Github authentication")
dryrun := flag.Bool("dry-run", false, "dry run switch")
flag.Parse()
@@ -37,6 +38,7 @@ func main() {
log.Printf("running in [dry run mode]")
}
+ var repoDataAll []*RepoData
prow.Initialize(*serviceAccount) // Explicit authenticate with gcs Client
// Clean up local artifacts directory, this will be used later for artifacts uploads
@@ -50,7 +52,6 @@ func main() {
log.Fatalf("Failed preparing local artifacts directory: %v", err)
}
- var repoDataAll []*RepoData
for _, jc := range jobConfigs {
log.Printf("collecting results for repo '%s'\n", jc.Repo)
rd, err := collectTestResultsForRepo(&jc)
@@ -62,6 +63,6 @@ func main() {
}
repoDataAll = append(repoDataAll, rd)
}
-
- // TODO(chaodaiG): pass repoDataAll to functions for Github issues tracking and Slack notification
+
+ processGithubIssues(repoDataAll, githubToken, dryrun)
}
diff --git a/tools/flaky-test-reporter/result.go b/tools/flaky-test-reporter/result.go
index ba5bced3af..5a0837d88b 100644
--- a/tools/flaky-test-reporter/result.go
+++ b/tools/flaky-test-reporter/result.go
@@ -32,6 +32,13 @@ import (
"github.com/knative/test-infra/shared/junit"
)
+const (
+ flakyStatus = "Flaky"
+ passedStatus = "Passed"
+ lackDataStatus = "NotEnoughData"
+ failedStatus = "Failed"
+)
+
// RepoData struct contains all configurations and test results for a repo
type RepoData struct {
Config *JobConfig
@@ -68,7 +75,20 @@ func (ts *TestStat) hasEnoughRuns() bool {
return len(ts.Passed) + len(ts.Failed) >= requiredCount
}
-func getFlakyRate(testStats map[string]TestStat) (float32, error) {
+func (ts *TestStat) getTestStatus() string {
+ switch {
+ case ts.isFlaky():
+ return flakyStatus
+ case ts.isPassed():
+ return passedStatus
+ case !ts.hasEnoughRuns():
+ return lackDataStatus
+ default:
+ return failedStatus
+ }
+}
+
+func getFlakyRate(testStats map[string]*TestStat) (float32, error) {
totalCount := len(testStats)
if 0 == totalCount {
return 0.0, nil
@@ -167,6 +187,34 @@ func collectTestResultsForRepo(jc *JobConfig) (*RepoData, error) {
return rd, nil
}
+func (rd *RepoData) getResultSliceForTest(testName string) []junit.TestStatusEnum {
+ res := make([]junit.TestStatusEnum, len(rd.BuildIDs), len(rd.BuildIDs))
+ ts := rd.TestStats[testName]
+ for i, buildID := range rd.BuildIDs {
+ switch {
+ case true == intSliceContains(ts.Failed, buildID):
+ res[i] = junit.Failed
+ case true == intSliceContains(ts.Passed, buildID):
+ res[i] = junit.Passed
+ default:
+ res[i] = junit.Skipped
+ }
+ }
+ return res
+}
+
+func intSliceContains(its []int, target int) bool {
+ if nil == its {
+ return false
+ }
+ for _, it := range its {
+ if it == target {
+ return true
+ }
+ }
+ return false
+}
+
// getLatestFinishedBuilds is an inexpensive way of listing latest finished builds, in comparing to
// the GetLatestBuilds function from prow package, as it doesn't precompute start/finish time before sorting.
// This function takes the assumption that build IDs are always incremental integers, it would fail if it doesn't
diff --git a/tools/flaky-test-reporter/run.go b/tools/flaky-test-reporter/run.go
new file mode 100644
index 0000000000..5dc339a7ae
--- /dev/null
+++ b/tools/flaky-test-reporter/run.go
@@ -0,0 +1,32 @@
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// run.go controls how to run functions that needs dryrun support
+
+package main
+
+import (
+ "log"
+)
+
+func run(message string, call func() error, dryrun *bool) error {
+ if nil != dryrun && true == *dryrun {
+ log.Printf("[dry run] %s", message)
+ return nil
+ }
+ log.Printf(message)
+ return call()
+}