Flaky test reporting(part knative#4) Github issues tracking for flaky…

… tests (knative#506)

* working on github issues tracking

* Update tools/flaky-test-reporter/github_issue.go

Co-Authored-By: chaodaiG <[email protected]>

* update based on PR comments

tools/flaky-test-reporter/config.go

@@ -24,11 +24,16 @@ import (
 
 const (
 	// Builds to be analyzed, this is an arbitrary number
-	buildsCount         = 10
+	buildsCount   = 10
 	// Minimal number of results to be counted as valid results for each testcase, this is an arbitrary number
-	requiredCount       = 8
-	// Don't do anything if found more than 5% tests flaky, this is an arbitrary number
-	threshold           = 0.05
+	requiredCount = 8
+	// Don't do anything if found more than 1% tests flaky, this is an arbitrary number
+	threshold     = 0.01
+
+	org           = "knative"
+	// Temporarily creating issues under "test-infra" for better management
+	// TODO(chaodaiG): repo for issue same as the src of the test
+	repoForIssue  = "test-infra"
 )
 
 // jobConfigs defines which job to be monitored for giving repo

tools/flaky-test-reporter/error.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2019 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// error.go helps with error handling
+
+package main
+
+import (
+	"fmt"
+	"strings"
+)
+
+// combineErrors combines slice of errors and return a single error
+func combineErrors(errs []error) error {
+	if nil == errs || 0 == len(errs) {
+		return nil
+	}
+	var errStrs []string
+	for _, err := range errs {
+		errStrs = append(errStrs, err.Error())
+	}
+	return fmt.Errorf(strings.Join(errStrs, "\n"))
+}

tools/flaky-test-reporter/ghutil/ghutil.go

@@ -38,7 +38,7 @@ const (
 	// IssueOpenState is the state of open github issue
 	IssueOpenState IssueStateEnum = "open"
 	// IssueCloseState is the state of closed github issue
-	IssueCloseState IssueStateEnum = "close"
+	IssueCloseState IssueStateEnum = "closed"
 	// IssueAllState is the state for all, useful when querying issues
 	IssueAllState  IssueStateEnum = "all"
 )
@@ -80,6 +80,31 @@ func GetCurrentUser() (*github.User, error) {
 	return res, err
 }
 
+// ListRepos lists repos under org
+func ListRepos(org string) ([]string, error) {
+	var res []string
+	options := &github.ListOptions{}
+	genericList, err := depaginate(
+		"listing repos",
+		maxRetryCount,
+		options,
+		func() ([]interface{}, *github.Response, error) {
+			page, resp, err := client.Repositories.List(ctx, org, nil)
+			var interfaceList []interface{}
+			if nil == err {
+				for _, repo := range page {
+					interfaceList = append(interfaceList, repo)
+				}
+			}
+			return interfaceList, resp, err
+		},
+	)
+	for _, repo := range genericList {
+		res = append(res, repo.(*github.Repository).GetName())
+	}
+	return res, err
+}
+
 // ListIssuesByRepo lists issues within given repo, filters by labels if provided
 func ListIssuesByRepo(org, repo string, labels []string) ([]*github.Issue, error) {
 	issueListOptions := github.IssueListByRepoOptions{
@@ -92,7 +117,7 @@ func ListIssuesByRepo(org, repo string, labels []string) ([]*github.Issue, error
 	var res []*github.Issue
 	options := &github.ListOptions{}
 	genericList, err := depaginate(
-		fmt.Sprintf("listing genericList with label '%v'", labels),
+		fmt.Sprintf("listing issues with label '%v'", labels),
 		maxRetryCount,
 		options,
 		func() ([]interface{}, *github.Response, error) {