Skip to content

Commit

Permalink
Merge cockroachdb#107444
Browse files Browse the repository at this point in the history 
107444: issues: use prefix match to check for existing issue r=renatolabs,tbg a=rafiss

It is conventional to annotate a test failure issue title with some additional information about the failure cause. The previous logic was too strict in requiring an exact title match. Now it will match as long as the title matches exactly OR if the prefix matches and is followed by whitespace.

Example:
I added a suffix to [`sql/tests: TestRandomSyntaxFunctions failed [panic in request_job_execution_details]`](cockroachdb#106970)  since the issue was with a function the jobs team added. But the next day cockroachdb#107237 was opened anew. This PR would prevent that from happening.

Epic: None
Release note: None

Co-authored-by: Rafi Shamim <[email protected]>
  • Loading branch information
@rafiss
craig[bot] and rafiss committed Jul 24, 2023
2 parents 06fb4c1 + 77f32c4 commit c917a6b
Show file tree
Hide file tree
Showing 20 changed files with 62 additions and 60 deletions.
22 changes: 12 additions & 10 deletions pkg/cmd/internal/issues/issues.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"log"
"net/url"
"os"
"regexp"
"strings"

"github.com/cockroachdb/cockroach/pkg/build"
Expand Down Expand Up @@ -316,7 +317,7 @@ func (p *poster) post(origCtx context.Context, formatter IssueFormatter, req Pos
rRelated = &github.IssuesSearchResult{}
}

existingIssues := filterByExactTitleMatch(rExisting, title)
existingIssues := filterByPrefixTitleMatch(rExisting, title)
var foundIssue *int
if len(existingIssues) > 0 {
// We found an existing issue to post a comment into.
Expand All @@ -327,7 +328,7 @@ func (p *poster) post(origCtx context.Context, formatter IssueFormatter, req Pos
data.MentionOnCreate = nil
}

data.RelatedIssues = filterByExactTitleMatch(rRelated, title)
data.RelatedIssues = filterByPrefixTitleMatch(rRelated, title)
data.InternalLog = ctx.Builder.String()
r := &Renderer{}
if err := formatter.Body(r, data); err != nil {
Expand Down Expand Up @@ -486,18 +487,19 @@ func HelpCommandAsLink(title, href string) func(r *Renderer) {
}
}

// filterByExactTitleMatch filters the search result passed and
// removes any issues where the title does not match the expected
// title exactly. This is done because the GitHub API does not support
// searching by exact title; as a consequence, without this function,
// there is a chance we would group together test failures for two
// similarly named tests. That is confusing and undesirable behavior.
func filterByExactTitleMatch(
// filterByPrefixTitleMatch filters the search result passed and removes any
// issues where the title does not match the expected title, optionally followed
// by whitespace. This is done because the GitHub API does not support searching
// by exact title; as a consequence, without this function, there is a chance we
// would group together test failures for two similarly named tests. That is
// confusing and undesirable behavior.
func filterByPrefixTitleMatch(
result *github.IssuesSearchResult, expectedTitle string,
) []github.Issue {
expectedTitleRegex := regexp.MustCompile(`^` + expectedTitle + `(\s+|$)`)
var issues []github.Issue
for _, issue := range result.Issues {
if title := issue.Title; title != nil && *title == expectedTitle {
if title := issue.Title; title != nil && expectedTitleRegex.MatchString(*title) {
issues = append(issues, issue)
}
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/cmd/internal/issues/issues_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@ test logs left over in: /go/src/github.com/cockroachdb/cockroach/artifacts/logTe
issueNum = *base.Number
}
return github.Issue{
Title: github.String(fmt.Sprintf("%s: %s%s failed", packageName, testName, suffix)),
Title: github.String(fmt.Sprintf("%s: %s%s failed [failure reason]", packageName, testName, suffix)),
Number: &issueNum,
Labels: base.Labels,
}
Expand Down Expand Up @@ -213,7 +213,7 @@ test logs left over in: /go/src/github.com/cockroachdb/cockroach/artifacts/logTe
relatedIssue := github.Issue{
// Title is generated during the test using the test case's
// package and test names
Number: github.Int(issueNumber + 1),
Number: github.Int(issueNumber + 10),
Labels: []github.Label{{
Name: github.String("C-test-failure"),
URL: github.String("fake"),
Expand Down
8 changes: 4 additions & 4 deletions pkg/cmd/internal/issues/testdata/post/failure-matching-and-related-issue.txt
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
post
----
----
searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestReplicateQueueRebalance failed" label:branch-release-0.1 -label:X-noreuse: [github.Issue{Number:31, Title:"storage: TestReplicateQueueRebalance-similar failed", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.1"}]} github.Issue{Number:30, Title:"storage: TestReplicateQueueRebalance failed", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.1"}]}]
searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestReplicateQueueRebalance failed" -label:branch-release-0.1: [github.Issue{Number:31, Title:"storage: TestReplicateQueueRebalance failed", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.2"}]}]
searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestReplicateQueueRebalance failed" label:branch-release-0.1 -label:X-noreuse: [github.Issue{Number:31, Title:"storage: TestReplicateQueueRebalance-similar failed [failure reason]", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.1"}]} github.Issue{Number:30, Title:"storage: TestReplicateQueueRebalance failed [failure reason]", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.1"}]}]
searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestReplicateQueueRebalance failed" -label:branch-release-0.1: [github.Issue{Number:40, Title:"storage: TestReplicateQueueRebalance failed [failure reason]", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.2"}]}]
createComment owner=cockroachdb repo=cockroach issue=30:

storage.TestReplicateQueueRebalance [failed](https://teamcity.example.com/buildConfiguration/nightly123/8008135?buildTab=log) on release-0.1 @ [abcd123](https://github.com/cockroachdb/cockroach/commits/abcd123):
Expand All @@ -24,7 +24,7 @@ See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachl
<details><summary>Same failure on other branches</summary>
<p>

- #31 storage: TestReplicateQueueRebalance failed [C-test-failure O-robot release-0.2]
- #40 storage: TestReplicateQueueRebalance failed [failure reason] [C-test-failure O-robot release-0.2]
</p>
</details>
<sub>
Expand All @@ -33,6 +33,6 @@ See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachl
</sub>


Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+storage%3A+TestReplicateQueueRebalance+failed+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E
Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2340+storage%3A+TestReplicateQueueRebalance+failed+%5Bfailure+reason%5D+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=%3Ccomment%3E
----
----
2 changes: 1 addition & 1 deletion pkg/cmd/internal/issues/testdata/post/failure-matching-issue.txt
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
post
----
----
searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestReplicateQueueRebalance failed" label:branch-release-0.1 -label:X-noreuse: [github.Issue{Number:30, Title:"storage: TestReplicateQueueRebalance failed", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.1"}]} github.Issue{Number:32, Title:"storage: TestReplicateQueueRebalance-similar failed", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.1"}]}]
searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestReplicateQueueRebalance failed" label:branch-release-0.1 -label:X-noreuse: [github.Issue{Number:30, Title:"storage: TestReplicateQueueRebalance failed [failure reason]", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.1"}]} github.Issue{Number:32, Title:"storage: TestReplicateQueueRebalance-similar failed [failure reason]", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.1"}]}]
searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestReplicateQueueRebalance failed" -label:branch-release-0.1: []
createComment owner=cockroachdb repo=cockroach issue=30:

Expand Down
6 changes: 3 additions & 3 deletions pkg/cmd/internal/issues/testdata/post/failure-related-issue.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ post
----
----
searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestReplicateQueueRebalance failed" label:branch-release-0.1 -label:X-noreuse: []
searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestReplicateQueueRebalance failed" -label:branch-release-0.1: [github.Issue{Number:32, Title:"storage: TestReplicateQueueRebalance-similar failed", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.2"}]} github.Issue{Number:31, Title:"storage: TestReplicateQueueRebalance failed", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.2"}]}]
searchIssue repo:"cockroach" user:"cockroachdb" is:issue is:open in:title label:"C-test-failure" sort:created-desc "storage: TestReplicateQueueRebalance failed" -label:branch-release-0.1: [github.Issue{Number:41, Title:"storage: TestReplicateQueueRebalance-similar failed [failure reason]", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.2"}]} github.Issue{Number:40, Title:"storage: TestReplicateQueueRebalance failed [failure reason]", Labels:[github.Label{URL:"fake", Name:"C-test-failure"} github.Label{URL:"fake", Name:"O-robot"} github.Label{URL:"fake", Name:"release-0.2"}]}]
getBinaryVersion: result v3.3.0
listMilestones owner=cockroachdb repo=cockroach: result [github.Milestone{Number:2, Title:"3.3"} github.Milestone{Number:1, Title:"3.2"}]
createIssue owner=cockroachdb repo=cockroach:
Expand All @@ -29,7 +29,7 @@ See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachl
<details><summary>Same failure on other branches</summary>
<p>

- #31 storage: TestReplicateQueueRebalance failed [C-test-failure O-robot release-0.2]
- #40 storage: TestReplicateQueueRebalance failed [failure reason] [C-test-failure O-robot release-0.2]
</p>
</details>
/cc @cockroachdb/idonotexistbecausethisisatest
Expand All @@ -39,6 +39,6 @@ See also: [How To Investigate a Go Test Failure \(internal\)](https://cockroachl
</sub>


Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2331+storage%3A+TestReplicateQueueRebalance+failed+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+TestReplicateQueueRebalance+failed
Rendered: https://github.com/cockroachdb/cockroach/issues/new?body=storage.TestReplicateQueueRebalance+%5Bfailed%5D%28https%3A%2F%2Fteamcity.example.com%2FbuildConfiguration%2Fnightly123%2F8008135%3FbuildTab%3Dlog%29+on+release-0.1+%40+%5Babcd123%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Fcommits%2Fabcd123%29%3A%0A%0A%0A%60%60%60%0A%09%3Cautogenerated%3E%3A12%3A+storage%2Freplicate_queue_test.go%3A103%2C+condition+failed+to+evaluate+within+45s%3A+not+balanced%3A+%5B10+1+10+1+8%5D%0A%60%60%60%0A%3Cp%3EParameters%3A+%3Ccode%3EGOFLAGS%3Drace%3C%2Fcode%3E%0A%2C+%3Ccode%3EROACHTEST_cloud%3Dgce%3C%2Fcode%3E%0A%2C+%3Ccode%3ETAGS%3Ddeadlock%3C%2Fcode%3E%0A%3C%2Fp%3E%0A%3Cdetails%3E%3Csummary%3EHelp%3C%2Fsummary%3E%0A%3Cp%3E%0A%0ASee+also%3A+%5BHow+To+Investigate+a+Go+Test+Failure+%5C%28internal%5C%29%5D%28https%3A%2F%2Fcockroachlabs.atlassian.net%2Fl%2Fc%2FHgfXfJgM%29%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%3Cdetails%3E%3Csummary%3ESame+failure+on+other+branches%3C%2Fsummary%3E%0A%3Cp%3E%0A%0A-+%2340+storage%3A+TestReplicateQueueRebalance+failed+%5Bfailure+reason%5D+%5BC-test-failure+O-robot+release-0.2%5D%0A%3C%2Fp%3E%0A%3C%2Fdetails%3E%0A%2Fcc+%40cockroachdb%2Fidonotexistbecausethisisatest%0A%3Csub%3E%0A%0A%5BThis+test+on+roachdash%5D%28https%3A%2F%2Froachdash.crdb.dev%2F%3Ffilter%3Dstatus%3Aopen%2520t%3A.%2ATestReplicateQueueRebalance.%2A%26sort%3Dtitle%2Bcreated%26display%3Dlastcommented%2Bproject%29+%7C+%5BImprove+this+report%21%5D%28https%3A%2F%2Fgithub.com%2Fcockroachdb%2Fcockroach%2Ftree%2Fmaster%2Fpkg%2Fcmd%2Finternal%2Fissues%29%0A%3C%2Fsub%3E%0A&title=storage%3A+TestReplicateQueueRebalance+failed
----
----
Loading

0 comments on commit c917a6b

Please sign in to comment.