diff --git a/build/teamcity/internal/cockroach/nightlies/private_roachtest.sh b/build/teamcity/internal/cockroach/nightlies/private_roachtest.sh index 1098e44af316..ccd51e14c24d 100755 --- a/build/teamcity/internal/cockroach/nightlies/private_roachtest.sh +++ b/build/teamcity/internal/cockroach/nightlies/private_roachtest.sh @@ -11,9 +11,13 @@ export TESTS="${TESTS:-costfuzz/workload-replay}" export ROACHTEST_BUCKET="${ROACHTEST_BUCKET:-cockroach-nightly-private}" export GCE_PROJECT="e2e-infra-381422" export BACKUP_TESTING_BUCKET="cockroach-backup-testing-private" + +# TODO(msbutler): use a different bucket once it is created. Sadly, I don't have the permissions +# currently to create a new bucket in this gce project. +export BACKUP_TESTING_BUCKET_LONG_TTL="cockroach-backup-testing-private" export COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING=1 export COCKROACH_NO_EXAMPLE_DATABASE=1 export COCKROACH_AUTO_BALLAST=false -BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD=gce -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e ROACHTEST_PRIVATE -e ROACHTEST_BUCKET -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e COCKROACH_DEV_LICENSE -e BACKUP_TESTING_BUCKET -e SFUSER -e SFPASSWORD -e COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING -e COCKROACH_NO_EXAMPLE_DATABASE -e COCKROACH_AUTO_BALLAST -e GCE_PROJECT" \ +BAZEL_SUPPORT_EXTRA_DOCKER_ARGS="-e LITERAL_ARTIFACTS_DIR=$root/artifacts -e BUILD_VCS_NUMBER -e CLOUD=gce -e TESTS -e COUNT -e GITHUB_API_TOKEN -e GITHUB_ORG -e GITHUB_REPO -e GOOGLE_EPHEMERAL_CREDENTIALS -e ROACHTEST_PRIVATE -e ROACHTEST_BUCKET -e SLACK_TOKEN -e TC_BUILDTYPE_ID -e TC_BUILD_BRANCH -e TC_BUILD_ID -e TC_SERVER_URL -e COCKROACH_DEV_LICENSE -e BACKUP_TESTING_BUCKET -e BACKUP_TESTING_BUCKET_LONG_TTL -e SFUSER -e SFPASSWORD -e COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING -e COCKROACH_NO_EXAMPLE_DATABASE -e COCKROACH_AUTO_BALLAST -e GCE_PROJECT" \ run_bazel build/teamcity/internal/cockroach/nightlies/private_roachtest_impl.sh diff --git a/pkg/ccl/streamingccl/streamingest/replication_stream_e2e_test.go b/pkg/ccl/streamingccl/streamingest/replication_stream_e2e_test.go index 02ca8ff9f410..0ae0e8b0f52f 100644 --- a/pkg/ccl/streamingccl/streamingest/replication_stream_e2e_test.go +++ b/pkg/ccl/streamingccl/streamingest/replication_stream_e2e_test.go @@ -1218,4 +1218,31 @@ func TestStreamingRegionalConstraint(t *testing.T) { testutils.SucceedsSoon(t, checkLocalities(tableDesc.PrimaryIndexSpan(destCodec), rangedesc.NewScanner(c.DestSysServer.DB()))) + + tableName := "test" + tabledIDQuery := fmt.Sprintf(`SELECT id FROM system.namespace WHERE name ='%s'`, tableName) + + var tableID uint32 + c.SrcTenantSQL.QueryRow(t, tabledIDQuery).Scan(&tableID) + fmt.Printf("%d", tableID) + + checkLocalityRanges(t, c.SrcSysSQL, srcCodec, uint32(tableDesc.GetID()), "mars") + +} + +func checkLocalityRanges( + t *testing.T, sysSQL *sqlutils.SQLRunner, codec keys.SQLCodec, tableID uint32, region string, +) { + targetPrefix := codec.TablePrefix(tableID) + distinctQuery := fmt.Sprintf(` +SELECT + DISTINCT replica_localities +FROM + [SHOW CLUSTER RANGES] +WHERE + start_key ~ '%s' +`, targetPrefix) + var locality string + sysSQL.QueryRow(t, distinctQuery).Scan(&locality) + require.Contains(t, locality, region) } diff --git a/pkg/cmd/docs-issue-generation/BUILD.bazel b/pkg/cmd/docs-issue-generation/BUILD.bazel index 100f0d4c48f3..dceee74edffa 100644 --- a/pkg/cmd/docs-issue-generation/BUILD.bazel +++ b/pkg/cmd/docs-issue-generation/BUILD.bazel @@ -4,12 +4,19 @@ go_library( name = "docs-issue-generation_lib", srcs = [ "docs_issue_generation.go", + "extract.go", + "format.go", + "github.go", + "jira.go", "main.go", + "requests.go", + "structs.go", ], importpath = "github.com/cockroachdb/cockroach/pkg/cmd/docs-issue-generation", visibility = ["//visibility:private"], deps = [ "//pkg/util/timeutil", + "@com_github_cockroachdb_errors//:errors", "@com_github_spf13_cobra//:cobra", ], ) diff --git a/pkg/cmd/docs-issue-generation/docs_issue_generation.go b/pkg/cmd/docs-issue-generation/docs_issue_generation.go index d3aa2e19e227..a7bd4a575b18 100644 --- a/pkg/cmd/docs-issue-generation/docs_issue_generation.go +++ b/pkg/cmd/docs-issue-generation/docs_issue_generation.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Cockroach Authors. +// Copyright 2023 The Cockroach Authors. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt. @@ -11,903 +11,49 @@ package main import ( - "bytes" - "encoding/json" "fmt" - "io" - "net/http" - "regexp" - "strconv" - "strings" "time" - - "github.com/cockroachdb/cockroach/pkg/util/timeutil" -) - -// docsIssue contains details about each formatted commit to be committed to the docs repo. -type docsIssue struct { - sourceCommitSha string - title string - body string - labels []string -} - -// parameters stores the GitHub API token, a dry run flag to output the issues it would create, and the -// start and end times of the search. -type parameters struct { - Token string // GitHub API token - DryRun bool - StartTime time.Time - EndTime time.Time -} - -// pageInfo contains pagination information for querying the GraphQL API. -type pageInfo struct { - HasNextPage bool `json:"hasNextPage"` - EndCursor string `json:"endCursor"` -} - -// gqlCockroachPRCommit contains details about commits within PRs in the cockroach repo. -type gqlCockroachPRCommit struct { - Data struct { - Repository struct { - PullRequest struct { - Commits struct { - Edges []struct { - Node struct { - Commit struct { - Oid string `json:"oid"` - MessageHeadline string `json:"messageHeadline"` - MessageBody string `json:"messageBody"` - } `json:"commit"` - } `json:"node"` - } `json:"edges"` - PageInfo pageInfo `json:"pageInfo"` - } `json:"commits"` - } `json:"pullRequest"` - } `json:"repository"` - } `json:"data"` -} - -// gqlCockroachPR contains details about PRs within the cockroach repo. -type gqlCockroachPR struct { - Data struct { - Search struct { - Nodes []struct { - Title string `json:"title"` - Number int `json:"number"` - Body string `json:"body"` - BaseRefName string `json:"baseRefName"` - Commits struct { - Edges []struct { - Node struct { - Commit struct { - Oid string `json:"oid"` - MessageHeadline string `json:"messageHeadline"` - MessageBody string `json:"messageBody"` - } `json:"commit"` - } `json:"node"` - } `json:"edges"` - PageInfo pageInfo `json:"pageInfo"` - } `json:"commits"` - } `json:"nodes"` - PageInfo pageInfo `json:"pageInfo"` - } `json:"search"` - } `json:"data"` -} - -// gqlDocsIssue contains details about existing issues within the docs repo. -type gqlDocsIssue struct { - Data struct { - Search struct { - Nodes []struct { - Number int `json:"number"` - Body string `json:"body"` - } `json:"nodes"` - PageInfo pageInfo `json:"pageInfo"` - } `json:"search"` - } `json:"data"` -} - -type gqlSingleIssue struct { - Data struct { - Repository struct { - Issue struct { - Body string `json:"body"` - } `json:"issue"` - } `json:"repository"` - } `json:"data"` -} - -// gqlDocsRepoLabels contains details about the labels within the cockroach repo. In order to create issues using the -// GraphQL API, we need to use the label IDs and no -type gqlDocsRepoLabels struct { - Data struct { - Repository struct { - ID string `json:"id"` - Labels struct { - Edges []struct { - Node struct { - Name string `json:"name"` - ID string `json:"id"` - } `json:"node"` - } `json:"edges"` - PageInfo pageInfo `json:"pageInfo"` - } `json:"labels"` - } `json:"repository"` - } `json:"data"` -} - -type gqlCreateIssueMutation struct { - Data struct { - CreateIssue struct { - ClientMutationID interface{} `json:"clientMutationId"` - } `json:"createIssue"` - } `json:"data"` -} - -type cockroachPR struct { - Title string `json:"title"` - Number int `json:"number"` - Body string `json:"body"` - BaseRefName string `json:"baseRefName"` - Commits []cockroachCommit -} - -type cockroachCommit struct { - Sha string `json:"oid"` - MessageHeadline string `json:"messageHeadline"` - MessageBody string `json:"messageBody"` -} - -type epicIssueRefInfo struct { - epicRefs map[string]int - epicNone bool - issueCloseRefs map[string]int - issueInformRefs map[string]int - isBugFix bool -} - -// Regex components for finding and validating issue and epic references in a string -var ( - ghIssuePart = `(#\d+)` // e.g., #12345 - ghIssueRepoPart = `([\w.-]+[/][\w.-]+#\d+)` // e.g., cockroachdb/cockroach#12345 - ghURLPart = `(https://github.com/[-a-z0-9]+/[-._a-z0-9/]+/issues/\d+)` // e.g., https://github.com/cockroachdb/cockroach/issues/12345 - jiraIssuePart = `([[:alpha:]]+-\d+)` // e.g., DOC-3456 - exalateJiraRefPart = `Jira issue: ` + jiraIssuePart // e.g., Jira issue: CRDB-54321 - jiraBaseUrlPart = "https://cockroachlabs.atlassian.net/browse/" - jiraURLPart = jiraBaseUrlPart + jiraIssuePart // e.g., https://cockroachlabs.atlassian.net/browse/DOC-3456 - issueRefPart = ghIssuePart + "|" + ghIssueRepoPart + "|" + ghURLPart + "|" + jiraIssuePart + "|" + jiraURLPart - afterRefPart = `[,.;]?(?:[ \t\n\r]+|$)` -) - -// RegExes of each issue part -var ( - ghIssuePartRE = regexp.MustCompile(ghIssuePart) - ghIssueRepoPartRE = regexp.MustCompile(ghIssueRepoPart) - ghURLPartRE = regexp.MustCompile(ghURLPart) - jiraIssuePartRE = regexp.MustCompile(jiraIssuePart) - jiraURLPartRE = regexp.MustCompile(jiraURLPart) -) - -// Fully composed regexs used to match strings. -var ( - fixIssueRefRE = regexp.MustCompile(`(?im)(?i:close[sd]?|fix(?:e[sd])?|resolve[sd]?):?\s+(?:(?:` + issueRefPart + `)` + afterRefPart + ")+") - informIssueRefRE = regexp.MustCompile(`(?im)(?:part of|see also|informs):?\s+(?:(?:` + issueRefPart + `)` + afterRefPart + ")+") - epicRefRE = regexp.MustCompile(`(?im)epic:?\s+(?:(?:` + jiraIssuePart + "|" + jiraURLPart + `)` + afterRefPart + ")+") - epicNoneRE = regexp.MustCompile(`(?im)epic:?\s+(?:(none)` + afterRefPart + ")+") - githubJiraIssueRefRE = regexp.MustCompile(issueRefPart) - jiraIssueRefRE = regexp.MustCompile(jiraIssuePart + "|" + jiraURLPart) - releaseNoteNoneRE = regexp.MustCompile(`(?i)release note:? [nN]one`) - allRNRE = regexp.MustCompile(`(?i)release note:? \(.*`) - nonBugFixRNRE = regexp.MustCompile(`(?i)release note:? \(([^b]|b[^u]|bu[^g]|bug\S|bug [^f]|bug f[^i]|bug fi[^x]).*`) - bugFixRNRE = regexp.MustCompile(`(?i)release note:? \(bug fix\):.*`) - releaseJustificationRE = regexp.MustCompile(`(?i)release justification:.*`) - prNumberRE = regexp.MustCompile(`Related PR: \[?https://github.com/cockroachdb/cockroach/pull/(\d+)\D`) - commitShaRE = regexp.MustCompile(`Commit: \[?https://github.com/cockroachdb/cockroach/commit/(\w+)\W`) - exalateJiraRefRE = regexp.MustCompile(exalateJiraRefPart) -) - -const ( - docsOrganization = "cockroachdb" - docsRepo = "docs" ) // the heart of the script to fetch and manipulate all data and create the individual docs issues -func docsIssueGeneration(params parameters) { - repoID, labelMap, err := searchDocsRepoLabels(params.Token) +func docsIssueGeneration(params queryParameters) error { + prs, err := searchCockroachPRs(params.StartTime, params.EndTime) if err != nil { - fmt.Println(err) + return err } - prs, err := searchCockroachPRs(params.StartTime, params.EndTime, params.Token) + docsIssues, err := constructDocsIssues(prs) if err != nil { - fmt.Println(err) + return err } - docsIssues := constructDocsIssues(prs, params.Token) if params.DryRun { - fmt.Printf("Start time: %#v\n", params.StartTime.Format(time.RFC3339)) - fmt.Printf("End time: %#v\n", params.EndTime.Format(time.RFC3339)) + fmt.Printf("Start time: %+v\n", params.StartTime.Format(time.RFC3339)) + fmt.Printf("End time: %+v\n", params.EndTime.Format(time.RFC3339)) + fmt.Printf("Number of PRs found: %d\n", len(prs)) if len(docsIssues) > 0 { fmt.Printf("Dry run is enabled. The following %d docs issue(s) would be created:\n", len(docsIssues)) - fmt.Println(docsIssues) + fmt.Printf("%+v\n", docsIssues) } else { fmt.Println("No docs issues need to be created.") } } else { - for _, di := range docsIssues { - err := di.createDocsIssues(params.Token, repoID, labelMap) - if err != nil { - fmt.Println(err) - } - } - } -} - -// searchDocsRepoLabels passes in a GitHub API token and returns the repo ID of the docs repo as well as a map -// of all the labels and their respective label IDs in GitHub. -func searchDocsRepoLabels(token string) (string, map[string]string, error) { - var labels = map[string]string{} - var repoID string - repoID, hasNextPage, nextCursor, err := searchDocsRepoLabelsSingle("", labels, token) - if err != nil { - fmt.Println(err) - return "", nil, err - } - for hasNextPage { - _, hasNextPage, nextCursor, err = searchDocsRepoLabelsSingle(nextCursor, labels, token) - if err != nil { - fmt.Println(err) - return "", nil, err - } - } - return repoID, labels, nil -} - -// searchDocsRepoLabelsSingle runs once per page of 100 labels within the docs repo. It returns the repo ID, -// whether or not there is another page after the one that just ran, the next cursor value used to query the next -// page, and any error. -func searchDocsRepoLabelsSingle( - cursor string, m map[string]string, token string, -) (string, bool, string, error) { - docsIssueGQLQuery := fmt.Sprintf(`query ($cursor: String) { - repository(owner: "%s", name: "%s") { - id - labels(first: 100, after: $cursor) { - edges { - node { - name - id - } - } - pageInfo { - hasNextPage - endCursor - } - } - } - }`, docsOrganization, docsRepo) - var search gqlDocsRepoLabels - queryVariables := map[string]interface{}{} - if cursor != "" { - queryVariables["cursor"] = cursor - } - err := queryGraphQL(docsIssueGQLQuery, queryVariables, token, &search) - if err != nil { - fmt.Println(err) - return "", false, "", err - } - repoID := search.Data.Repository.ID - for _, x := range search.Data.Repository.Labels.Edges { - m[x.Node.Name] = x.Node.ID - } - pageInfo := search.Data.Repository.Labels.PageInfo - return repoID, pageInfo.HasNextPage, pageInfo.EndCursor, nil -} - -// queryGraphQL is the function that interfaces directly with the GitHub GraphQL API. Given a query, variables, and -// token, it will return a struct containing the requested data or an error if one exists. -func queryGraphQL( - query string, queryVariables map[string]interface{}, token string, out interface{}, -) error { - const graphQLURL = "https://api.github.com/graphql" - client := &http.Client{} - bodyInt := map[string]interface{}{ - "query": query, - } - if queryVariables != nil { - bodyInt["variables"] = queryVariables - } - requestBody, err := json.Marshal(bodyInt) - if err != nil { - return err - } - req, err := http.NewRequest("POST", graphQLURL, bytes.NewBuffer(requestBody)) - if err != nil { - return err - } - req.Header.Set("Authorization", "token "+token) - res, err := client.Do(req) - if err != nil { - return err - } - bs, err := io.ReadAll(res.Body) - if err != nil { - return err - } - // unmarshal (convert) the byte slice into an interface - var tmp interface{} - err = json.Unmarshal(bs, &tmp) - if err != nil { - fmt.Println("Error: unable to unmarshal JSON into an empty interface") - fmt.Println(string(bs[:])) - return err - } - err = json.Unmarshal(bs, out) - if err != nil { - return err - } - return nil -} - -func searchCockroachPRs( - startTime time.Time, endTime time.Time, token string, -) ([]cockroachPR, error) { - prCommitsToExclude, err := searchDocsIssues(startTime, token) - if err != nil { - fmt.Println(err) - } - hasNextPage, nextCursor, prs, err := searchCockroachPRsSingle(startTime, endTime, "", prCommitsToExclude, token) - if err != nil { - fmt.Println(err) - return nil, err - } - result := prs - for hasNextPage { - hasNextPage, nextCursor, prs, err = searchCockroachPRsSingle(startTime, endTime, nextCursor, prCommitsToExclude, token) - if err != nil { - fmt.Println(err) - return nil, err - } - result = append(result, prs...) - } - return result, nil -} - -// searchDocsIssues returns a map containing all the product change docs issues that have been created since the given -// start time. For reference, it's structured as map[crdb_pr_number]map[crdb_commit]docs_pr_number. -func searchDocsIssues(startTime time.Time, token string) (map[int]map[string]int, error) { - var result = map[int]map[string]int{} - hasNextPage, nextCursor, err := searchDocsIssuesSingle(startTime, "", result, token) - if err != nil { - fmt.Println(err) - return nil, err - } - for hasNextPage { - hasNextPage, nextCursor, err = searchDocsIssuesSingle(startTime, nextCursor, result, token) - if err != nil { - fmt.Println(err) - return nil, err - } - } - return result, nil -} - -// searchDocsIssuesSingle searches one page of docs issues at a time. These docs issues will ultimately be excluded -// from the PRs through which we iterate to create new product change docs issues. This function returns a bool to -// check if there are more than 100 results, the cursor to query for the next page of results, and an error if -// one exists. -func searchDocsIssuesSingle( - startTime time.Time, cursor string, m map[int]map[string]int, token string, -) (bool, string, error) { - query := `query ($cursor: String, $ghSearchQuery: String!) { - search(first: 100, query: $ghSearchQuery, type: ISSUE, after: $cursor) { - nodes { - ... on Issue { - number - body - } - } - pageInfo { - hasNextPage - endCursor + batchSize := 50 + for i := 0; i < len(docsIssues); i += batchSize { + end := i + batchSize + if end > len(docsIssues) { + end = len(docsIssues) } - } - }` - var search gqlDocsIssue - today := timeutil.Now().Format(time.RFC3339) - queryVariables := map[string]interface{}{ - "ghSearchQuery": fmt.Sprintf(`repo:%s/%s is:issue label:C-product-change created:%s..%s`, docsOrganization, docsRepo, startTime.Format(time.RFC3339), today), - } - if cursor != "" { - queryVariables["cursor"] = cursor - } - err := queryGraphQL(query, queryVariables, token, &search) - if err != nil { - fmt.Println(err) - return false, "", err - } - for _, x := range search.Data.Search.Nodes { - prNumber, commitSha, err := parseDocsIssueBody(x.Body) - if err != nil { - fmt.Println(err) - } - if prNumber != 0 && commitSha != "" { - _, ok := m[prNumber] - if !ok { - m[prNumber] = make(map[string]int) - } - m[prNumber][commitSha] = x.Number - } - } - pageInfo := search.Data.Search.PageInfo - return pageInfo.HasNextPage, pageInfo.EndCursor, nil -} - -func parseDocsIssueBody(body string) (int, string, error) { - prMatches := prNumberRE.FindStringSubmatch(body) - if len(prMatches) < 2 { - return 0, "", fmt.Errorf("error: No PR number found in issue body") - } - prNumber, err := strconv.Atoi(prMatches[1]) - if err != nil { - fmt.Println(err) - return 0, "", err - } - commitShaMatches := commitShaRE.FindStringSubmatch(body) - if len(commitShaMatches) < 2 { - return 0, "", fmt.Errorf("error: No commit SHA found in issue body") - } - return prNumber, commitShaMatches[1], nil -} - -func searchCockroachPRsSingle( - startTime time.Time, - endTime time.Time, - cursor string, - prCommitsToExclude map[int]map[string]int, - token string, -) (bool, string, []cockroachPR, error) { - var search gqlCockroachPR - var result []cockroachPR - query := `query ($cursor: String, $ghSearchQuery: String!) { - search(first: 100, query: $ghSearchQuery, type: ISSUE, after: $cursor) { - nodes { - ... on PullRequest { - title - number - body - baseRefName - commits(first: 100) { - edges { - node { - commit { - oid - messageHeadline - messageBody - } - } - } - pageInfo { - hasNextPage - endCursor - } - } - } - } - pageInfo { - hasNextPage - endCursor - } - } - }` - queryVariables := map[string]interface{}{ - "ghSearchQuery": fmt.Sprintf( - `repo:cockroachdb/cockroach is:pr is:merged merged:%s..%s`, - startTime.Format(time.RFC3339), - endTime.Format(time.RFC3339), - ), - } - if cursor != "" { - queryVariables["cursor"] = cursor - } - err := queryGraphQL(query, queryVariables, token, &search) - if err != nil { - fmt.Println(err) - return false, "", nil, err - } - for _, x := range search.Data.Search.Nodes { - var commits []cockroachCommit - for _, y := range x.Commits.Edges { - matchingDocsIssue := prCommitsToExclude[x.Number][y.Node.Commit.Oid] - if nonBugFixRNRE.MatchString(y.Node.Commit.MessageBody) && matchingDocsIssue == 0 { - commit := cockroachCommit{ - Sha: y.Node.Commit.Oid, - MessageHeadline: y.Node.Commit.MessageHeadline, - MessageBody: y.Node.Commit.MessageBody, - } - commits = append(commits, commit) + batch := docsIssueBatch{ + IssueUpdates: docsIssues[i:end], } - } - // runs if there are more than 100 results - if x.Commits.PageInfo.HasNextPage { - additionalCommits, err := searchCockroachPRCommits(x.Number, x.Commits.PageInfo.EndCursor, prCommitsToExclude, token) + err := batch.createDocsIssuesInBulk() if err != nil { - return false, "", nil, err - } - commits = append(commits, additionalCommits...) - } - if len(commits) > 0 { - pr := cockroachPR{ - Number: x.Number, - Title: x.Title, - Body: x.Body, - BaseRefName: x.BaseRefName, - Commits: commits, - } - result = append(result, pr) - } - } - pageInfo := search.Data.Search.PageInfo - return pageInfo.HasNextPage, pageInfo.EndCursor, result, nil -} - -func searchCockroachPRCommits( - pr int, cursor string, prCommitsToExclude map[int]map[string]int, token string, -) ([]cockroachCommit, error) { - hasNextPage, nextCursor, commits, err := searchCockroachPRCommitsSingle(pr, cursor, prCommitsToExclude, token) - if err != nil { - fmt.Println(err) - return nil, err - } - result := commits - for hasNextPage { - hasNextPage, nextCursor, commits, err = searchCockroachPRCommitsSingle(pr, nextCursor, prCommitsToExclude, token) - if err != nil { - fmt.Println(err) - return nil, err - } - result = append(result, commits...) - } - return result, nil -} - -func searchCockroachPRCommitsSingle( - prNumber int, cursor string, prCommitsToExclude map[int]map[string]int, token string, -) (bool, string, []cockroachCommit, error) { - var result []cockroachCommit - var search gqlCockroachPRCommit - query := `query ($cursor: String, $prNumber: Int!) { - repository(owner: "cockroachdb", name: "cockroach") { - pullRequest(number: $prNumber) { - commits(first: 100, after: $cursor) { - edges { - node { - commit { - oid - messageHeadline - messageBody - } - } - } - } - } - } - }` - queryVariables := map[string]interface{}{ - "prNumber": prNumber, - } - if cursor != "" { - queryVariables["cursor"] = cursor - } - err := queryGraphQL( - query, - queryVariables, - token, - &search, - ) - if err != nil { - fmt.Println(err) - return false, "", nil, err - } - for _, x := range search.Data.Repository.PullRequest.Commits.Edges { - matchingDocsIssue := prCommitsToExclude[prNumber][x.Node.Commit.Oid] - if nonBugFixRNRE.MatchString(x.Node.Commit.MessageHeadline) && matchingDocsIssue == 0 { - commit := cockroachCommit{ - Sha: x.Node.Commit.Oid, - MessageHeadline: x.Node.Commit.MessageHeadline, - MessageBody: x.Node.Commit.MessageHeadline, - } - result = append(result, commit) - } - } - pageInfo := search.Data.Repository.PullRequest.Commits.PageInfo - return pageInfo.HasNextPage, pageInfo.EndCursor, result, nil -} - -// extractStringsFromMessage takes in a commit message or PR body as well as two regular expressions. The first -// regular expression checks for a valid formatted epic or issue reference. If one is found, it searches that exact -// string for the individual issue references. The output is a map where the key is each epic or issue ref and the -// value is the count of references of that ref. -func extractStringsFromMessage( - message string, firstMatch, secondMatch *regexp.Regexp, -) map[string]int { - ids := map[string]int{} - if allMatches := firstMatch.FindAllString(message, -1); len(allMatches) > 0 { - for _, x := range allMatches { - matches := secondMatch.FindAllString(x, -1) - for _, match := range matches { - ids[match]++ - } - } - } - return ids -} - -func extractFixIssueIDs(message string) map[string]int { - return extractStringsFromMessage(message, fixIssueRefRE, githubJiraIssueRefRE) -} - -func extractInformIssueIDs(message string) map[string]int { - return extractStringsFromMessage(message, informIssueRefRE, githubJiraIssueRefRE) -} - -func extractEpicIDs(message string) map[string]int { - return extractStringsFromMessage(message, epicRefRE, jiraIssueRefRE) -} - -func containsEpicNone(message string) bool { - if allMatches := epicNoneRE.FindAllString(message, -1); len(allMatches) > 0 { - return true - } - return false -} - -func containsBugFix(message string) bool { - if allMatches := bugFixRNRE.FindAllString(message, -1); len(allMatches) > 0 { - return true - } - return false -} - -// org/repo#issue: PROJECT-NUMBER - -// getJiraIssueFromGitHubIssue takes a GitHub issue and returns the appropriate Jira key from the issue body. -// getJiraIssueFromGitHubIssue is specified as a function closure to allow for testing -// of getJiraIssueFromGitHubIssue* methods. -var getJiraIssueFromGitHubIssue = func(org, repo string, issue int, token string) (string, error) { - query := `query ($org: String!, $repo: String!, $issue: Int!) { - repository(owner: $org, name: $repo) { - issue(number: $issue) { - body + return err } } - }` - var search gqlSingleIssue - queryVariables := map[string]interface{}{ - "org": org, - "repo": repo, - "issue": issue, - } - err := queryGraphQL(query, queryVariables, token, &search) - if err != nil { - fmt.Println(err) - return "", err - } - var jiraIssue string - exalateRef := exalateJiraRefRE.FindString(search.Data.Repository.Issue.Body) - if len(exalateRef) > 0 { - jiraIssue = jiraIssuePartRE.FindString(exalateRef) } - return jiraIssue, nil + return nil } func splitBySlashOrHash(r rune) bool { return r == '/' || r == '#' } - -func getJiraIssueFromRef(ref, token string) string { - if jiraIssuePartRE.MatchString(ref) { - return ref - } else if jiraURLPartRE.MatchString(ref) { - return strings.Replace(ref, "https://cockroachlabs.atlassian.net/browse/", "", 1) - } else if ghIssueRepoPartRE.MatchString(ref) { - split := strings.FieldsFunc(ref, splitBySlashOrHash) - issueNumber, err := strconv.Atoi(split[2]) - if err != nil { - fmt.Println(err) - return "" - } - issueRef, err := getJiraIssueFromGitHubIssue(split[0], split[1], issueNumber, token) - if err != nil { - fmt.Println(err) - } - return issueRef - } else if ghIssuePartRE.MatchString(ref) { - issueNumber, err := strconv.Atoi(strings.Replace(ref, "#", "", 1)) - if err != nil { - fmt.Println(err) - return "" - } - issueRef, err := getJiraIssueFromGitHubIssue("cockroachdb", "cockroach", issueNumber, token) - if err != nil { - fmt.Println(err) - } - return issueRef - } else if ghURLPartRE.MatchString(ref) { - replace1 := strings.Replace(ref, "https://github.com/", "", 1) - replace2 := strings.Replace(replace1, "/issues", "", 1) - split := strings.FieldsFunc(replace2, splitBySlashOrHash) - issueNumber, err := strconv.Atoi(split[2]) - if err != nil { - fmt.Println(err) - return "" - } - issueRef, err := getJiraIssueFromGitHubIssue(split[0], split[1], issueNumber, token) - if err != nil { - fmt.Println(err) - } - return issueRef - } else { - return "Malformed epic/issue ref (" + ref + ")" - } -} - -func extractIssueEpicRefs(prBody, commitBody, token string) string { - refInfo := epicIssueRefInfo{ - epicRefs: extractEpicIDs(commitBody + "\n" + prBody), - epicNone: containsEpicNone(commitBody + "\n" + prBody), - issueCloseRefs: extractFixIssueIDs(commitBody + "\n" + prBody), - issueInformRefs: extractInformIssueIDs(commitBody + "\n" + prBody), - isBugFix: containsBugFix(commitBody + "\n" + prBody), - } - var builder strings.Builder - if len(refInfo.epicRefs) > 0 { - builder.WriteString("Epic:") - for x := range refInfo.epicRefs { - builder.WriteString(" " + getJiraIssueFromRef(x, token)) - } - builder.WriteString("\n") - } - if len(refInfo.issueCloseRefs) > 0 { - builder.WriteString("Fixes:") - for x := range refInfo.issueCloseRefs { - builder.WriteString(" " + getJiraIssueFromRef(x, token)) - } - builder.WriteString("\n") - } - if len(refInfo.issueInformRefs) > 0 { - builder.WriteString("Informs:") - for x := range refInfo.issueInformRefs { - builder.WriteString(" " + getJiraIssueFromRef(x, token)) - } - builder.WriteString("\n") - } - if refInfo.epicNone && builder.Len() == 0 { - builder.WriteString("Epic: none\n") - } - return builder.String() -} - -// getIssues takes a list of commits from GitHub as well as the PR number associated with those commits and outputs a -// formatted list of docs issues with valid release notes -func constructDocsIssues(prs []cockroachPR, token string) []docsIssue { - var result []docsIssue - for _, pr := range prs { - for _, commit := range pr.Commits { - rns := formatReleaseNotes(commit.MessageBody, pr.Number, pr.Body, commit.Sha, token) - for i, rn := range rns { - x := docsIssue{ - sourceCommitSha: commit.Sha, - title: formatTitle(commit.MessageHeadline, pr.Number, i+1, len(rns)), - body: rn, - labels: []string{ - "C-product-change", - pr.BaseRefName, - }, - } - result = append(result, x) - - } - } - } - return result -} - -func formatTitle(title string, prNumber int, index int, totalLength int) string { - result := fmt.Sprintf("PR #%d - %s", prNumber, title) - if totalLength > 1 { - result += fmt.Sprintf(" (%d of %d)", index, totalLength) - } - return result -} - -// formatReleaseNotes generates a list of docsIssue bodies for the docs repo based on a given CRDB sha -func formatReleaseNotes( - commitMessage string, prNumber int, prBody, crdbSha, token string, -) []string { - rnBodySlice := []string{} - if releaseNoteNoneRE.MatchString(commitMessage) { - return rnBodySlice - } - epicIssueRefs := extractIssueEpicRefs(prBody, commitMessage, token) - splitString := strings.Split(commitMessage, "\n") - releaseNoteLines := []string{} - var rnBody string - for _, x := range splitString { - validRn := allRNRE.MatchString(x) - bugFixRn := bugFixRNRE.MatchString(x) - releaseJustification := releaseJustificationRE.MatchString(x) - if len(releaseNoteLines) > 0 && (validRn || releaseJustification) { - rnBody = fmt.Sprintf( - "Related PR: https://github.com/cockroachdb/cockroach/pull/%s\n"+ - "Commit: https://github.com/cockroachdb/cockroach/commit/%s\n"+ - "%s\n---\n\n%s", - strconv.Itoa(prNumber), - crdbSha, - epicIssueRefs, - strings.Join(releaseNoteLines, "\n"), - ) - rnBodySlice = append(rnBodySlice, strings.TrimSuffix(rnBody, "\n")) - rnBody = "" - releaseNoteLines = []string{} - } - if (validRn && !bugFixRn) || (len(releaseNoteLines) > 0 && !bugFixRn && !releaseJustification) { - releaseNoteLines = append(releaseNoteLines, x) - } - } - if len(releaseNoteLines) > 0 { // commit whatever is left in the buffer to the rnBodySlice set - rnBody = fmt.Sprintf( - "Related PR: https://github.com/cockroachdb/cockroach/pull/%s\n"+ - "Commit: https://github.com/cockroachdb/cockroach/commit/%s\n"+ - "%s\n---\n\n%s", - strconv.Itoa(prNumber), - crdbSha, - epicIssueRefs, - strings.Join(releaseNoteLines, "\n"), - ) - rnBodySlice = append(rnBodySlice, strings.TrimSuffix(rnBody, "\n")) - } - if len(rnBodySlice) > 1 { - relatedProductChanges := "Related product changes: " + - "https://cockroachlabs.atlassian.net/issues/?jql=project%20%3D%20%22DOC%22%20and%20%22Doc%20Type%5BDropdown%5D" + - "%22%20%3D%20%22Product%20Change%22%20AND%20description%20~%20%22commit%2F" + - crdbSha + "%22%20ORDER%20BY%20created%20DESC\n\n---" - for i, rn := range rnBodySlice { - rnBodySlice[i] = strings.Replace(rn, "\n---", relatedProductChanges, -1) - } - } - return rnBodySlice -} - -// TODO: Redo this function - -func (di docsIssue) createDocsIssues( - token string, repoID string, labelMap map[string]string, -) error { - var output gqlCreateIssueMutation - mutation := `mutation ($repoId: ID!, $title: String!, $body: String!, $labelIds: [ID!]) { - createIssue( - input: {repositoryId: $repoId, title: $title, body: $body, labelIds: $labelIds} - ) { - clientMutationId - } - }` - var labelIds []string - for _, x := range di.labels { - labelIds = append(labelIds, labelMap[x]) - } - mutationVariables := map[string]interface{}{ - "repoId": repoID, - "title": di.title, - "body": di.body, - } - if len(labelIds) > 0 { - mutationVariables["labelIds"] = labelIds - } - err := queryGraphQL(mutation, mutationVariables, token, &output) - if err != nil { - fmt.Println(err) - return err - } - return nil -} diff --git a/pkg/cmd/docs-issue-generation/docs_issue_generation_test.go b/pkg/cmd/docs-issue-generation/docs_issue_generation_test.go index b56848553424..8c91828369e8 100644 --- a/pkg/cmd/docs-issue-generation/docs_issue_generation_test.go +++ b/pkg/cmd/docs-issue-generation/docs_issue_generation_test.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Cockroach Authors. +// Copyright 2023 The Cockroach Authors. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt. @@ -19,7 +19,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestParseDocsIssueBody(t *testing.T) { +func TestExtractPRNumberCommitFromDocsIssueBody(t *testing.T) { testCases := []struct { testName string body string @@ -42,56 +42,49 @@ func TestParseDocsIssueBody(t *testing.T) { err: fmt.Errorf("error: No PR number found in issue body"), }, }, - { - testName: "Result with markdown URLs", - body: `Exalate commented: - -Related PR: [https://github.com/cockroachdb/cockroach/pull/80670](https://github.com/cockroachdb/cockroach/pull/80670) - -Commit: [https://github.com/cockroachdb/cockroach/commit/84a3833ee30eed278de0571c8c7d9f2f5e3b8b5d](https://github.com/cockroachdb/cockroach/commit/84a3833ee30eed278de0571c8c7d9f2f5e3b8b5d) - -— Release note (enterprise change): Backups run by secondary tenants now write protected timestamp records to protect their target schema objects from garbage collection during backup execution. - -Jira Issue: DOC-3619`, - result: struct { - prNumber int - commitSha string - err error - }{ - prNumber: 80670, - commitSha: "84a3833ee30eed278de0571c8c7d9f2f5e3b8b5d", - err: nil, - }, - }, { testName: "Result with regular URLs", - body: `Related PR: https://github.com/cockroachdb/cockroach/pull/90789 -Commit: https://github.com/cockroachdb/cockroach/commit/c7ce65697535daacf2a75df245c3bae1179faeda + body: `

Related PR: https://github.com/cockroachdb/cockroach/pull/108627
+Commit: https://github.com/cockroachdb/cockroach/commit/c5219e38b61d070bceb260d2c1b6d1f4c6ff5426
+Epic: none

+ +

—

---- +

Release note (ops change): BACKUP now skips contacting the ranges for tables on which exclude_data_from_backup is set, and can thus succeed even if an excluded table is unavailable.
+Epic: none.

-Release note (ops change): The cluster setting -` + "server.web_session.auto_logout.timeout" + ` has been removed. It had -never been effective.`, +

Jira Issue: + + + + DOC-8571 + + In Review/Testing + +

`, result: struct { prNumber int commitSha string err error }{ - prNumber: 90789, - commitSha: "c7ce65697535daacf2a75df245c3bae1179faeda", + prNumber: 108627, + commitSha: "c5219e38b61d070bceb260d2c1b6d1f4c6ff5426", err: nil, }, }, } for _, tc := range testCases { t.Run(tc.testName, func(t *testing.T) { - prNumber, commitSha, err := parseDocsIssueBody(tc.body) + prNumber, commitSha, err := extractPRNumberCommitFromDocsIssueBody(tc.body) result := struct { prNumber int commitSha string err error - }{prNumber: prNumber, commitSha: commitSha, err: err} + }{ + prNumber: prNumber, + commitSha: commitSha, + err: err, + } assert.Equal(t, tc.result, result) }) } @@ -105,23 +98,112 @@ func TestConstructDocsIssues(t *testing.T) { }{ { testName: "Single PR - 91345 - Epic: none", - cockroachPRs: []cockroachPR{{ - Title: "release-22.2: clusterversion: allow forcing release binary to dev version", - Number: 91345, - Body: "Backport 1/1 commits from #90002, using the simplifications from #91344\r\n\r\n/cc @cockroachdb/release\r\n\r\n---\r\n\r\nPreviously it was impossible to start a release binary that supported up to say, 23.1, in a cluster where the cluster version was in the 'development' range (+1m). While this was somewhat intentional -- to mark a dev cluster as dev forever -- we still want the option to try to run release binaries in that cluster.\r\n\r\nThe new environment variable COCKROACH_FORCE_DEV_VERSION will cause a binary to identify itself as development and offset its version even if it is a release binary.\r\n\r\nEpic: none.\r\n\r\nRelease note (ops change): Release version binaries can now be instructed via the enviroment variable COCKROACH_FORCE_DEV_VERSION to override their cluster version support to match that of develeopment builds, which can allow a release binary to be started in a cluster that is run or has previously run a development build.\r\n\r\nRelease justification: bug fix in new functionality.", - BaseRefName: "release-22.2", - Commits: []cockroachCommit{{ - Sha: "8dc44d23bb7e0688cd435b6f7908fab615f1aa39", - MessageHeadline: "clusterversion: allow forcing release binary to dev version", - MessageBody: "Previously it was impossible to start a release binary that supported up\nto say, 23.1, in a cluster where the cluster version was in the 'development'\nrange (+1m). While this was somewhat intentional -- to mark a dev cluster as dev\nforever -- we still want the option to try to run release binaries in that cluster.\n\nThe new environment variable COCKROACH_FORCE_DEV_VERSION will cause a binary to\nidentify itself as development and offset its version even if it is a release binary.\n\nEpic: none.\n\nRelease note (ops change): Release version binaries can now be instructed via the enviroment\nvariable COCKROACH_FORCE_DEV_VERSION to override their cluster version support to match that\nof develeopment builds, which can allow a release binary to be started in a cluster that is\nrun or has previously run a development build.", - }}, - }}, - docsIssues: []docsIssue{{ - sourceCommitSha: "8dc44d23bb7e0688cd435b6f7908fab615f1aa39", - title: "PR #91345 - clusterversion: allow forcing release binary to dev version", - body: "Related PR: https://github.com/cockroachdb/cockroach/pull/91345\nCommit: https://github.com/cockroachdb/cockroach/commit/8dc44d23bb7e0688cd435b6f7908fab615f1aa39\nEpic: none\n\n---\n\nRelease note (ops change): Release version binaries can now be instructed via the enviroment\nvariable COCKROACH_FORCE_DEV_VERSION to override their cluster version support to match that\nof develeopment builds, which can allow a release binary to be started in a cluster that is\nrun or has previously run a development build.", - labels: []string{"C-product-change", "release-22.2"}, - }}, + cockroachPRs: []cockroachPR{ + { + Title: "release-22.2: clusterversion: allow forcing release binary to dev version", + Number: 91345, + Body: "Backport 1/1 commits from #90002, using the simplifications from #91344\r\n\r\n/cc @cockroachdb/release\r\n\r\n---\r\n\r\nPreviously it was impossible to start a release binary that supported up to say, 23.1, in a cluster where the cluster version was in the 'development' range (+1m). While this was somewhat intentional -- to mark a dev cluster as dev forever -- we still want the option to try to run release binaries in that cluster.\r\n\r\nThe new environment variable COCKROACH_FORCE_DEV_VERSION will cause a binary to identify itself as development and offset its version even if it is a release binary.\r\n\r\nEpic: none.\r\n\r\nRelease note (ops change): Release version binaries can now be instructed via the enviroment variable COCKROACH_FORCE_DEV_VERSION to override their cluster version support to match that of develeopment builds, which can allow a release binary to be started in a cluster that is run or has previously run a development build.\r\n\r\nRelease justification: bug fix in new functionality.", + BaseRefName: "release-22.2", + Commits: []cockroachCommit{ + { + Sha: "8dc44d23bb7e0688cd435b6f7908fab615f1aa39", + MessageHeadline: "clusterversion: allow forcing release binary to dev version", + MessageBody: "Previously it was impossible to start a release binary that supported up\nto say, 23.1, in a cluster where the cluster version was in the 'development'\nrange (+1m). While this was somewhat intentional -- to mark a dev cluster as dev\nforever -- we still want the option to try to run release binaries in that cluster.\n\nThe new environment variable COCKROACH_FORCE_DEV_VERSION will cause a binary to\nidentify itself as development and offset its version even if it is a release binary.\n\nEpic: none.\n\nRelease note (ops change): Release version binaries can now be instructed via the enviroment\nvariable COCKROACH_FORCE_DEV_VERSION to override their cluster version support to match that\nof develeopment builds, which can allow a release binary to be started in a cluster that is\nrun or has previously run a development build.", + }, + }, + }, + }, + docsIssues: []docsIssue{ + { + Fields: docsIssueFields{ + IssueType: jiraFieldId{Id: "10084"}, + Project: jiraFieldId{Id: "10047"}, + Summary: "PR #91345 - clusterversion: allow forcing release binary to dev version", + Reporter: jiraFieldId{Id: "712020:f8672db2-443f-4232-b01a-f97746f89805"}, + Description: adfRoot{ + Version: 1, + Type: "doc", + Content: []adfNode{ + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "Related PR: ", + }, + { + Type: "text", + Text: "https://github.com/cockroachdb/cockroach/pull/91345", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{ + "href": "https://github.com/cockroachdb/cockroach/pull/91345", + }, + }, + }, + }, + { + Type: "hardBreak", + Text: "", + }, + { + Type: "text", + Text: "Commit: ", + }, + { + Type: "text", + Text: "https://github.com/cockroachdb/cockroach/commit/8dc44d23bb7e0688cd435b6f7908fab615f1aa39", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{ + "href": "https://github.com/cockroachdb/cockroach/commit/8dc44d23bb7e0688cd435b6f7908fab615f1aa39", + }, + }, + }, + }, + { + Type: "hardBreak", + }, + { + Type: "text", + Text: "Epic: none", + }, + }, + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "---", + }, + }, + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "Release note (ops change): Release version binaries can now be instructed via the enviroment\nvariable COCKROACH_FORCE_DEV_VERSION to override their cluster version support to match that\nof develeopment builds, which can allow a release binary to be started in a cluster that is\nrun or has previously run a development build.", + }, + }, + }, + }, + }, + DocType: jiraFieldId{Id: "10781"}, + FixVersions: []jiraFieldId{ + { + Id: "10186", + }, + }, + EpicLink: "", + ProductChangePrNumber: "91345", + ProductChangeCommitSHA: "8dc44d23bb7e0688cd435b6f7908fab615f1aa39", + }, + }, + }, }, { testName: "Multiple PRs", @@ -131,100 +213,549 @@ func TestConstructDocsIssues(t *testing.T) { Number: 91294, Body: "Backport 1/1 commits from #88078.\r\n\r\n/cc @cockroachdb/release\r\n\r\n---\r\n\r\nUpdate filter label from \"App\" to \"Application Name\" on SQL Activity.\r\n\r\nFixes #87960\r\n\r\n\"Screen\r\n\r\n\r\nRelease note (ui change): Update filter labels from \"App\" to \"Application Name\" and from \"Username\" to \"User Name\" on SQL Activity and Insights pages.\r\n\r\n---\r\n\r\nRelease justification: small change\r\n", BaseRefName: "release-22.1", - Commits: []cockroachCommit{{ - Sha: "8d15073f329cf8d72e09977b34a3b339d1436000", - MessageHeadline: "ui: update filter labels", - MessageBody: "Update filter label from \"App\" to \"Application Name\"\nand \"Username\" to \"User Name\" on SQL Activity pages.\n\nFixes #87960\n\nRelease note (ui change): Update filter labels from\n\"App\" to \"Application Name\" and from \"Username\" to\n\"User Name\" on SQL Activity pages.", - }}, + Commits: []cockroachCommit{ + { + Sha: "8d15073f329cf8d72e09977b34a3b339d1436000", + MessageHeadline: "ui: update filter labels", + MessageBody: "Update filter label from \"App\" to \"Application Name\"\nand \"Username\" to \"User Name\" on SQL Activity pages.\n\nFixes #87960\n\nRelease note (ui change): Update filter labels from\n\"App\" to \"Application Name\" and from \"Username\" to\n\"User Name\" on SQL Activity pages.", + }, + }, }, { Title: "release-22.2.0: sql/ttl: rename num_active_ranges metrics", Number: 90381, Body: "Backport 1/1 commits from #90175.\r\n\r\n/cc @cockroachdb/release\r\n\r\nRelease justification: metrics rename that should be done in a major release\r\n\r\n---\r\n\r\nfixes https://github.com/cockroachdb/cockroach/issues/90094\r\n\r\nRelease note (ops change): These TTL metrics have been renamed: \r\njobs.row_level_ttl.range_total_duration -> jobs.row_level_ttl.span_total_duration\r\njobs.row_level_ttl.num_active_ranges -> jobs.row_level_ttl.num_active_spans\r\n", BaseRefName: "release-22.2.0", - Commits: []cockroachCommit{{ - Sha: "1829a72664f28ddfa50324c9ff5352380029560b", - MessageHeadline: "sql/ttl: rename num_active_ranges metrics", - MessageBody: "fixes https://github.com/cockroachdb/cockroach/issues/90094\n\nRelease note (ops change): These TTL metrics have been renamed:\njobs.row_level_ttl.range_total_duration -> jobs.row_level_ttl.span_total_duration\njobs.row_level_ttl.num_active_ranges -> jobs.row_level_ttl.num_active_spans", - }}, + Commits: []cockroachCommit{ + { + Sha: "1829a72664f28ddfa50324c9ff5352380029560b", + MessageHeadline: "sql/ttl: rename num_active_ranges metrics", + MessageBody: "fixes https://github.com/cockroachdb/cockroach/issues/90094\n\nRelease note (ops change): These TTL metrics have been renamed:\njobs.row_level_ttl.range_total_duration -> jobs.row_level_ttl.span_total_duration\njobs.row_level_ttl.num_active_ranges -> jobs.row_level_ttl.num_active_spans", + }, + }, }, { Title: "release-22.2: opt/props: shallow-copy props.Histogram when applying selectivity", Number: 89957, Body: "Backport 2/2 commits from #88526 on behalf of @michae2.\r\n\r\n/cc @cockroachdb/release\r\n\r\n----\r\n\r\n**opt/props: add benchmark for props.Histogram**\r\n\r\nAdd a benchmark that measures various common props.Histogram operations.\r\n\r\nRelease note: None\r\n\r\n**opt/props: shallow-copy props.Histogram when applying selectivity**\r\n\r\n`pkg/sql/opt/props.(*Histogram).copy` showed up as a heavy allocator in\r\na recent customer OOM. The only caller of this function is\r\n`Histogram.ApplySelectivity` which deep-copies the histogram before\r\nadjusting each bucket's `NumEq`, `NumRange`, and `DistinctRange` by the\r\ngiven selectivity.\r\n\r\nInstead of deep-copying the histogram, we can change `ApplySelectivity`\r\nto shallow-copy the histogram and remember the current selectivity. We\r\ncan then wait to adjust counts in each bucket by the selectivity until\r\nsomeone actually calls `ValuesCount` or `DistinctValuesCount`.\r\n\r\nThis doesn't eliminate all copying of histograms. We're still doing some\r\ncopying in `Filter` when applying a constraint to the histogram.\r\n\r\nFixes: #89941\r\n\r\nRelease note (performance improvement): The optimizer now does less\r\ncopying of histograms while planning queries, which will reduce memory\r\npressure a little.\r\n\r\n----\r\n\r\nRelease justification: Low risk, high reward change to existing functionality.", BaseRefName: "release-22.2", - Commits: []cockroachCommit{{ - Sha: "43de8ff30e3e6e1d9b2272ed4f62c543dc0a037c", - MessageHeadline: "opt/props: shallow-copy props.Histogram when applying selectivity", - MessageBody: "`pkg/sql/opt/props.(*Histogram).copy` showed up as a heavy allocator in\na recent customer OOM. The only caller of this function is\n`Histogram.ApplySelectivity` which deep-copies the histogram before\nadjusting each bucket's `NumEq`, `NumRange`, and `DistinctRange` by the\ngiven selectivity.\n\nInstead of deep-copying the histogram, we can change `ApplySelectivity`\nto shallow-copy the histogram and remember the current selectivity. We\ncan then wait to adjust counts in each bucket by the selectivity until\nsomeone actually calls `ValuesCount` or `DistinctValuesCount`.\n\nThis doesn't eliminate all copying of histograms. We're still doing some\ncopying in `Filter` when applying a constraint to the histogram.\n\nFixes: #89941\n\nRelease note (performance improvement): The optimizer now does less\ncopying of histograms while planning queries, which will reduce memory\npressure a little.", - }}, + Commits: []cockroachCommit{ + { + Sha: "43de8ff30e3e6e1d9b2272ed4f62c543dc0a037c", + MessageHeadline: "opt/props: shallow-copy props.Histogram when applying selectivity", + MessageBody: "`pkg/sql/opt/props.(*Histogram).copy` showed up as a heavy allocator in\na recent customer OOM. The only caller of this function is\n`Histogram.ApplySelectivity` which deep-copies the histogram before\nadjusting each bucket's `NumEq`, `NumRange`, and `DistinctRange` by the\ngiven selectivity.\n\nInstead of deep-copying the histogram, we can change `ApplySelectivity`\nto shallow-copy the histogram and remember the current selectivity. We\ncan then wait to adjust counts in each bucket by the selectivity until\nsomeone actually calls `ValuesCount` or `DistinctValuesCount`.\n\nThis doesn't eliminate all copying of histograms. We're still doing some\ncopying in `Filter` when applying a constraint to the histogram.\n\nFixes: #89941\n\nRelease note (performance improvement): The optimizer now does less\ncopying of histograms while planning queries, which will reduce memory\npressure a little.", + }, + }, }, }, docsIssues: []docsIssue{ { - sourceCommitSha: "8d15073f329cf8d72e09977b34a3b339d1436000", - title: "PR #91294 - ui: update filter labels", - body: "Related PR: https://github.com/cockroachdb/cockroach/pull/91294\nCommit: https://github.com/cockroachdb/cockroach/commit/8d15073f329cf8d72e09977b34a3b339d1436000\nFixes: CRDB-19614\n\n---\n\nRelease note (ui change): Update filter labels from\n\"App\" to \"Application Name\" and from \"Username\" to\n\"User Name\" on SQL Activity pages.", - labels: []string{"C-product-change", "release-22.1"}, + Fields: docsIssueFields{ + IssueType: jiraFieldId{Id: "10084"}, + Project: jiraFieldId{Id: "10047"}, + Summary: "PR #91294 - ui: update filter labels", + Reporter: jiraFieldId{Id: "712020:f8672db2-443f-4232-b01a-f97746f89805"}, + Description: adfRoot{ + Version: 1, + Type: "doc", + Content: []adfNode{ + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "Related PR: ", + }, + { + Type: "text", + Text: "https://github.com/cockroachdb/cockroach/pull/91294", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/pull/91294"}, + }, + }, + }, + {Type: "hardBreak"}, + { + Type: "text", + Text: "Commit: ", + }, + { + Type: "text", + Text: "https://github.com/cockroachdb/cockroach/commit/8d15073f329cf8d72e09977b34a3b339d1436000", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/commit/8d15073f329cf8d72e09977b34a3b339d1436000"}, + }, + }, + }, + {Type: "hardBreak"}, + { + Type: "text", + Text: "Fixes:", + }, + { + Type: "text", + Text: " ", + }, + { + Type: "text", + Text: "CRDB-19614", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://cockroachlabs.atlassian.net/browse/CRDB-19614"}, + }, + }, + }, + }, + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "---", + }, + }, + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "Release note (ui change): Update filter labels from\n\"App\" to \"Application Name\" and from \"Username\" to\n\"User Name\" on SQL Activity pages.", + }, + }, + }, + }, + }, + DocType: jiraFieldId{Id: "10781"}, + FixVersions: []jiraFieldId{ + { + Id: "10185", + }, + }, + EpicLink: "", + ProductChangePrNumber: "91294", + ProductChangeCommitSHA: "8d15073f329cf8d72e09977b34a3b339d1436000", + }, }, { - sourceCommitSha: "1829a72664f28ddfa50324c9ff5352380029560b", - title: "PR #90381 - sql/ttl: rename num_active_ranges metrics", - body: "Related PR: https://github.com/cockroachdb/cockroach/pull/90381\nCommit: https://github.com/cockroachdb/cockroach/commit/1829a72664f28ddfa50324c9ff5352380029560b\nFixes: CRDB-20636\n\n---\n\nRelease note (ops change): These TTL metrics have been renamed:\njobs.row_level_ttl.range_total_duration -> jobs.row_level_ttl.span_total_duration\njobs.row_level_ttl.num_active_ranges -> jobs.row_level_ttl.num_active_spans", - labels: []string{"C-product-change", "release-22.2.0"}, + Fields: docsIssueFields{ + IssueType: jiraFieldId{Id: "10084"}, + Project: jiraFieldId{Id: "10047"}, + Summary: "PR #90381 - sql/ttl: rename num_active_ranges metrics", + Reporter: jiraFieldId{Id: "712020:f8672db2-443f-4232-b01a-f97746f89805"}, + Description: adfRoot{ + Version: 1, + Type: "doc", + Content: []adfNode{ + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "Related PR: ", + }, + { + Type: "text", + Text: "https://github.com/cockroachdb/cockroach/pull/90381", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/pull/90381"}, + }, + }, + }, + {Type: "hardBreak"}, + { + Type: "text", + Text: "Commit: ", + }, + { + Type: "text", + Text: "https://github.com/cockroachdb/cockroach/commit/1829a72664f28ddfa50324c9ff5352380029560b", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/commit/1829a72664f28ddfa50324c9ff5352380029560b"}, + }, + }, + }, + {Type: "hardBreak"}, + { + Type: "text", + Text: "Fixes:", + }, + { + Type: "text", + Text: " ", + }, + { + Type: "text", + Text: "CRDB-20636", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://cockroachlabs.atlassian.net/browse/CRDB-20636"}, + }, + }, + }, + }, + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "---", + }, + }, + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "Release note (ops change): These TTL metrics have been renamed:\njobs.row_level_ttl.range_total_duration -> jobs.row_level_ttl.span_total_duration\njobs.row_level_ttl.num_active_ranges -> jobs.row_level_ttl.num_active_spans", + }, + }, + }, + }, + }, + + DocType: jiraFieldId{Id: "10781"}, + FixVersions: []jiraFieldId{ + { + Id: "10186", + }, + }, + EpicLink: "", + ProductChangePrNumber: "90381", + ProductChangeCommitSHA: "1829a72664f28ddfa50324c9ff5352380029560b", + }, }, { - sourceCommitSha: "43de8ff30e3e6e1d9b2272ed4f62c543dc0a037c", - title: "PR #89957 - opt/props: shallow-copy props.Histogram when applying selectivity", - body: "Related PR: https://github.com/cockroachdb/cockroach/pull/89957\nCommit: https://github.com/cockroachdb/cockroach/commit/43de8ff30e3e6e1d9b2272ed4f62c543dc0a037c\nFixes: CRDB-20505\n\n---\n\nRelease note (performance improvement): The optimizer now does less\ncopying of histograms while planning queries, which will reduce memory\npressure a little.", - labels: []string{"C-product-change", "release-22.2"}, + Fields: docsIssueFields{ + IssueType: jiraFieldId{Id: "10084"}, + Project: jiraFieldId{Id: "10047"}, + Summary: "PR #89957 - opt/props: shallow-copy props.Histogram when applying selectivity", + Reporter: jiraFieldId{Id: "712020:f8672db2-443f-4232-b01a-f97746f89805"}, + Description: adfRoot{ + Version: 1, + Type: "doc", + Content: []adfNode{ + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "Related PR: ", + }, + { + Type: "text", + Text: "https://github.com/cockroachdb/cockroach/pull/89957", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/pull/89957"}, + }, + }, + }, + {Type: "hardBreak"}, + { + Type: "text", + Text: "Commit: ", + }, + { + Type: "text", + Text: "https://github.com/cockroachdb/cockroach/commit/43de8ff30e3e6e1d9b2272ed4f62c543dc0a037c", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/commit/43de8ff30e3e6e1d9b2272ed4f62c543dc0a037c"}, + }, + }, + }, + {Type: "hardBreak"}, + { + Type: "text", + Text: "Fixes:", + }, + { + Type: "text", + Text: " ", + }, + { + Type: "text", + Text: "CRDB-20505", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://cockroachlabs.atlassian.net/browse/CRDB-20505"}, + }, + }, + }, + }, + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "---", + }, + }, + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "Release note (performance improvement): The optimizer now does less\ncopying of histograms while planning queries, which will reduce memory\npressure a little.", + }, + }, + }, + }, + }, + DocType: jiraFieldId{Id: "10781"}, + FixVersions: []jiraFieldId{ + { + Id: "10186", + }, + }, + EpicLink: "", + ProductChangePrNumber: "89957", + ProductChangeCommitSHA: "43de8ff30e3e6e1d9b2272ed4f62c543dc0a037c", + }, }, }, }, { testName: "Epic none", - cockroachPRs: []cockroachPR{{ - Title: "Epic none PR", - Number: 123456, - Body: "This fixes a thing.\n\nEpic: none\nRelease note (enterprise change): enterprise changes", - BaseRefName: "master", - Commits: []cockroachCommit{{ - Sha: "690da3e2ac3b1b7268accfb1dcbe5464e948e9d1", - MessageHeadline: "Epic none PR", - MessageBody: "Epic: none\nRelease note (enterprise change): enterprise changes", - }}, - }}, - docsIssues: []docsIssue{{ - sourceCommitSha: "690da3e2ac3b1b7268accfb1dcbe5464e948e9d1", - title: "PR #123456 - Epic none PR", - body: "Related PR: https://github.com/cockroachdb/cockroach/pull/123456\nCommit: https://github.com/cockroachdb/cockroach/commit/690da3e2ac3b1b7268accfb1dcbe5464e948e9d1\nEpic: none\n\n---\n\nRelease note (enterprise change): enterprise changes", - labels: []string{"C-product-change", "master"}, - }}, + cockroachPRs: []cockroachPR{ + { + Title: "Epic none PR", + Number: 123456, + Body: "This fixes a thing.\n\nEpic: none\nRelease note (enterprise change): enterprise changes", + BaseRefName: "master", + Commits: []cockroachCommit{ + { + Sha: "690da3e2ac3b1b7268accfb1dcbe5464e948e9d1", + MessageHeadline: "Epic none PR", + MessageBody: "Epic: none\nRelease note (enterprise change): enterprise changes", + }, + }, + }, + }, + docsIssues: []docsIssue{ + { + Fields: docsIssueFields{ + IssueType: jiraFieldId{Id: "10084"}, + Project: jiraFieldId{Id: "10047"}, + Summary: "PR #123456 - Epic none PR", + Reporter: jiraFieldId{Id: "712020:f8672db2-443f-4232-b01a-f97746f89805"}, + Description: adfRoot{ + Version: 1, + Type: "doc", + Content: []adfNode{ + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "Related PR: ", + }, + { + Type: "text", + Text: "https://github.com/cockroachdb/cockroach/pull/123456", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/pull/123456"}, + }, + }, + }, + { + Type: "hardBreak", + }, + { + Type: "text", + Text: "Commit: ", + }, + { + Type: "text", + Text: "https://github.com/cockroachdb/cockroach/commit/690da3e2ac3b1b7268accfb1dcbe5464e948e9d1", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/commit/690da3e2ac3b1b7268accfb1dcbe5464e948e9d1"}, + }, + }, + }, + { + Type: "hardBreak", + }, + { + Type: "text", + Text: "Epic: none", + }, + }, + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "---", + }, + }, + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "Release note (enterprise change): enterprise changes", + }, + }, + }, + }, + }, + DocType: jiraFieldId{Id: "10781"}, + FixVersions: []jiraFieldId{ + { + Id: "10365", + }, + }, + EpicLink: "", + ProductChangePrNumber: "123456", + ProductChangeCommitSHA: "690da3e2ac3b1b7268accfb1dcbe5464e948e9d1", + }, + }, + }, }, { testName: "Epic extraction", - cockroachPRs: []cockroachPR{{ - Title: "Epic extraction PR", - Number: 123456, - Body: "This fixes another thing.\n\nEpic: CRDB-24680\nRelease note (cli change): cli changes", - BaseRefName: "master", - Commits: []cockroachCommit{{ - Sha: "aaada3e2ac3b1b7268accfb1dcbe5464e948e9d1", - MessageHeadline: "Epic extraction PR", - MessageBody: "Epic: CRDB-24680\nRelease note (cli change): cli changes", - }}, - }}, - docsIssues: []docsIssue{{ - sourceCommitSha: "aaada3e2ac3b1b7268accfb1dcbe5464e948e9d1", - title: "PR #123456 - Epic extraction PR", - body: "Related PR: https://github.com/cockroachdb/cockroach/pull/123456\nCommit: https://github.com/cockroachdb/cockroach/commit/aaada3e2ac3b1b7268accfb1dcbe5464e948e9d1\nEpic: CRDB-24680\n\n---\n\nRelease note (cli change): cli changes", - labels: []string{"C-product-change", "master"}, - }}, + cockroachPRs: []cockroachPR{ + { + Title: "Epic extraction PR", + Number: 123456, + Body: "This fixes another thing.\n\nEpic: CRDB-31495\nRelease note (cli change): cli changes", + BaseRefName: "master", + Commits: []cockroachCommit{ + { + Sha: "aaada3e2ac3b1b7268accfb1dcbe5464e948e9d1", + MessageHeadline: "Epic extraction PR", + MessageBody: "Epic: CRDB-31495\nRelease note (cli change): cli changes", + }, + }, + }, + }, + docsIssues: []docsIssue{ + { + Fields: docsIssueFields{ + IssueType: jiraFieldId{Id: "10084"}, + Project: jiraFieldId{Id: "10047"}, + Summary: "PR #123456 - Epic extraction PR", + Reporter: jiraFieldId{Id: "712020:f8672db2-443f-4232-b01a-f97746f89805"}, + Description: adfRoot{ + Version: 1, + Type: "doc", + Content: []adfNode{ + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "Related PR: ", + }, + { + Type: "text", + Text: "https://github.com/cockroachdb/cockroach/pull/123456", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/pull/123456"}, + }, + }, + }, + {Type: "hardBreak"}, + { + Type: "text", + Text: "Commit: ", + }, + { + Type: "text", + Text: "https://github.com/cockroachdb/cockroach/commit/aaada3e2ac3b1b7268accfb1dcbe5464e948e9d1", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/commit/aaada3e2ac3b1b7268accfb1dcbe5464e948e9d1"}, + }, + }, + }, + {Type: "hardBreak"}, + { + Type: "text", + Text: "Epic:", + }, + { + Type: "text", + Text: " ", + }, + { + Type: "text", + Text: "CRDB-31495", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://cockroachlabs.atlassian.net/browse/CRDB-31495"}, + }, + }, + }, + }, + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "---", + }, + }, + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "Release note (cli change): cli changes", + }, + }, + }, + }, + }, + DocType: jiraFieldId{Id: "10781"}, + FixVersions: []jiraFieldId{ + { + Id: "10365", + }, + }, + EpicLink: "CRDB-31495", + ProductChangePrNumber: "123456", + ProductChangeCommitSHA: "aaada3e2ac3b1b7268accfb1dcbe5464e948e9d1", + }, + }, + }, }, } for _, tc := range testCases { t.Run(tc.testName, func(t *testing.T) { - defer testutils.TestingHook(&getJiraIssueFromGitHubIssue, func(org, repo string, issue int, token string) (string, error) { + defer testutils.TestingHook(&getJiraIssueFromGitHubIssue, func(org, repo string, issue int) (string, error) { // getJiraIssueFromGitHubIssue requires a network call to the GitHub GraphQL API to calculate the Jira issue given // a GitHub org/repo/issue. To help eliminate the need of a network call and minimize the chances of this test // flaking, we define a pre-built map that is used to mock the network call and allow the tests to run as expected. @@ -234,9 +765,1345 @@ func TestConstructDocsIssues(t *testing.T) { ghJiraIssueMap["cockroachdb"]["cockroach"][87960] = "CRDB-19614" ghJiraIssueMap["cockroachdb"]["cockroach"][90094] = "CRDB-20636" ghJiraIssueMap["cockroachdb"]["cockroach"][89941] = "CRDB-20505" + ghJiraIssueMap["cockroachdb"]["cockroach"][123456] = "CRDB-31495" return ghJiraIssueMap[org][repo][issue], nil })() - result := constructDocsIssues(tc.cockroachPRs, "") + defer testutils.TestingHook(&getJiraIssueCreateMeta, func() (jiraIssueCreateMeta, error) { + result := jiraIssueCreateMeta{ + Projects: []struct { + Issuetypes []struct { + Fields struct { + Issuetype struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []interface{} "json:\"operations\"" + AllowedValues []struct { + Self string "json:\"self\"" + Id string "json:\"id\"" + Description string "json:\"description\"" + IconUrl string "json:\"iconUrl\"" + Name string "json:\"name\"" + Subtask bool "json:\"subtask\"" + AvatarId int "json:\"avatarId\"" + HierarchyLevel int "json:\"hierarchyLevel\"" + } "json:\"allowedValues\"" + } "json:\"issuetype\"" + Description struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"description\"" + Project struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + AllowedValues []struct { + Self string "json:\"self\"" + Id string "json:\"id\"" + Key string "json:\"key\"" + Name string "json:\"name\"" + ProjectTypeKey string "json:\"projectTypeKey\"" + Simplified bool "json:\"simplified\"" + AvatarUrls struct { + X48 string "json:\"48x48\"" + X24 string "json:\"24x24\"" + X16 string "json:\"16x16\"" + X32 string "json:\"32x32\"" + } "json:\"avatarUrls\"" + ProjectCategory struct { + Self string "json:\"self\"" + Id string "json:\"id\"" + Description string "json:\"description\"" + Name string "json:\"name\"" + } "json:\"projectCategory\"" + } "json:\"allowedValues\"" + } "json:\"project\"" + DocType struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + AllowedValues []struct { + Self string "json:\"self\"" + Value string "json:\"value\"" + Id string "json:\"id\"" + } "json:\"allowedValues\"" + } "json:\"customfield_10175\"" + FixVersions struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Items string "json:\"items\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + AllowedValues []jiraCreateIssueMetaFixVersionsAllowedValue "json:\"allowedValues\"" + } "json:\"fixVersions\"" + EpicLink struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"customfield_10014\"" + Summary struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"summary\"" + Reporter struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + AutoCompleteUrl string "json:\"autoCompleteUrl\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"reporter\"" + ProductChangePRNumber struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"customfield_10435\"" + ProductChangeCommitSHA struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"customfield_10436\"" + } "json:\"fields\"" + } "json:\"issuetypes\"" + }{ + { + Issuetypes: []struct { + Fields struct { + Issuetype struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []interface{} "json:\"operations\"" + AllowedValues []struct { + Self string "json:\"self\"" + Id string "json:\"id\"" + Description string "json:\"description\"" + IconUrl string "json:\"iconUrl\"" + Name string "json:\"name\"" + Subtask bool "json:\"subtask\"" + AvatarId int "json:\"avatarId\"" + HierarchyLevel int "json:\"hierarchyLevel\"" + } "json:\"allowedValues\"" + } "json:\"issuetype\"" + Description struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"description\"" + Project struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + AllowedValues []struct { + Self string "json:\"self\"" + Id string "json:\"id\"" + Key string "json:\"key\"" + Name string "json:\"name\"" + ProjectTypeKey string "json:\"projectTypeKey\"" + Simplified bool "json:\"simplified\"" + AvatarUrls struct { + X48 string "json:\"48x48\"" + X24 string "json:\"24x24\"" + X16 string "json:\"16x16\"" + X32 string "json:\"32x32\"" + } "json:\"avatarUrls\"" + ProjectCategory struct { + Self string "json:\"self\"" + Id string "json:\"id\"" + Description string "json:\"description\"" + Name string "json:\"name\"" + } "json:\"projectCategory\"" + } "json:\"allowedValues\"" + } "json:\"project\"" + DocType struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + AllowedValues []struct { + Self string "json:\"self\"" + Value string "json:\"value\"" + Id string "json:\"id\"" + } "json:\"allowedValues\"" + } "json:\"customfield_10175\"" + FixVersions struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Items string "json:\"items\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + AllowedValues []jiraCreateIssueMetaFixVersionsAllowedValue "json:\"allowedValues\"" + } "json:\"fixVersions\"" + EpicLink struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"customfield_10014\"" + Summary struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"summary\"" + Reporter struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + AutoCompleteUrl string "json:\"autoCompleteUrl\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"reporter\"" + ProductChangePRNumber struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"customfield_10435\"" + ProductChangeCommitSHA struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"customfield_10436\"" + } "json:\"fields\"" + }{ + { + Fields: struct { + Issuetype struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []interface{} "json:\"operations\"" + AllowedValues []struct { + Self string "json:\"self\"" + Id string "json:\"id\"" + Description string "json:\"description\"" + IconUrl string "json:\"iconUrl\"" + Name string "json:\"name\"" + Subtask bool "json:\"subtask\"" + AvatarId int "json:\"avatarId\"" + HierarchyLevel int "json:\"hierarchyLevel\"" + } "json:\"allowedValues\"" + } "json:\"issuetype\"" + Description struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"description\"" + Project struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + AllowedValues []struct { + Self string "json:\"self\"" + Id string "json:\"id\"" + Key string "json:\"key\"" + Name string "json:\"name\"" + ProjectTypeKey string "json:\"projectTypeKey\"" + Simplified bool "json:\"simplified\"" + AvatarUrls struct { + X48 string "json:\"48x48\"" + X24 string "json:\"24x24\"" + X16 string "json:\"16x16\"" + X32 string "json:\"32x32\"" + } "json:\"avatarUrls\"" + ProjectCategory struct { + Self string "json:\"self\"" + Id string "json:\"id\"" + Description string "json:\"description\"" + Name string "json:\"name\"" + } "json:\"projectCategory\"" + } "json:\"allowedValues\"" + } "json:\"project\"" + DocType struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + AllowedValues []struct { + Self string "json:\"self\"" + Value string "json:\"value\"" + Id string "json:\"id\"" + } "json:\"allowedValues\"" + } "json:\"customfield_10175\"" + FixVersions struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Items string "json:\"items\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + AllowedValues []jiraCreateIssueMetaFixVersionsAllowedValue "json:\"allowedValues\"" + } "json:\"fixVersions\"" + EpicLink struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"customfield_10014\"" + Summary struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"summary\"" + Reporter struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + AutoCompleteUrl string "json:\"autoCompleteUrl\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"reporter\"" + ProductChangePRNumber struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"customfield_10435\"" + ProductChangeCommitSHA struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + } "json:\"customfield_10436\"" + }{ + Issuetype: struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []interface{} "json:\"operations\"" + AllowedValues []struct { + Self string "json:\"self\"" + Id string "json:\"id\"" + Description string "json:\"description\"" + IconUrl string "json:\"iconUrl\"" + Name string "json:\"name\"" + Subtask bool "json:\"subtask\"" + AvatarId int "json:\"avatarId\"" + HierarchyLevel int "json:\"hierarchyLevel\"" + } "json:\"allowedValues\"" + }{ + Required: true, + Schema: struct { + Type string "json:\"type\"" + System string "json:\"system\"" + }{ + Type: "issuetype", + System: "issuetype", + }, + Name: "Issue Type", + Key: "issuetype", + HasDefaultValue: false, + Operations: []interface{}{}, + AllowedValues: []struct { + Self string "json:\"self\"" + Id string "json:\"id\"" + Description string "json:\"description\"" + IconUrl string "json:\"iconUrl\"" + Name string "json:\"name\"" + Subtask bool "json:\"subtask\"" + AvatarId int "json:\"avatarId\"" + HierarchyLevel int "json:\"hierarchyLevel\"" + }{ + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/issuetype/10084", + Id: "10084", + Description: "", + IconUrl: "https://cockroachlabs.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10568?size=medium", + Name: "Docs", + Subtask: false, + AvatarId: 10568, + HierarchyLevel: 0, + }, + }, + }, + Description: struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + }{ + Required: false, + Schema: struct { + Type string "json:\"type\"" + System string "json:\"system\"" + }{ + Type: "string", + System: "description", + }, + Name: "Description", + Key: "description", + HasDefaultValue: false, + Operations: []string{"set"}, + }, + Project: struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + AllowedValues []struct { + Self string "json:\"self\"" + Id string "json:\"id\"" + Key string "json:\"key\"" + Name string "json:\"name\"" + ProjectTypeKey string "json:\"projectTypeKey\"" + Simplified bool "json:\"simplified\"" + AvatarUrls struct { + X48 string "json:\"48x48\"" + X24 string "json:\"24x24\"" + X16 string "json:\"16x16\"" + X32 string "json:\"32x32\"" + } "json:\"avatarUrls\"" + ProjectCategory struct { + Self string "json:\"self\"" + Id string "json:\"id\"" + Description string "json:\"description\"" + Name string "json:\"name\"" + } "json:\"projectCategory\"" + } "json:\"allowedValues\"" + }{ + Required: true, + Schema: struct { + Type string "json:\"type\"" + System string "json:\"system\"" + }{ + Type: "project", + System: "project", + }, + Name: "Project", + Key: "project", + HasDefaultValue: false, + Operations: []string{"set"}, + AllowedValues: []struct { + Self string "json:\"self\"" + Id string "json:\"id\"" + Key string "json:\"key\"" + Name string "json:\"name\"" + ProjectTypeKey string "json:\"projectTypeKey\"" + Simplified bool "json:\"simplified\"" + AvatarUrls struct { + X48 string "json:\"48x48\"" + X24 string "json:\"24x24\"" + X16 string "json:\"16x16\"" + X32 string "json:\"32x32\"" + } "json:\"avatarUrls\"" + ProjectCategory struct { + Self string "json:\"self\"" + Id string "json:\"id\"" + Description string "json:\"description\"" + Name string "json:\"name\"" + } "json:\"projectCategory\"" + }{ + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/project/10047", + Id: "10047", + Key: "DOC", + Name: "Documentation", + ProjectTypeKey: "software", + Simplified: false, + AvatarUrls: struct { + X48 string "json:\"48x48\"" + X24 string "json:\"24x24\"" + X16 string "json:\"16x16\"" + X32 string "json:\"32x32\"" + }{ + X48: "https://cockroachlabs.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10412", + X24: "https://cockroachlabs.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10412?size=small", + X16: "https://cockroachlabs.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10412?size=xsmall", + X32: "https://cockroachlabs.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10412?size=medium", + }, + ProjectCategory: struct { + Self string "json:\"self\"" + Id string "json:\"id\"" + Description string "json:\"description\"" + Name string "json:\"name\"" + }{ + Self: "https://cockroachlabs.atlassian.net/rest/api/3/projectCategory/10001", + Id: "10001", + Description: "", + Name: "Agile Delivery", + }, + }, + }, + }, + DocType: struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + AllowedValues []struct { + Self string "json:\"self\"" + Value string "json:\"value\"" + Id string "json:\"id\"" + } "json:\"allowedValues\"" + }{ + Required: false, + Schema: struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + }{ + Type: "option", + Custom: "com.atlassian.jira.plugin.system.customfieldtypes:select", + CustomId: 10175, + }, + Name: "Doc Type", + Key: "customfield_10175", + HasDefaultValue: false, + Operations: []string{"set"}, + AllowedValues: []struct { + Self string "json:\"self\"" + Value string "json:\"value\"" + Id string "json:\"id\"" + }{ + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/customFieldOption/10781", + Value: "Doc Bug", + Id: "10781", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/customFieldOption/10780", + Value: "Doc Improvement", + Id: "10780", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/customFieldOption/11164", + Value: "General admin", + Id: "11164", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/customFieldOption/10782", + Value: "Microcopy", + Id: "10782", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/customFieldOption/10779", + Value: "Product Change", + Id: "10779", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/customFieldOption/11432", + Value: "Release Notes", + Id: "11432", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/customFieldOption/11493", + Value: "Roadmap Feature", + Id: "11493", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/customFieldOption/11569", + Value: "Style", + Id: "11569", + }, + }, + }, + FixVersions: struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Items string "json:\"items\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + AllowedValues []jiraCreateIssueMetaFixVersionsAllowedValue "json:\"allowedValues\"" + }{ + Required: false, + Schema: struct { + Type string "json:\"type\"" + Items string "json:\"items\"" + System string "json:\"system\"" + }{ + Type: "array", + Items: "version", + System: "fixVersions", + }, + Name: "Fix versions", + Key: "fixVersions", + HasDefaultValue: false, + Operations: []string{ + "set", + "add", + "remove", + }, + AllowedValues: []jiraCreateIssueMetaFixVersionsAllowedValue{ + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10181", + Id: "10181", + Description: "20.1 (Spring '20)", + Name: "20.1 (Spring '20)", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "", + Overdue: false, + UserStartDate: "", + UserReleaseDate: "", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10182", + Id: "10182", + Description: "20.2 (Fall '20)", + Name: "20.2 (Fall '20)", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "", + Overdue: false, + UserStartDate: "", + UserReleaseDate: "", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10183", + Id: "10183", + Description: "21.1 (Spring '21)", + Name: "21.1 (Spring '21)", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "", + Overdue: false, + UserStartDate: "", + UserReleaseDate: "", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10184", + Id: "10184", + Description: "21.2 (Fall '21)", + Name: "21.2 (Fall '21)", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "", + Overdue: false, + UserStartDate: "", + UserReleaseDate: "", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10185", + Id: "10185", + Description: "22.1 (Spring '22)", + Name: "22.1 (Spring '22)", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "", + Overdue: false, + UserStartDate: "", + UserReleaseDate: "", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10186", + Id: "10186", + Description: "22.2 (Fall '22)", + Name: "22.2 (Fall '22)", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "", + Overdue: false, + UserStartDate: "", + UserReleaseDate: "", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10187", + Id: "10187", + Description: "Later", + Name: "Later", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "", + Overdue: false, + UserStartDate: "", + UserReleaseDate: "", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10213", + Id: "10213", + Description: "22.FEB", + Name: "22.FEB", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "2022-02-01", + ReleaseDate: "2022-02-28", + Overdue: true, + UserStartDate: "31/Jan/22", + UserReleaseDate: "27/Feb/22", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10214", + Id: "10214", + Description: "22.MAR", + Name: "22.MAR", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "2022-03-01", + ReleaseDate: "2022-03-31", + Overdue: true, + UserStartDate: "28/Feb/22", + UserReleaseDate: "30/Mar/22", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10215", + Id: "10215", + Description: "22.APR", + Name: "22.APR", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "2022-04-01", + ReleaseDate: "2022-04-30", + Overdue: true, + UserStartDate: "31/Mar/22", + UserReleaseDate: "29/Apr/22", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10216", + Id: "10216", + Description: "22.MAY", + Name: "22.MAY", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "2022-05-01", + ReleaseDate: "2022-05-31", + Overdue: true, + UserStartDate: "30/Apr/22", + UserReleaseDate: "30/May/22", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10217", + Id: "10217", + Description: "22.JUN", + Name: "22.JUN", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "2022-06-01", + ReleaseDate: "2022-06-30", + Overdue: true, + UserStartDate: "31/May/22", + UserReleaseDate: "29/Jun/22", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10218", + Id: "10218", + Description: "22.JUL", + Name: "22.JUL", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "2022-07-01", + ReleaseDate: "2022-07-31", + Overdue: true, + UserStartDate: "30/Jun/22", + UserReleaseDate: "30/Jul/22", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10219", + Id: "10219", + Description: "22.AUG", + Name: "22.AUG", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "2022-08-01", + ReleaseDate: "2022-08-31", + Overdue: true, + UserStartDate: "31/Jul/22", + UserReleaseDate: "30/Aug/22", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10220", + Id: "10220", + Description: "22.SEP", + Name: "22.SEP", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "2022-09-01", + ReleaseDate: "2022-09-30", + Overdue: true, + UserStartDate: "31/Aug/22", + UserReleaseDate: "29/Sep/22", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10221", + Id: "10221", + Description: "22.OCT", + Name: "22.OCT", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "2022-10-01", + ReleaseDate: "2022-10-31", + Overdue: true, + UserStartDate: "30/Sep/22", + UserReleaseDate: "30/Oct/22", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10222", + Id: "10222", + Description: "22.NOV", + Name: "22.NOV", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "2022-11-01", + ReleaseDate: "2022-11-30", + Overdue: true, + UserStartDate: "31/Oct/22", + UserReleaseDate: "29/Nov/22", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10223", + Id: "10223", + Description: "22.DEC", + Name: "22.DEC", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "2022-12-01", + ReleaseDate: "2022-12-31", + Overdue: true, + UserStartDate: "30/Nov/22", + UserReleaseDate: "30/Dec/22", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10237", + Id: "10237", + Description: "23.1 (Spring 23)", + Name: "23.1 (Spring 23)", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "", + Overdue: false, + UserStartDate: "", + UserReleaseDate: "", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10310", + Id: "10310", + Description: "22.DEC PCI SAQ-D", + Name: "22.DEC PCI SAQ-D", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "2022-12-30", + Overdue: true, + UserStartDate: "", + UserReleaseDate: "29/Dec/22", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10311", + Id: "10311", + Description: "22.SEP PCI Customer Enablement", + Name: "22.SEP PCI Customer Enablement", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "2022-09-30", + Overdue: true, + UserStartDate: "", + UserReleaseDate: "29/Sep/22", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10312", + Id: "10312", + Description: "22.JUL PCI SAQ-A", + Name: "22.JUL PCI SAQ-A", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "2022-07-29", + Overdue: true, + UserStartDate: "", + UserReleaseDate: "28/Jul/22", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10328", + Id: "10328", + Description: "", + Name: "23.JAN", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "", + Overdue: false, + UserStartDate: "", + UserReleaseDate: "", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10329", + Id: "10329", + Description: "", + Name: "23.FEB", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "", + Overdue: false, + UserStartDate: "", + UserReleaseDate: "", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10330", + Id: "10330", + Description: "", + Name: "23.MAR", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "", + Overdue: false, + UserStartDate: "", + UserReleaseDate: "", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10331", + Id: "10331", + Description: "", + Name: "23.APR", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "", + Overdue: false, + UserStartDate: "", + UserReleaseDate: "", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10332", + Id: "10332", + Description: "", + Name: "23.MAY", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "", + Overdue: false, + UserStartDate: "", + UserReleaseDate: "", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10333", + Id: "10333", + Description: "", + Name: "23.JUN", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "", + Overdue: false, + UserStartDate: "", + UserReleaseDate: "", + }, + { + Self: "https://cockroachlabs.atlassian.net/rest/api/3/version/10365", + Id: "10365", + Description: "", + Name: "23.2 (Fall 23)", + Archived: false, + Released: false, + ProjectId: 10047, + StartDate: "", + ReleaseDate: "", + Overdue: false, + UserStartDate: "", + UserReleaseDate: "", + }, + }, + }, + EpicLink: struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + }{ + Required: false, + Schema: struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + }{ + Type: "any", + Custom: "com.pyxis.greenhopper.jira:gh-epic-link", + CustomId: 10014, + }, + Name: "Epic Link", + Key: "customfield_10014", + HasDefaultValue: false, + Operations: []string{"set"}, + }, + Summary: struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + }{ + Required: true, + Schema: struct { + Type string "json:\"type\"" + System string "json:\"system\"" + }{ + Type: "string", + System: "summary", + }, + Name: "Summary", + Key: "summary", + HasDefaultValue: false, + Operations: []string{"set"}, + }, + Reporter: struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + System string "json:\"system\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + AutoCompleteUrl string "json:\"autoCompleteUrl\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + }{ + Required: true, + Schema: struct { + Type string "json:\"type\"" + System string "json:\"system\"" + }{ + Type: "user", + System: "reporter", + }, + Name: "Reporter", + Key: "reporter", + AutoCompleteUrl: "https://cockroachlabs.atlassian.net/rest/api/3/user/search?query=", + HasDefaultValue: true, + Operations: []string{"set"}, + }, + ProductChangePRNumber: struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + }{ + Required: false, + Schema: struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + }{ + Type: "string", + Custom: "com.atlassian.jira.plugin.system.customfieldtypes:textfield", + CustomId: 10435, + }, + Name: "Product Change PR Number", + Key: "customfield_10435", + HasDefaultValue: false, + Operations: []string{"set"}, + }, + ProductChangeCommitSHA: struct { + Required bool "json:\"required\"" + Schema struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + } "json:\"schema\"" + Name string "json:\"name\"" + Key string "json:\"key\"" + HasDefaultValue bool "json:\"hasDefaultValue\"" + Operations []string "json:\"operations\"" + }{ + Required: false, + Schema: struct { + Type string "json:\"type\"" + Custom string "json:\"custom\"" + CustomId int "json:\"customId\"" + }{ + Type: "string", + Custom: "com.atlassian.jira.plugin.system.customfieldtypes:textfield", + CustomId: 10436, + }, + Name: "Product Change Commit SHA", + Key: "customfield_10436", + HasDefaultValue: false, + Operations: []string{"set"}, + }, + }, + }, + }, + }, + }, + } + return result, nil + })() + defer testutils.TestingHook(&searchCockroachReleaseBranches, func() ([]string, error) { + result := []string{ + "release-23.1.10-rc", + "release-23.1.9-rc.FROZEN", + "release-23.1", + "release-22.2", + "release-22.2.0", + "release-22.1", + "release-21.2", + "release-21.1", + "release-20.2", + "release-20.1", + "release-19.2", + "release-19.1", + "release-2.1", + "release-2.0", + "release-1.1", + "release-1.0", + } + return result, nil + })() + defer testutils.TestingHook(&getValidEpicRef, func(issueKey string) (bool, string, error) { + var epicMap = make(map[string]struct { + IsEpic bool + EpicKey string + }) + epicMap["CRDB-31495"] = struct { + IsEpic bool + EpicKey string + }{IsEpic: true, EpicKey: "CRDB-31495"} + return epicMap[issueKey].IsEpic, epicMap[issueKey].EpicKey, nil + })() + result, _ := constructDocsIssues(tc.cockroachPRs) assert.Equal(t, tc.docsIssues, result) }) } @@ -249,7 +2116,7 @@ func TestFormatReleaseNotes(t *testing.T) { prBody string sha string commitMessage string - rns []string + rns []adfRoot }{ { prNum: "79069", @@ -271,16 +2138,85 @@ Release note (sql change): ` + "`ALTER TABLE ... INJECT STATISTICS ...`" + ` wil now docsIssue notices when the given statistics JSON includes non-existent columns, rather than resulting in an error. Any statistics in the JSON for existing columns will be injected successfully.`, - rns: []string{`Related PR: https://github.com/cockroachdb/cockroach/pull/79069 -Commit: https://github.com/cockroachdb/cockroach/commit/5ec9343b0e0a00bfd4603e55ca6533e2b77db2f9 -Informs: CRDB-8919 - ---- - -Release note (sql change): ` + "`ALTER TABLE ... INJECT STATISTICS ...`" + ` will -now docsIssue notices when the given statistics JSON includes non-existent -columns, rather than resulting in an error. Any statistics in the JSON -for existing columns will be injected successfully.`}, + rns: []adfRoot{ + { + Version: 1, + Type: "doc", + Content: []adfNode{ + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "Related PR: ", + }, + { + Type: "text", + Text: "https://github.com/cockroachdb/cockroach/pull/79069", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/pull/79069"}, + }, + }, + }, + {Type: "hardBreak"}, + { + Type: "text", + Text: "Commit: ", + }, + { + Type: "text", + Text: "https://github.com/cockroachdb/cockroach/commit/5ec9343b0e0a00bfd4603e55ca6533e2b77db2f9", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/commit/5ec9343b0e0a00bfd4603e55ca6533e2b77db2f9"}, + }, + }, + }, + {Type: "hardBreak"}, + { + Type: "text", + Text: "Informs:", + }, + { + Type: "text", + Text: " ", + }, + { + Type: "text", + Text: "CRDB-8919", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://cockroachlabs.atlassian.net/browse/CRDB-8919"}, + }, + }, + }, + }, + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "---", + }, + }, + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "Release note (sql change): `ALTER TABLE ... INJECT STATISTICS ...` will\nnow docsIssue notices when the given statistics JSON includes non-existent\ncolumns, rather than resulting in an error. Any statistics in the JSON\nfor existing columns will be injected successfully.", + }, + }, + }, + }, + }, + }, }, { prNum: "79361", @@ -295,13 +2231,83 @@ from being displayed. Release note (enterprise change): Remove the default values from the SHOW CHANGEFEED JOB output`, - rns: []string{`Related PR: https://github.com/cockroachdb/cockroach/pull/79361 -Commit: https://github.com/cockroachdb/cockroach/commit/88be04bd64283b1d77000a3f88588e603465e81b - ---- - -Release note (enterprise change): Remove the default -values from the SHOW CHANGEFEED JOB output`}, + rns: []adfRoot{ + { + Version: 1, + Type: "doc", + Content: []adfNode{ + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Content: []adfNode(nil), + Text: "Related PR: ", + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "https://github.com/cockroachdb/cockroach/pull/79361", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/pull/79361"}, + }, + }, + }, + { + Type: "hardBreak", + Content: []adfNode(nil), + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "Commit: ", + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "https://github.com/cockroachdb/cockroach/commit/88be04bd64283b1d77000a3f88588e603465e81b", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/commit/88be04bd64283b1d77000a3f88588e603465e81b"}, + }, + }, + }, + }, + Marks: []adfMark(nil), + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Content: []adfNode(nil), + Text: "---", + Marks: []adfMark(nil), + }, + }, + Marks: []adfMark(nil), + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Content: []adfNode(nil), + Text: "Release note (enterprise change): Remove the default\nvalues from the SHOW CHANGEFEED JOB output", + Marks: []adfMark(nil), + }, + }, + Marks: []adfMark(nil), + }, + }, + }, + }, }, { prNum: "78685", @@ -331,7 +2337,7 @@ contains a semi-join, such as queries in the form: columns that prefix the equality column, and 4) the prefix columns are ` + "`NOT NULL`" + " and are constrained to a set of constant values via a " + "`CHECK`" + ` constraint or an ` + "`IN`" + " condition in the filter.", - rns: []string{}, + rns: []adfRoot{}, }, { prNum: "66328", @@ -341,14 +2347,83 @@ constraint or an ` + "`IN`" + " condition in the filter.", Release note (cli change): When log entries are written to disk, the first few header lines written at the start of every new file now report the configured logging format.`, - rns: []string{`Related PR: https://github.com/cockroachdb/cockroach/pull/66328 -Commit: https://github.com/cockroachdb/cockroach/commit/0f329965acccb3e771ec1657c7def9e881dc78bb - ---- - -Release note (cli change): When log entries are written to disk, -the first few header lines written at the start of every new file -now report the configured logging format.`}, + rns: []adfRoot{ + { + Version: 1, + Type: "doc", + Content: []adfNode{ + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Content: []adfNode(nil), + Text: "Related PR: ", + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "https://github.com/cockroachdb/cockroach/pull/66328", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/pull/66328"}, + }, + }, + }, + { + Type: "hardBreak", + Content: []adfNode(nil), + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "Commit: ", + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "https://github.com/cockroachdb/cockroach/commit/0f329965acccb3e771ec1657c7def9e881dc78bb", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/commit/0f329965acccb3e771ec1657c7def9e881dc78bb"}, + }, + }, + }, + }, + Marks: []adfMark(nil), + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Content: []adfNode(nil), + Text: "---", + Marks: []adfMark(nil), + }, + }, + Marks: []adfMark(nil), + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Content: []adfNode(nil), + Text: "Release note (cli change): When log entries are written to disk,\nthe first few header lines written at the start of every new file\nnow report the configured logging format.", + Marks: []adfMark(nil), + }, + }, + Marks: []adfMark(nil), + }, + }, + }, + }, }, { prNum: "66328", @@ -358,7 +2433,7 @@ now report the configured logging format.`}, This increases troubleshootability. Release note: None`, - rns: []string{}, + rns: []adfRoot{}, }, { prNum: "104265", @@ -394,58 +2469,304 @@ formats ` + "`crdb-v1`" + ` or ` + "`crdb-v2`" + `, it becomes impossible to pro new output log files using the ` + "`cockroach debug merge-log`" + ` command from a previous version. The newest ` + "`cockroach debug merge-log`" + ` code must be used instead.`, - rns: []string{`Related PR: https://github.com/cockroachdb/cockroach/pull/104265 -Commit: https://github.com/cockroachdb/cockroach/commit/d756dec1b9d7245305ab706e68e2ec3de0e61ffc -Related product changes: https://cockroachlabs.atlassian.net/issues/?jql=project%20%3D%20%22DOC%22%20and%20%22Doc%20Type%5BDropdown%5D%22%20%3D%20%22Product%20Change%22%20AND%20description%20~%20%22commit%2Fd756dec1b9d7245305ab706e68e2ec3de0e61ffc%22%20ORDER%20BY%20created%20DESC - ---- - -Release note (cli change): The log output formats ` + "`crdb-v1`" + ` and -` + "`crdb-v2`" + ` now support the format option ` + "`timezone`" + `. When specified, -the corresponding time zone is used to produce the timestamp column. - -For example: -` + "```" + `yaml -file-defaults: - format: crdb-v2 - format-options: {timezone: america/new_york} -` + "```" + ` - -Example logging output: -` + "```" + ` -I230606 12:43:01.553407-040000 1 1@cli/start.go:575 ⋮ [n?] 4 soft memory limit of Go runtime is set to 35 GiB -^^^^^^^ indicates GMT-4 was used. -` + "```" + ` - -The timezone offset is also always included in the format if it is not -zero (e.g. for non-UTC time zones). This is necessary to ensure that -the times can be read back precisely.`, - `Related PR: https://github.com/cockroachdb/cockroach/pull/104265 -Commit: https://github.com/cockroachdb/cockroach/commit/d756dec1b9d7245305ab706e68e2ec3de0e61ffc -Related product changes: https://cockroachlabs.atlassian.net/issues/?jql=project%20%3D%20%22DOC%22%20and%20%22Doc%20Type%5BDropdown%5D%22%20%3D%20%22Product%20Change%22%20AND%20description%20~%20%22commit%2Fd756dec1b9d7245305ab706e68e2ec3de0e61ffc%22%20ORDER%20BY%20created%20DESC - ---- - -Release note (cli change): The command ` + "`cockroach debug merge-log`" + ` was -adapted to understand time zones in input files read with format -` + "`crdb-v1`" + ` or ` + "`crdb-v2`" + `.`, - `Related PR: https://github.com/cockroachdb/cockroach/pull/104265 -Commit: https://github.com/cockroachdb/cockroach/commit/d756dec1b9d7245305ab706e68e2ec3de0e61ffc -Related product changes: https://cockroachlabs.atlassian.net/issues/?jql=project%20%3D%20%22DOC%22%20and%20%22Doc%20Type%5BDropdown%5D%22%20%3D%20%22Product%20Change%22%20AND%20description%20~%20%22commit%2Fd756dec1b9d7245305ab706e68e2ec3de0e61ffc%22%20ORDER%20BY%20created%20DESC - ---- - -Release note (backward-incompatible change): When a deployment is -configured to use a time zone (new feature) for log file output using -formats ` + "`crdb-v1`" + ` or ` + "`crdb-v2`" + `, it becomes impossible to process the -new output log files using the ` + "`cockroach debug merge-log`" + ` command -from a previous version. The newest ` + "`cockroach debug merge-log`" + ` code -must be used instead.`}, + rns: []adfRoot{ + { + Version: 1, + Type: "doc", + Content: []adfNode{ + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Content: []adfNode(nil), + Text: "Related PR: ", + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "https://github.com/cockroachdb/cockroach/pull/104265", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/pull/104265"}, + }, + }, + }, + { + Type: "hardBreak", + Content: []adfNode(nil), + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "Commit: ", + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "https://github.com/cockroachdb/cockroach/commit/d756dec1b9d7245305ab706e68e2ec3de0e61ffc", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/commit/d756dec1b9d7245305ab706e68e2ec3de0e61ffc"}, + }, + }, + }, + { + Type: "hardBreak", + Content: []adfNode(nil), + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "Related product changes: ", + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "https://cockroachlabs.atlassian.net/issues/?jql=project%20%3D%20%22DOC%22%20and%20%22Doc%20Type%5BDropdown%5D%22%20%3D%20%22Product%20Change%22%20AND%20description%20~%20%22commit%2Fd756dec1b9d7245305ab706e68e2ec3de0e61ffc%22%20ORDER%20BY%20created%20DESC\n\n---", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://cockroachlabs.atlassian.net/issues/?jql=project%20%3D%20%22DOC%22%20and%20%22Doc%20Type%5BDropdown%5D%22%20%3D%20%22Product%20Change%22%20AND%20description%20~%20%22commit%2Fd756dec1b9d7245305ab706e68e2ec3de0e61ffc%22%20ORDER%20BY%20created%20DESC\n\n---"}, + }, + }, + }, + }, + Marks: []adfMark(nil), + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Content: []adfNode(nil), + Text: "---", + Marks: []adfMark(nil), + }, + }, + Marks: []adfMark(nil), + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Content: []adfNode(nil), + Text: "Release note (cli change): The log output formats `crdb-v1` and\n`crdb-v2` now support the format option `timezone`. When specified,\nthe corresponding time zone is used to produce the timestamp column.\n\nFor example:\n```yaml\nfile-defaults:\n\tformat: crdb-v2\n\tformat-options: {timezone: america/new_york}\n```\n\nExample logging output:\n```\nI230606 12:43:01.553407-040000 1 1@cli/start.go:575 ⋮ [n?] 4 soft memory limit of Go runtime is set to 35 GiB\n^^^^^^^ indicates GMT-4 was used.\n```\n\nThe timezone offset is also always included in the format if it is not\nzero (e.g. for non-UTC time zones). This is necessary to ensure that\nthe times can be read back precisely.", + Marks: []adfMark(nil), + }, + }, + Marks: []adfMark(nil), + }, + }, + }, + { + Version: 1, + Type: "doc", + Content: []adfNode{ + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Content: []adfNode(nil), + Text: "Related PR: ", + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "https://github.com/cockroachdb/cockroach/pull/104265", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/pull/104265"}, + }, + }, + }, + { + Type: "hardBreak", + Content: []adfNode(nil), + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "Commit: ", + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "https://github.com/cockroachdb/cockroach/commit/d756dec1b9d7245305ab706e68e2ec3de0e61ffc", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/commit/d756dec1b9d7245305ab706e68e2ec3de0e61ffc"}, + }, + }, + }, + { + Type: "hardBreak", + Content: []adfNode(nil), + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "Related product changes: ", + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "https://cockroachlabs.atlassian.net/issues/?jql=project%20%3D%20%22DOC%22%20and%20%22Doc%20Type%5BDropdown%5D%22%20%3D%20%22Product%20Change%22%20AND%20description%20~%20%22commit%2Fd756dec1b9d7245305ab706e68e2ec3de0e61ffc%22%20ORDER%20BY%20created%20DESC\n\n---", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://cockroachlabs.atlassian.net/issues/?jql=project%20%3D%20%22DOC%22%20and%20%22Doc%20Type%5BDropdown%5D%22%20%3D%20%22Product%20Change%22%20AND%20description%20~%20%22commit%2Fd756dec1b9d7245305ab706e68e2ec3de0e61ffc%22%20ORDER%20BY%20created%20DESC\n\n---"}, + }, + }, + }, + }, + Marks: []adfMark(nil), + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Content: []adfNode(nil), + Text: "---", + Marks: []adfMark(nil), + }, + }, + Marks: []adfMark(nil), + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Content: []adfNode(nil), + Text: "Release note (cli change): The command `cockroach debug merge-log` was\nadapted to understand time zones in input files read with format\n`crdb-v1` or `crdb-v2`.", + Marks: []adfMark(nil), + }, + }, + Marks: []adfMark(nil), + }, + }, + }, + { + Version: 1, + Type: "doc", + Content: []adfNode{ + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Content: []adfNode(nil), + Text: "Related PR: ", + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "https://github.com/cockroachdb/cockroach/pull/104265", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/pull/104265"}, + }, + }, + }, + { + Type: "hardBreak", + Content: []adfNode(nil), + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "Commit: ", + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "https://github.com/cockroachdb/cockroach/commit/d756dec1b9d7245305ab706e68e2ec3de0e61ffc", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://github.com/cockroachdb/cockroach/commit/d756dec1b9d7245305ab706e68e2ec3de0e61ffc"}, + }, + }, + }, + { + Type: "hardBreak", + Content: []adfNode(nil), + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "Related product changes: ", + Marks: []adfMark(nil), + }, + { + Type: "text", + Content: []adfNode(nil), + Text: "https://cockroachlabs.atlassian.net/issues/?jql=project%20%3D%20%22DOC%22%20and%20%22Doc%20Type%5BDropdown%5D%22%20%3D%20%22Product%20Change%22%20AND%20description%20~%20%22commit%2Fd756dec1b9d7245305ab706e68e2ec3de0e61ffc%22%20ORDER%20BY%20created%20DESC\n\n---", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{"href": "https://cockroachlabs.atlassian.net/issues/?jql=project%20%3D%20%22DOC%22%20and%20%22Doc%20Type%5BDropdown%5D%22%20%3D%20%22Product%20Change%22%20AND%20description%20~%20%22commit%2Fd756dec1b9d7245305ab706e68e2ec3de0e61ffc%22%20ORDER%20BY%20created%20DESC\n\n---"}, + }, + }, + }, + }, + Marks: []adfMark(nil), + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Content: []adfNode(nil), + Text: "---", + Marks: []adfMark(nil), + }, + }, + Marks: []adfMark(nil), + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Content: []adfNode(nil), + Text: "Release note (backward-incompatible change): When a deployment is\nconfigured to use a time zone (new feature) for log file output using\nformats `crdb-v1` or `crdb-v2`, it becomes impossible to process the\nnew output log files using the `cockroach debug merge-log` command\nfrom a previous version. The newest `cockroach debug merge-log` code\nmust be used instead.", + Marks: []adfMark(nil), + }, + }, + Marks: []adfMark(nil), + }, + }, + }, + }, }, } for _, tc := range testCases { t.Run(tc.prNum, func(t *testing.T) { - defer testutils.TestingHook(&getJiraIssueFromGitHubIssue, func(org, repo string, issue int, token string) (string, error) { + defer testutils.TestingHook(&getJiraIssueFromGitHubIssue, func(org, repo string, issue int) (string, error) { var ghJiraIssueMap = make(map[string]map[string]map[int]string) ghJiraIssueMap["cockroachdb"] = make(map[string]map[int]string) ghJiraIssueMap["cockroachdb"]["cockroach"] = make(map[int]string) @@ -453,7 +2774,7 @@ must be used instead.`}, return ghJiraIssueMap[org][repo][issue], nil })() prNumInt, _ := strconv.Atoi(tc.prNum) - result := formatReleaseNotes(tc.commitMessage, prNumInt, tc.prBody, tc.sha, "") + result, _ := formatReleaseNotes(tc.commitMessage, prNumInt, tc.prBody, tc.sha) assert.Equal(t, tc.rns, result) }) } @@ -545,7 +2866,13 @@ close #75201 closed #592 RESOLVE #5555 Release Notes: None`, - expected: map[string]int{"#75200": 1, "#98922": 1, "#75201": 1, "#592": 1, "#5555": 1}, + expected: map[string]int{ + "#75200": 1, + "#98922": 1, + "#75201": 1, + "#592": 1, + "#5555": 1, + }, }, { message: `logictestccl: skip flaky TestCCLLogic/fakedist-metadata/partitioning_enum @@ -566,7 +2893,17 @@ Fixed: #29833 example/repo#941 see also: #9243 informs: #912, #4729 #2911 cockroachdb/cockroach#2934 Release note (sql change): Import now checks readability...`, - expected: map[string]int{"#74932": 1, "#74889": 1, "#74482": 1, "#74784": 1, "#65117": 1, "#79299": 1, "#73834": 1, "#29833": 1, "example/repo#941": 1}, + expected: map[string]int{ + "#74932": 1, + "#74889": 1, + "#74482": 1, + "#74784": 1, + "#65117": 1, + "#79299": 1, + "#73834": 1, + "#29833": 1, + "example/repo#941": 1, + }, }, { message: `lots of variations 2 @@ -581,7 +2918,16 @@ Fixes #65200. The last remaining 21.1 version (V21_1) can be removed as Release note (build change): Upgrade to Go 1.17.6 Release note (ops change): Added a metric Release notes (enterprise change): Client certificates may...`, - expected: map[string]int{"#4921": 1, "#72829": 1, "#71901": 1, "#491": 1, "#71614": 1, "#71607": 1, "#69765": 1, "#65200": 1}, + expected: map[string]int{ + "#4921": 1, + "#72829": 1, + "#71901": 1, + "#491": 1, + "#71614": 1, + "#71607": 1, + "#69765": 1, + "#65200": 1, + }, }, { message: `testing JIRA tickets @@ -591,7 +2937,14 @@ This fixes everything. Closes CC-1234. Resolves crdb-23456, Resolves DEVINFHD-12345 Fixes #12345 Release notes (sql change): Excellent sql change...`, - expected: map[string]int{"doc-4321": 1, "CC-1234": 1, "CRDB-12345": 1, "crdb-23456": 1, "DEVINFHD-12345": 1, "#12345": 1}, + expected: map[string]int{ + "doc-4321": 1, + "CC-1234": 1, + "CRDB-12345": 1, + "crdb-23456": 1, + "DEVINFHD-12345": 1, + "#12345": 1, + }, }, { message: `testing URL refs @@ -627,7 +2980,10 @@ Part of #45791 Epic CRDB-491 Fix: #73834 Release note (bug fix): Fixin the bug`, - expected: map[string]int{"#75227": 1, "#45791": 1}, + expected: map[string]int{ + "#75227": 1, + "#45791": 1, + }, }, { message: `lots of variations @@ -637,7 +2993,15 @@ informs: #912, #4729 #2911 cockroachdb/cockroach#2934 Informs #69765 (point 4). This informs #59293 with these additions: Release note (sql change): Import now checks readability...`, - expected: map[string]int{"#9243": 1, "#912": 1, "#4729": 1, "#2911": 1, "cockroachdb/cockroach#2934": 1, "#69765": 1, "#59293": 1}, + expected: map[string]int{ + "#9243": 1, + "#912": 1, + "#4729": 1, + "#2911": 1, + "cockroachdb/cockroach#2934": 1, + "#69765": 1, + "#59293": 1, + }, }, { message: `testing JIRA keys with varying cases @@ -645,7 +3009,13 @@ Fixed: CRDB-12345 example/repo#941 informs: doc-1234, crdb-24680 Informs DEVINF-123, #69765 and part of DEVINF-3891 Release note (sql change): Something something something...`, - expected: map[string]int{"doc-1234": 1, "DEVINF-123": 1, "#69765": 1, "crdb-24680": 1, "DEVINF-3891": 1}, + expected: map[string]int{ + "doc-1234": 1, + "DEVINF-123": 1, + "#69765": 1, + "crdb-24680": 1, + "DEVINF-3891": 1, + }, }, { message: `testing URL refs @@ -685,21 +3055,45 @@ Release note (bug fix): Fixin the bug`, { message: `lots of variations epic: CRDB-9234. -epic CRDB-235, CRDB-40192; DEVINF-392 https://cockroachlabs.atlassian.net/browse/CRDB-97531 -Epic doc-9238 +epic CRDB-235, DOC-6883; DEVINF-392 https://cockroachlabs.atlassian.net/browse/CRDB-28708 Release note (sql change): Import now checks readability...`, - expected: map[string]int{ - "CRDB-9234": 1, - "CRDB-235": 1, - "CRDB-40192": 1, - "DEVINF-392": 1, - "doc-9238": 1, - "https://cockroachlabs.atlassian.net/browse/CRDB-97531": 1, - }, + expected: map[string]int{"CRDB-18955": 1, "CRDB-235": 1, "DOC-6883": 1}, }, } for _, tc := range testCases { + defer testutils.TestingHook(&getValidEpicRef, func(issueKey string) (bool, string, error) { + var epicMap = map[string]struct { + IsEpic bool + EpicKey string + }{ + "CRDB-491": { + IsEpic: true, + EpicKey: "CRDB-491", + }, + "CRDB-9234": { + IsEpic: false, + EpicKey: "", + }, + "CRDB-235": { + IsEpic: true, + EpicKey: "CRDB-235", + }, + "DOC-6883": { + IsEpic: true, + EpicKey: "DOC-6883", + }, + "DEVINF-392": { + IsEpic: false, + EpicKey: "", + }, + "https://cockroachlabs.atlassian.net/browse/CRDB-28708": { + IsEpic: false, + EpicKey: "CRDB-18955", + }, + } + return epicMap[issueKey].IsEpic, epicMap[issueKey].EpicKey, nil + })() t.Run(tc.message, func(t *testing.T) { result := extractEpicIDs(tc.message) assert.Equal(t, tc.expected, result) diff --git a/pkg/cmd/docs-issue-generation/extract.go b/pkg/cmd/docs-issue-generation/extract.go new file mode 100644 index 000000000000..cb983358abe1 --- /dev/null +++ b/pkg/cmd/docs-issue-generation/extract.go @@ -0,0 +1,231 @@ +// Copyright 2023 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +import ( + "fmt" + "regexp" + "strconv" + "strings" +) + +// Regex components for finding and validating issue and epic references in a string +var ( + jiraBrowseUrlPart = crlJiraBaseUrl + "browse/" + ghIssuePart = `(#\d+)` // e.g., #12345 + ghIssueRepoPart = `([\w.-]+[/][\w.-]+#\d+)` // e.g., cockroachdb/cockroach#12345 + ghURLPart = `(https://github.com/[-a-z0-9]+/[-._a-z0-9/]+/issues/\d+)` // e.g., https://github.com/cockroachdb/cockroach/issues/12345 + jiraURLPart = jiraBrowseUrlPart + jiraIssuePart // e.g., https://cockroachlabs.atlassian.net/browse/DOC-3456 + afterRefPart = `[,.;]?(?:[ \t\n\r]+|$)` + jiraIssuePart = `([[:alpha:]]+-\d+)` // e.g., DOC-3456 + ghIssuePartRE = regexp.MustCompile(ghIssuePart) + ghIssueRepoPartRE = regexp.MustCompile(ghIssueRepoPart) + ghURLPartRE = regexp.MustCompile(ghURLPart) + jiraURLPartRE = regexp.MustCompile(jiraURLPart) + jiraIssuePartRE = regexp.MustCompile(jiraIssuePart) + issueRefPart = ghIssuePart + "|" + ghIssueRepoPart + "|" + ghURLPart + "|" + jiraIssuePart + "|" + jiraURLPart + fixIssueRefRE = regexp.MustCompile(`(?im)(?i:close[sd]?|fix(?:e[sd])?|resolve[sd]?):?\s+(?:(?:` + issueRefPart + `)` + afterRefPart + ")+") + informIssueRefRE = regexp.MustCompile(`(?im)(?:part of|see also|informs):?\s+(?:(?:` + issueRefPart + `)` + afterRefPart + ")+") + epicRefRE = regexp.MustCompile(`(?im)epic:?\s+(?:(?:` + jiraIssuePart + "|" + jiraURLPart + `)` + afterRefPart + ")+") + epicNoneRE = regexp.MustCompile(`(?im)epic:?\s+(?:(none)` + afterRefPart + ")+") + githubJiraIssueRefRE = regexp.MustCompile(issueRefPart) + jiraIssueRefRE = regexp.MustCompile(jiraIssuePart + "|" + jiraURLPart) + prNumberHTMLRE = regexp.MustCompile(`Related PR: 0 { + for _, x := range allMatches { + matches := secondMatch.FindAllString(x, -1) + for _, match := range matches { + ids[match]++ + } + } + } + return ids +} + +func extractFixIssueIDs(message string) map[string]int { + return extractStringsFromMessage(message, fixIssueRefRE, githubJiraIssueRefRE) +} + +func extractInformIssueIDs(message string) map[string]int { + return extractStringsFromMessage(message, informIssueRefRE, githubJiraIssueRefRE) +} + +func extractEpicIDs(message string) map[string]int { + result := extractStringsFromMessage(message, epicRefRE, jiraIssueRefRE) + for issueKey, count := range result { + var isEpic bool + epicKey, ok := invalidEpicRefs[issueKey] + if ok { + isEpic = epicKey == issueKey + } else { + var err error + isEpic, epicKey, err = getValidEpicRef(issueKey) + if err != nil { + // if the supplied issueKey is bad or there's a problem with the Jira REST API, simply print out + // the error message, but don't return it. Instead, remove the epic from the list, since we were + // unable to validate whether it was an epic, and we strictly need a valid epic key. + fmt.Printf("error: Unable to determine whether %s is a valid epic. Caused by:\n%s\n", issueKey, err) + delete(result, issueKey) + continue + } + if epicKey != "" { + invalidEpicRefs[issueKey] = epicKey + } + } + if isEpic { + continue + } else if issueKey != epicKey { + if epicKey != "" { + result[epicKey] = count + } + delete(result, issueKey) + } + } + return result +} + +func containsEpicNone(message string) bool { + return epicNoneRE.MatchString(message) +} + +func extractIssueEpicRefs(prBody, commitBody string) ([]adfNode, error) { + epicRefs := extractEpicIDs(commitBody + "\n" + prBody) + refInfo := epicIssueRefInfo{ + epicRefs: epicRefs, + epicNone: containsEpicNone(commitBody + "\n" + prBody), + issueCloseRefs: extractFixIssueIDs(commitBody + "\n" + prBody), + issueInformRefs: extractInformIssueIDs(commitBody + "\n" + prBody), + } + var result []adfNode + makeAdfNode := func(nodeType string, nodeText string, isLink bool) adfNode { + result := adfNode{ + Type: nodeType, + Text: nodeText, + } + if isLink { + result.Marks = []adfMark{ + { + Type: "link", + Attrs: map[string]string{ + "href": jiraBrowseUrlPart + nodeText, + }, + }, + } + } + return result + } + handleRefs := func(refs map[string]int, refPrefix string) error { + if len(refs) > 0 { + result = append(result, makeAdfNode("hardBreak", "", false)) + result = append(result, makeAdfNode("text", refPrefix, false)) + for x := range refs { + ref, err := getJiraIssueFromRef(x) + if err != nil { + return err + } + result = append(result, makeAdfNode("text", " ", false)) + result = append(result, makeAdfNode("text", ref, true)) + } + } + return nil + } + err := handleRefs(refInfo.epicRefs, "Epic:") + if err != nil { + return []adfNode{}, err + } + err = handleRefs(refInfo.issueCloseRefs, "Fixes:") + if err != nil { + return []adfNode{}, err + } + err = handleRefs(refInfo.issueInformRefs, "Informs:") + if err != nil { + return []adfNode{}, err + } + if refInfo.epicNone && len(result) == 0 { + result = append(result, makeAdfNode("hardBreak", "", false)) + result = append(result, makeAdfNode("text", "Epic: none", false)) + } + return result, nil +} + +func getJiraIssueFromRef(ref string) (string, error) { + if jiraIssuePartRE.MatchString(ref) { + return ref, nil + } else if jiraURLPartRE.MatchString(ref) { + return strings.Replace(ref, jiraBrowseUrlPart, "", 1), nil + } else if ghIssueRepoPartRE.MatchString(ref) { + split := strings.FieldsFunc(ref, splitBySlashOrHash) + issueNumber, err := strconv.Atoi(split[2]) + if err != nil { + return "", err + } + issueRef, err := getJiraIssueFromGitHubIssue(split[0], split[1], issueNumber) + if err != nil { + return "", err + } + return issueRef, nil + } else if ghIssuePartRE.MatchString(ref) { + issueNumber, err := strconv.Atoi(strings.Replace(ref, "#", "", 1)) + if err != nil { + return "", err + } + issueRef, err := getJiraIssueFromGitHubIssue("cockroachdb", "cockroach", issueNumber) + if err != nil { + return "", err + } + return issueRef, nil + } else if ghURLPartRE.MatchString(ref) { + replace1 := strings.Replace(ref, "https://github.com/", "", 1) + replace2 := strings.Replace(replace1, "/issues", "", 1) + split := strings.FieldsFunc(replace2, splitBySlashOrHash) + issueNumber, err := strconv.Atoi(split[2]) + if err != nil { + return "", err + } + issueRef, err := getJiraIssueFromGitHubIssue(split[0], split[1], issueNumber) + if err != nil { + return "", err + } + return issueRef, nil + } else { + return "", fmt.Errorf("error: Malformed epic/issue ref (%s)", ref) + } +} diff --git a/pkg/cmd/docs-issue-generation/format.go b/pkg/cmd/docs-issue-generation/format.go new file mode 100644 index 000000000000..b6473c7d5ba3 --- /dev/null +++ b/pkg/cmd/docs-issue-generation/format.go @@ -0,0 +1,276 @@ +// Copyright 2023 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +import ( + "fmt" + "regexp" + "sort" + "strconv" + "strings" +) + +const ( + jiraDocsUserId = "712020:f8672db2-443f-4232-b01a-f97746f89805" +) + +var ( + releaseNoteNoneRE = regexp.MustCompile(`(?i)release note:? [nN]one`) + allRNRE = regexp.MustCompile(`(?i)release note:? \(.*`) + bugFixRNRE = regexp.MustCompile(`(?i)release note:? \(bug fix\):.*`) + releaseJustificationRE = regexp.MustCompile(`(?i)release justification:.*`) +) + +// constructDocsIssues takes a list of commits from GitHub as well as the PR number associated with those commits and +// outputs a formatted list of docs issues with valid release notes +func constructDocsIssues(prs []cockroachPR) ([]docsIssue, error) { + jiraIssueMeta, err := getJiraIssueCreateMeta() + if err != nil { + return []docsIssue{}, err + } + fixVersionMap, err := generateFixVersionMap(jiraIssueMeta.Projects[0].Issuetypes[0].Fields.FixVersions.AllowedValues) + if err != nil { + return []docsIssue{}, err + } + var result []docsIssue + for _, pr := range prs { + for _, commit := range pr.Commits { + rns, err := formatReleaseNotes(commit.MessageBody, pr.Number, pr.Body, commit.Sha) + if err != nil { + return []docsIssue{}, err + } + epicRefs := extractEpicIDs(commit.MessageBody + "\n" + pr.Body) + var epicRef string + for k := range epicRefs { + epicRef = k + break + } + for i, rn := range rns { + x := docsIssue{ + Fields: docsIssueFields{ + IssueType: jiraFieldId{ + Id: jiraIssueMeta.Projects[0].Issuetypes[0].Fields.Issuetype.AllowedValues[0].Id, + }, + Project: jiraFieldId{ + Id: jiraIssueMeta.Projects[0].Issuetypes[0].Fields.Project.AllowedValues[0].Id, + }, + Summary: formatTitle(commit.MessageHeadline, pr.Number, i+1, len(rns)), + Reporter: jiraFieldId{ + Id: jiraDocsUserId, + }, + Description: rn, + EpicLink: epicRef, + DocType: jiraFieldId{ + Id: jiraIssueMeta.Projects[0].Issuetypes[0].Fields.DocType.AllowedValues[0].Id, + }, + ProductChangeCommitSHA: commit.Sha, + ProductChangePrNumber: strconv.Itoa(pr.Number), + }, + } + _, ok := fixVersionMap[pr.BaseRefName] + if ok { + x.Fields.FixVersions = []jiraFieldId{ + { + Id: fixVersionMap[pr.BaseRefName], + }, + } + } + result = append(result, x) + } + } + } + return result, nil +} + +// generateFixVersionMap returns a mapping of release branches to the object ID in Jira of the matching fixVersions value +func generateFixVersionMap( + fixVersions []jiraCreateIssueMetaFixVersionsAllowedValue, +) (map[string]string, error) { + result := make(map[string]string) + labelNameRe := regexp.MustCompile(`^\d{2}\.\d \(`) + branchVersionRe := regexp.MustCompile(`release-(\d+\.\d+)`) + labelToLabelId := make(map[string]string) + for _, x := range fixVersions { + if labelNameRe.MatchString(x.Name) { + labelToLabelId[x.Name] = x.Id + } + } + labelKeys := make([]string, 0, len(labelToLabelId)) + for key := range labelToLabelId { + labelKeys = append(labelKeys, key) + } + sort.Strings(labelKeys) + releaseBranches, err := searchCockroachReleaseBranches() + if err != nil { + return nil, err + } + result["master"] = labelToLabelId[labelKeys[len(labelKeys)-1]] + for _, branch := range releaseBranches { + branchMatches := branchVersionRe.FindStringSubmatch(branch) + if len(branchMatches) >= 2 { + for _, label := range labelKeys { + if strings.Contains(label, branchMatches[1]) { + result[branch] = labelToLabelId[label] + //break + } + } + } + } + return result, nil +} + +func formatTitle(title string, prNumber int, index int, totalLength int) string { + result := fmt.Sprintf("PR #%d - %s", prNumber, title) + if totalLength > 1 { + result += fmt.Sprintf(" (%d of %d)", index, totalLength) + } + return result +} + +// formatReleaseNotes generates a list of docsIssue bodies for the docs repo based on a given CRDB sha +func formatReleaseNotes( + commitMessage string, prNumber int, prBody, crdbSha string, +) ([]adfRoot, error) { + rnBodySlice := []adfRoot{} + if releaseNoteNoneRE.MatchString(commitMessage) { + return rnBodySlice, nil + } + epicIssueRefs, err := extractIssueEpicRefs(prBody, commitMessage) + if err != nil { + return []adfRoot{}, err + } + splitString := strings.Split(commitMessage, "\n") + releaseNoteLines := []string{} + var rnBody adfRoot + for _, x := range splitString { + validRn := allRNRE.MatchString(x) + bugFixRn := bugFixRNRE.MatchString(x) + releaseJustification := releaseJustificationRE.MatchString(x) + if len(releaseNoteLines) > 0 && (validRn || releaseJustification) { + formattedRNText := strings.TrimSuffix(strings.Join(releaseNoteLines, "\n"), "\n") + rnBody = formatReleaseNotesSingle(prNumber, crdbSha, formattedRNText, epicIssueRefs) + rnBodySlice = append(rnBodySlice, rnBody) + releaseNoteLines = []string{} + } + if (validRn && !bugFixRn) || (len(releaseNoteLines) > 0 && !bugFixRn && !releaseJustification) { + releaseNoteLines = append(releaseNoteLines, x) + } + } + if len(releaseNoteLines) > 0 { // commit whatever is left in the buffer to the rnBodySlice set + formattedRNText := strings.TrimSuffix(strings.Join(releaseNoteLines, "\n"), "\n") + rnBody = formatReleaseNotesSingle(prNumber, crdbSha, formattedRNText, epicIssueRefs) + rnBodySlice = append(rnBodySlice, rnBody) + } + if len(rnBodySlice) > 1 { + relatedProductChanges := []adfNode{ + { + Type: "hardBreak", + }, + { + Type: "text", + Text: "Related product changes: ", + }, + { + Type: "text", + Text: crlJiraBaseUrl + "issues/?jql=project%20%3D%20%22DOC%22%20and%20%22Doc%20Type%5BDropdown%5D" + + "%22%20%3D%20%22Product%20Change%22%20AND%20description%20~%20%22commit%2F" + + crdbSha + "%22%20ORDER%20BY%20created%20DESC\n\n---", + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{ + "href": crlJiraBaseUrl + "issues/?jql=project%20%3D%20%22DOC%22%20and%20%22Doc%20Type%5BDropdown%5D" + + "%22%20%3D%20%22Product%20Change%22%20AND%20description%20~%20%22commit%2F" + + crdbSha + "%22%20ORDER%20BY%20created%20DESC\n\n---", + }, + }, + }, + }, + } + for _, rnBody := range rnBodySlice { + rnBody.Content[0].Content = append(rnBody.Content[0].Content, relatedProductChanges...) + } + } + return rnBodySlice, nil +} + +func formatReleaseNotesSingle( + prNumber int, crdbSha, releaseNoteBody string, epicIssueRefs []adfNode, +) adfRoot { + result := adfRoot{ + Version: 1, + Type: "doc", + Content: []adfNode{ + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "Related PR: ", + }, + { + Type: "text", + Text: fmt.Sprintf("https://github.com/cockroachdb/cockroach/pull/%d", prNumber), + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{ + "href": fmt.Sprintf("https://github.com/cockroachdb/cockroach/pull/%d", prNumber), + }, + }, + }, + }, + { + Type: "hardBreak", + }, + { + Type: "text", + Text: "Commit: ", + }, + { + Type: "text", + Text: fmt.Sprintf("https://github.com/cockroachdb/cockroach/commit/%s", crdbSha), + Marks: []adfMark{ + { + Type: "link", + Attrs: map[string]string{ + "href": fmt.Sprintf("https://github.com/cockroachdb/cockroach/commit/%s", crdbSha), + }, + }, + }, + }, + }, + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: "---", + }, + }, + }, + { + Type: "paragraph", + Content: []adfNode{ + { + Type: "text", + Text: releaseNoteBody, // TODO: Find a way to convert the Markdown contents in this variable to ADF + }, + }, + }, + }, + } + if len(epicIssueRefs) > 0 { + result.Content[0].Content = append(result.Content[0].Content, epicIssueRefs...) + } + return result +} diff --git a/pkg/cmd/docs-issue-generation/github.go b/pkg/cmd/docs-issue-generation/github.go new file mode 100644 index 000000000000..c2357e6ca7e7 --- /dev/null +++ b/pkg/cmd/docs-issue-generation/github.go @@ -0,0 +1,291 @@ +// Copyright 2023 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +import ( + "fmt" + "regexp" + "strings" + "time" +) + +var ( + exalateJiraRefPart = `Jira issue: ` + jiraIssuePart // e.g., Jira issue: CRDB-54321 + exalateJiraRefRE = regexp.MustCompile(exalateJiraRefPart) + nonBugFixRNRE = regexp.MustCompile(`(?i)release note:? \(([^b]|b[^u]|bu[^g]|bug\S|bug [^f]|bug f[^i]|bug fi[^x]).*`) +) + +func searchCockroachPRs(startTime time.Time, endTime time.Time) ([]cockroachPR, error) { + prCommitsToExclude, err := searchJiraDocsIssues(startTime) + if err != nil { + return nil, err + } + hasNextPage, nextCursor, prs, err := searchCockroachPRsSingle(startTime, endTime, "", prCommitsToExclude) + if err != nil { + return nil, err + } + result := prs + for hasNextPage { + hasNextPage, nextCursor, prs, err = searchCockroachPRsSingle(startTime, endTime, nextCursor, prCommitsToExclude) + if err != nil { + return nil, err + } + result = append(result, prs...) + } + return result, nil +} + +func searchCockroachPRsSingle( + startTime time.Time, + endTime time.Time, + cursor string, + prCommitsToExclude map[int]map[string]string, +) (bool, string, []cockroachPR, error) { + var search gqlCockroachPR + var result []cockroachPR + query := `query ($cursor: String, $ghSearchQuery: String!) { + search(first: 100, query: $ghSearchQuery, type: ISSUE, after: $cursor) { + nodes { + ... on PullRequest { + title + number + body + baseRefName + commits(first: 100) { + edges { + node { + commit { + oid + messageHeadline + messageBody + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + }` + queryVariables := map[string]interface{}{ + "ghSearchQuery": fmt.Sprintf( + `repo:cockroachdb/cockroach is:pr is:merged merged:%s..%s`, + startTime.Format(time.RFC3339), + endTime.Format(time.RFC3339), + ), + } + if cursor != "" { + queryVariables["cursor"] = cursor + } + err := queryGraphQL(query, queryVariables, &search) + if err != nil { + return false, "", nil, err + } + for _, x := range search.Data.Search.Nodes { + var commits []cockroachCommit + for _, y := range x.Commits.Edges { + matchingDocsIssue := prCommitsToExclude[x.Number][y.Node.Commit.Oid] + if nonBugFixRNRE.MatchString(y.Node.Commit.MessageBody) && matchingDocsIssue == "" { + commit := cockroachCommit{ + Sha: y.Node.Commit.Oid, + MessageHeadline: y.Node.Commit.MessageHeadline, + MessageBody: y.Node.Commit.MessageBody, + } + commits = append(commits, commit) + } + } + // runs if there are more than 100 results + if x.Commits.PageInfo.HasNextPage { + additionalCommits, err := searchCockroachPRCommits(x.Number, x.Commits.PageInfo.EndCursor, prCommitsToExclude) + if err != nil { + return false, "", nil, err + } + commits = append(commits, additionalCommits...) + } + if len(commits) > 0 { + pr := cockroachPR{ + Number: x.Number, + Title: x.Title, + Body: x.Body, + BaseRefName: x.BaseRefName, + Commits: commits, + } + result = append(result, pr) + } + } + pageInfo := search.Data.Search.PageInfo + return pageInfo.HasNextPage, pageInfo.EndCursor, result, nil +} + +func searchCockroachPRCommits( + pr int, cursor string, prCommitsToExclude map[int]map[string]string, +) ([]cockroachCommit, error) { + hasNextPage, nextCursor, commits, err := searchCockroachPRCommitsSingle(pr, cursor, prCommitsToExclude) + if err != nil { + return nil, err + } + result := commits + for hasNextPage { + hasNextPage, nextCursor, commits, err = searchCockroachPRCommitsSingle(pr, nextCursor, prCommitsToExclude) + if err != nil { + return nil, err + } + result = append(result, commits...) + } + return result, nil +} + +func searchCockroachPRCommitsSingle( + prNumber int, cursor string, prCommitsToExclude map[int]map[string]string, +) (bool, string, []cockroachCommit, error) { + var result []cockroachCommit + var search gqlCockroachPRCommit + query := `query ($cursor: String, $prNumber: Int!) { + repository(owner: "cockroachdb", name: "cockroach") { + pullRequest(number: $prNumber) { + commits(first: 100, after: $cursor) { + edges { + node { + commit { + oid + messageHeadline + messageBody + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + } +}` + queryVariables := map[string]interface{}{ + "prNumber": prNumber, + } + if cursor != "" { + queryVariables["cursor"] = cursor + } + err := queryGraphQL( + query, + queryVariables, + &search, + ) + if err != nil { + return false, "", nil, err + } + for _, x := range search.Data.Repository.PullRequest.Commits.Edges { + matchingDocsIssue := prCommitsToExclude[prNumber][x.Node.Commit.Oid] + if nonBugFixRNRE.MatchString(x.Node.Commit.MessageHeadline) && matchingDocsIssue == "" { + commit := cockroachCommit{ + Sha: x.Node.Commit.Oid, + MessageHeadline: x.Node.Commit.MessageHeadline, + MessageBody: x.Node.Commit.MessageHeadline, + } + result = append(result, commit) + } + } + pageInfo := search.Data.Repository.PullRequest.Commits.PageInfo + return pageInfo.HasNextPage, pageInfo.EndCursor, result, nil +} + +var searchCockroachReleaseBranches = func() ([]string, error) { + var result []string + hasNextPage, nextCursor, branches, err := searchCockroachReleaseBranchesSingle("") + if err != nil { + return []string{}, err + } + result = append(result, branches...) + for hasNextPage { + hasNextPage, nextCursor, branches, err = searchCockroachReleaseBranchesSingle(nextCursor) + if err != nil { + return []string{}, err + } + result = append(result, branches...) + } + return result, nil +} + +func searchCockroachReleaseBranchesSingle(cursor string) (bool, string, []string, error) { + var search gqlRef + var result []string + query := `query ($cursor: String) { + repository(owner: "cockroachdb", name: "cockroach") { + refs( + first: 100 + refPrefix: "refs/" + query: "heads/release-" + after: $cursor + orderBy: {field: TAG_COMMIT_DATE, direction: DESC} + ) { + edges { + node { + name + } + } + pageInfo { + hasNextPage + endCursor + } + } + } +}` + queryVariables := make(map[string]interface{}) + if cursor != "" { + queryVariables["cursor"] = cursor + } + err := queryGraphQL(query, queryVariables, &search) + if err != nil { + return false, "", []string{}, err + } + for _, x := range search.Data.Repository.Refs.Edges { + result = append(result, strings.Replace(x.Node.Name, "heads/", "", -1)) + } + pageInfo := search.Data.Repository.Refs.PageInfo + return pageInfo.HasNextPage, pageInfo.EndCursor, result, nil +} + +// getJiraIssueFromGitHubIssue takes a GitHub issue and returns the appropriate Jira key from the issue body. +// getJiraIssueFromGitHubIssue is specified as a function closure to allow for testing +// of getJiraIssueFromGitHubIssue* methods. +var getJiraIssueFromGitHubIssue = func(org, repo string, issue int) (string, error) { + query := `query ($org: String!, $repo: String!, $issue: Int!) { + repository(owner: $org, name: $repo) { + issue(number: $issue) { + body + } + } + }` + var search gqlSingleIssue + queryVariables := map[string]interface{}{ + "org": org, + "repo": repo, + "issue": issue, + } + err := queryGraphQL(query, queryVariables, &search) + if err != nil { + return "", err + } + var jiraIssue string + exalateRef := exalateJiraRefRE.FindString(search.Data.Repository.Issue.Body) + if len(exalateRef) > 0 { + jiraIssue = jiraIssuePartRE.FindString(exalateRef) + } + return jiraIssue, nil +} diff --git a/pkg/cmd/docs-issue-generation/jira.go b/pkg/cmd/docs-issue-generation/jira.go new file mode 100644 index 000000000000..7501cb82bca4 --- /dev/null +++ b/pkg/cmd/docs-issue-generation/jira.go @@ -0,0 +1,150 @@ +// Copyright 2023 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +import ( + "fmt" + "strconv" + "time" +) + +const ( + jiraDocsProjectCode = "DOC" + jiraEpicIssueTypeKey = 10000 +) + +// searchDocsIssues returns a map containing all the product change docs issues that have been created since the given +// start time. For reference, it's structured as map[crdb_pr_number]map[crdb_commit]docs_pr_number. +func searchJiraDocsIssues(startTime time.Time) (map[int]map[string]string, error) { + var result = map[int]map[string]string{} + startAt := 0 + pageSize := 100 + maxResults, total, err := searchJiraDocsIssuesSingle(startTime, pageSize, startAt, result) + if err != nil { + return nil, err + } + pageSize = maxResults // Jira REST API page sizes are subject to change at any time + for total > startAt+pageSize && pageSize > 0 { + startAt += pageSize + _, _, err = searchJiraDocsIssuesSingle(startTime, pageSize, startAt, result) + if err != nil { + return nil, err + } + } + return result, nil +} + +// searchDocsIssuesSingle searches one page of docs issues at a time. These docs issues will ultimately be excluded +// from the PRs through which we iterate to create new product change docs issues. This function returns a bool to +// check if there are more than 100 results, the cursor to query for the next page of results, and an error if +// one exists. +func searchJiraDocsIssuesSingle( + startTime time.Time, pageSize, startAt int, m map[int]map[string]string, +) (int, int, error) { + apiEndpoint := "search" + method := "POST" + headers := map[string]string{ + "Accept": "application/json", + "Content-Type": "application/json", + } + body := map[string]interface{}{ + "expand": []string{ + "renderedFields", + }, + "fields": []string{ + "description", + }, + "fieldsByKeys": false, + "jql": fmt.Sprintf( + `project = DOC and "Doc Type[Dropdown]" = "Product Change" and summary ~ "PR #" and createdDate >= "%s"`, + startTime.Format("2006-01-02 15:04"), + ), + "maxResults": pageSize, + "startAt": startAt, + } + var search jiraIssueSearch + err := queryJiraRESTAPI(apiEndpoint, method, headers, body, &search) + if err != nil { + return 0, 0, err + } + for _, issue := range search.Issues { + prNumber, commitSha, err := extractPRNumberCommitFromDocsIssueBody(issue.RenderedFields.Description) + if err != nil { + return 0, 0, err + } + if prNumber != 0 && commitSha != "" { + _, ok := m[prNumber] + if !ok { + m[prNumber] = map[string]string{} + } + m[prNumber][commitSha] = issue.Key + } + } + return search.MaxResults, search.Total, nil +} + +// getJiraIssueCreateMeta gets details about the metadata of a project to help assist in creating issues of type "Docs" +// within that project. +var getJiraIssueCreateMeta = func() (jiraIssueCreateMeta, error) { + apiEndpoint := fmt.Sprintf("issue/createmeta?projectKeys=%s&issuetypeNames=Docs&expand=projects.issuetypes.fields.parent", jiraDocsProjectCode) + method := "GET" + headers := map[string]string{ + "Accept": "application/json", + } + var search jiraIssueCreateMeta + err := queryJiraRESTAPI(apiEndpoint, method, headers, nil, &search) + if err != nil { + return jiraIssueCreateMeta{}, err + } + return search, nil +} + +// getValidEpicRef takes an issue key (PROJCODE-####) and outputs a bool if the issue key is an epic, a new epic key +// if the issue is part of another epic, and an error. +var getValidEpicRef = func(issueKey string) (bool, string, error) { + apiEndpoint := fmt.Sprintf("issue/%s?fields=issuetype,customfield_10014", issueKey) + method := "GET" + headers := map[string]string{ + "Accept": "application/json", + } + var issueResp jiraIssue + err := queryJiraRESTAPI(apiEndpoint, method, headers, nil, &issueResp) + if err != nil { + return false, "", err + } + var isEpic bool + var epicKey string + if issueResp.Fields.Issuetype.Id == strconv.Itoa(jiraEpicIssueTypeKey) { + isEpic = true + epicKey = issueKey + } else { + epicKey = issueResp.Fields.EpicLink + } + return isEpic, epicKey, nil +} + +func (dib docsIssueBatch) createDocsIssuesInBulk() error { + apiEndpoint := "issue/bulk" + method := "POST" + headers := map[string]string{ + "Accept": "application/json", + "Content-Type": "application/json", + } + var res jiraBulkIssueCreateResponse + err := queryJiraRESTAPI(apiEndpoint, method, headers, dib, &res) + if err != nil { + return err + } + if len(res.Errors) > 0 { + return fmt.Errorf("error: Could not create issues: %+v", res.Errors) + } + return nil +} diff --git a/pkg/cmd/docs-issue-generation/main.go b/pkg/cmd/docs-issue-generation/main.go index c25f760d930c..05b76093357a 100644 --- a/pkg/cmd/docs-issue-generation/main.go +++ b/pkg/cmd/docs-issue-generation/main.go @@ -1,4 +1,4 @@ -// Copyright 2022 The Cockroach Authors. +// Copyright 2023 The Cockroach Authors. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt. @@ -10,9 +10,11 @@ // Check that GitHub PR descriptions and commit messages contain the // expected epic and issue references. + package main import ( + "fmt" "os" "time" @@ -22,25 +24,25 @@ import ( var rootCmd = &cobra.Command{ Use: "docs-issue-generation", - Short: "Generate a new set of release issues in the docs repo for a given commit.", - Run: func(_ *cobra.Command, args []string) { + Short: "Generate a new set of product change issues in the DOC project in Jira for a given timeframe.", + RunE: func(_ *cobra.Command, args []string) error { params := defaultEnvParameters() - docsIssueGeneration(params) + return docsIssueGeneration(params) }, } func main() { if err := rootCmd.Execute(); err != nil { + fmt.Println(err) os.Exit(1) } } -func defaultEnvParameters() parameters { +func defaultEnvParameters() queryParameters { const ( - githubAPITokenEnv = "GITHUB_API_TOKEN" - dryRunEnv = "DRY_RUN_DOCS_ISSUE_GEN" - startTimeEnv = "DOCS_ISSUE_GEN_START_TIME" - endTimeEnv = "DOCS_ISSUE_GEN_END_TIME" + dryRunEnv = "DRY_RUN_DOCS_ISSUE_GEN" + startTimeEnv = "DOCS_ISSUE_GEN_START_TIME" + endTimeEnv = "DOCS_ISSUE_GEN_END_TIME" ) defaultStartTimeStr := timeutil.Now().Add(time.Hour * (-48)).Format(time.RFC3339) defaultEndTimeStr := timeutil.Now().Format(time.RFC3339) @@ -51,8 +53,7 @@ func defaultEnvParameters() parameters { startTimeTime, _ := time.Parse(time.RFC3339, startTimeStr) endTimeTime, _ := time.Parse(time.RFC3339, endTimeStr) - return parameters{ - Token: maybeEnv(githubAPITokenEnv, ""), + return queryParameters{ DryRun: stringToBool(maybeEnv(dryRunEnv, "false")), StartTime: startTimeTime, EndTime: endTimeTime, diff --git a/pkg/cmd/docs-issue-generation/requests.go b/pkg/cmd/docs-issue-generation/requests.go new file mode 100644 index 000000000000..0259d4e8e9ac --- /dev/null +++ b/pkg/cmd/docs-issue-generation/requests.go @@ -0,0 +1,133 @@ +// Copyright 2023 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/cockroachdb/errors" +) + +const ( + graphQLURL = "https://api.github.com/graphql" + crlJiraBaseUrl = "https://cockroachlabs.atlassian.net/" + jiraRESTURLPart = "rest/api/3/" + jiraDocsUserEmail = "cockroach-jira-docs@cockroachlabs.com" +) + +const ( + httpSourceGitHub httpReqSource = iota + httpSourceJira +) + +var ( + tokenParams = tokenParameters() + githubAuthHeader = fmt.Sprintf("Bearer %s", tokenParams.GitHubToken) +) + +func tokenParameters() apiTokenParameters { + const ( + githubApiTokenEnv = "GITHUB_API_TOKEN" + jiraApiTokenEnv = "JIRA_API_TOKEN" + ) + return apiTokenParameters{ + GitHubToken: maybeEnv(githubApiTokenEnv, ""), + JiraToken: maybeEnv(jiraApiTokenEnv, ""), + } +} + +// queryGraphQL is the function that interfaces directly with the GitHub GraphQL API. Given a query, variables, and +// token, it will return a struct containing the requested data or an error if one exists. +func queryGraphQL(query string, queryVariables map[string]interface{}, out interface{}) error { + body := map[string]interface{}{ + "query": query, + } + if queryVariables != nil { + body["variables"] = queryVariables + } + err := httpRequest(graphQLURL, "POST", httpSourceGitHub, nil, body, &out) + if err != nil { + return err + } + return nil +} + +func queryJiraRESTAPI( + apiEndpoint, method string, headers map[string]string, body interface{}, out interface{}, +) error { + url := crlJiraBaseUrl + jiraRESTURLPart + apiEndpoint + err := httpRequest(url, method, httpSourceJira, headers, body, &out) + if err != nil { + return err + } + return nil +} + +func httpRequest( + url, method string, + source httpReqSource, + headers map[string]string, + body interface{}, + out interface{}, +) error { + var requestBody bytes.Buffer + encoder := json.NewEncoder(&requestBody) + encoder.SetEscapeHTML(false) + err := encoder.Encode(body) + if err != nil { + return err + } + + req, err := http.NewRequest(method, url, &requestBody) + if err != nil { + return err + } + if source == httpSourceJira { + req.SetBasicAuth(jiraDocsUserEmail, tokenParams.JiraToken) + } else if source == httpSourceGitHub { + req.Header.Set("Authorization", githubAuthHeader) + } else { + return fmt.Errorf("error: Unexpected httpReqSource %d", source) + } + for key, value := range headers { + req.Header.Set(key, value) + } + client := &http.Client{} + res, err := client.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK && res.StatusCode != 201 { + err = fmt.Errorf("error: Request failed with status: %s", res.Status) + return err + } + bs, err := io.ReadAll(res.Body) + if err != nil { + return err + } + // unmarshal (convert) the byte slice into an interface + var tmp interface{} + err = json.Unmarshal(bs, &tmp) + if err != nil { + return errors.CombineErrors(err, fmt.Errorf("byte slice: %s", string(bs[:]))) + } + err = json.Unmarshal(bs, out) + if err != nil { + //return fmt.Errorf("%s\nResponse from server: %+v\n", err, tmp) + return errors.CombineErrors(err, fmt.Errorf("response from server: %+v", tmp)) + } + return nil +} diff --git a/pkg/cmd/docs-issue-generation/structs.go b/pkg/cmd/docs-issue-generation/structs.go new file mode 100644 index 000000000000..746b0a8ca199 --- /dev/null +++ b/pkg/cmd/docs-issue-generation/structs.go @@ -0,0 +1,387 @@ +// Copyright 2023 The Cockroach Authors. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package main + +import "time" + +type jiraFieldId struct { + Id string `json:"id"` +} + +// docsIssue contains details about each formatted commit to be committed to the docs repo. +type docsIssue struct { + Fields docsIssueFields `json:"fields"` +} + +type docsIssueFields struct { + IssueType jiraFieldId `json:"issuetype"` + Project jiraFieldId `json:"project"` + Summary string `json:"summary"` + Reporter jiraFieldId `json:"reporter"` + Description adfRoot `json:"description"` + DocType jiraFieldId `json:"customfield_10175"` + FixVersions []jiraFieldId `json:"fixVersions"` + EpicLink string `json:"customfield_10014,omitempty"` + ProductChangePrNumber string `json:"customfield_10435"` + ProductChangeCommitSHA string `json:"customfield_10436"` +} + +type docsIssueBatch struct { + IssueUpdates []docsIssue `json:"issueUpdates"` +} + +type jiraBulkIssueCreateResponse struct { + Issues []struct { + Id string `json:"id"` + Key string `json:"key"` + Self string `json:"self"` + Transition struct { + Status int `json:"status"` + ErrorCollection struct { + ErrorMessages []interface{} `json:"errorMessages"` + Errors struct { + } `json:"errors"` + } `json:"errorCollection"` + } `json:"transition,omitempty"` + } `json:"issues"` + Errors []interface{} `json:"errors"` +} + +// queryParameters stores the GitHub API token, a dry run flag to output the issues it would create, and the +// start and end times of the search. +type queryParameters struct { + DryRun bool + StartTime time.Time + EndTime time.Time +} + +type apiTokenParameters struct { + GitHubToken string + JiraToken string +} + +// pageInfo contains pagination information for querying the GraphQL API. +type pageInfo struct { + HasNextPage bool `json:"hasNextPage"` + EndCursor string `json:"endCursor"` +} + +// gqlCockroachPRCommit contains details about commits within PRs in the cockroach repo. +type gqlCockroachPRCommit struct { + Data struct { + Repository struct { + PullRequest struct { + Commits struct { + Edges []struct { + Node struct { + Commit struct { + Oid string `json:"oid"` + MessageHeadline string `json:"messageHeadline"` + MessageBody string `json:"messageBody"` + } `json:"commit"` + } `json:"node"` + } `json:"edges"` + PageInfo pageInfo `json:"pageInfo"` + } `json:"commits"` + } `json:"pullRequest"` + } `json:"repository"` + } `json:"data"` +} + +// gqlCockroachPR contains details about PRs within the cockroach repo. +type gqlCockroachPR struct { + Data struct { + Search struct { + Nodes []struct { + Title string `json:"title"` + Number int `json:"number"` + Body string `json:"body"` + BaseRefName string `json:"baseRefName"` + Commits struct { + Edges []struct { + Node struct { + Commit struct { + Oid string `json:"oid"` + MessageHeadline string `json:"messageHeadline"` + MessageBody string `json:"messageBody"` + } `json:"commit"` + } `json:"node"` + } `json:"edges"` + PageInfo pageInfo `json:"pageInfo"` + } `json:"commits"` + } `json:"nodes"` + PageInfo pageInfo `json:"pageInfo"` + } `json:"search"` + } `json:"data"` +} + +type gqlSingleIssue struct { + Data struct { + Repository struct { + Issue struct { + Body string `json:"body"` + } `json:"issue"` + } `json:"repository"` + } `json:"data"` +} + +type gqlRef struct { + Data struct { + Repository struct { + Refs struct { + Edges []struct { + Node struct { + Name string `json:"name"` + } `json:"node"` + } `json:"edges"` + PageInfo pageInfo `json:"pageInfo"` + } `json:"refs"` + } `json:"repository"` + } `json:"data"` +} + +type jiraIssueSearch struct { + StartAt int `json:"startAt"` + MaxResults int `json:"maxResults"` + Total int `json:"total"` + Issues []struct { + Key string `json:"key"` + RenderedFields struct { + Description string `json:"description"` + } `json:"renderedFields"` + } `json:"issues"` +} + +type jiraIssueCreateMeta struct { + Projects []struct { + Issuetypes []struct { + Fields struct { + Issuetype struct { + Required bool `json:"required"` + Schema struct { + Type string `json:"type"` + System string `json:"system"` + } `json:"schema"` + Name string `json:"name"` + Key string `json:"key"` + HasDefaultValue bool `json:"hasDefaultValue"` + Operations []interface{} `json:"operations"` + AllowedValues []struct { + Self string `json:"self"` + Id string `json:"id"` + Description string `json:"description"` + IconUrl string `json:"iconUrl"` + Name string `json:"name"` + Subtask bool `json:"subtask"` + AvatarId int `json:"avatarId"` + HierarchyLevel int `json:"hierarchyLevel"` + } `json:"allowedValues"` + } `json:"issuetype"` + Description struct { + Required bool `json:"required"` + Schema struct { + Type string `json:"type"` + System string `json:"system"` + } `json:"schema"` + Name string `json:"name"` + Key string `json:"key"` + HasDefaultValue bool `json:"hasDefaultValue"` + Operations []string `json:"operations"` + } `json:"description"` + Project struct { + Required bool `json:"required"` + Schema struct { + Type string `json:"type"` + System string `json:"system"` + } `json:"schema"` + Name string `json:"name"` + Key string `json:"key"` + HasDefaultValue bool `json:"hasDefaultValue"` + Operations []string `json:"operations"` + AllowedValues []struct { + Self string `json:"self"` + Id string `json:"id"` + Key string `json:"key"` + Name string `json:"name"` + ProjectTypeKey string `json:"projectTypeKey"` + Simplified bool `json:"simplified"` + AvatarUrls struct { + X48 string `json:"48x48"` + X24 string `json:"24x24"` + X16 string `json:"16x16"` + X32 string `json:"32x32"` + } `json:"avatarUrls"` + ProjectCategory struct { + Self string `json:"self"` + Id string `json:"id"` + Description string `json:"description"` + Name string `json:"name"` + } `json:"projectCategory"` + } `json:"allowedValues"` + } `json:"project"` + DocType struct { + Required bool `json:"required"` + Schema struct { + Type string `json:"type"` + Custom string `json:"custom"` + CustomId int `json:"customId"` + } `json:"schema"` + Name string `json:"name"` + Key string `json:"key"` + HasDefaultValue bool `json:"hasDefaultValue"` + Operations []string `json:"operations"` + AllowedValues []struct { + Self string `json:"self"` + Value string `json:"value"` + Id string `json:"id"` + } `json:"allowedValues"` + } `json:"customfield_10175"` + FixVersions struct { + Required bool `json:"required"` + Schema struct { + Type string `json:"type"` + Items string `json:"items"` + System string `json:"system"` + } `json:"schema"` + Name string `json:"name"` + Key string `json:"key"` + HasDefaultValue bool `json:"hasDefaultValue"` + Operations []string `json:"operations"` + AllowedValues []jiraCreateIssueMetaFixVersionsAllowedValue `json:"allowedValues"` + } `json:"fixVersions"` + EpicLink struct { + Required bool `json:"required"` + Schema struct { + Type string `json:"type"` + Custom string `json:"custom"` + CustomId int `json:"customId"` + } `json:"schema"` + Name string `json:"name"` + Key string `json:"key"` + HasDefaultValue bool `json:"hasDefaultValue"` + Operations []string `json:"operations"` + } `json:"customfield_10014"` + Summary struct { + Required bool `json:"required"` + Schema struct { + Type string `json:"type"` + System string `json:"system"` + } `json:"schema"` + Name string `json:"name"` + Key string `json:"key"` + HasDefaultValue bool `json:"hasDefaultValue"` + Operations []string `json:"operations"` + } `json:"summary"` + Reporter struct { + Required bool `json:"required"` + Schema struct { + Type string `json:"type"` + System string `json:"system"` + } `json:"schema"` + Name string `json:"name"` + Key string `json:"key"` + AutoCompleteUrl string `json:"autoCompleteUrl"` + HasDefaultValue bool `json:"hasDefaultValue"` + Operations []string `json:"operations"` + } `json:"reporter"` + ProductChangePRNumber struct { + Required bool `json:"required"` + Schema struct { + Type string `json:"type"` + Custom string `json:"custom"` + CustomId int `json:"customId"` + } `json:"schema"` + Name string `json:"name"` + Key string `json:"key"` + HasDefaultValue bool `json:"hasDefaultValue"` + Operations []string `json:"operations"` + } `json:"customfield_10435"` + ProductChangeCommitSHA struct { + Required bool `json:"required"` + Schema struct { + Type string `json:"type"` + Custom string `json:"custom"` + CustomId int `json:"customId"` + } `json:"schema"` + Name string `json:"name"` + Key string `json:"key"` + HasDefaultValue bool `json:"hasDefaultValue"` + Operations []string `json:"operations"` + } `json:"customfield_10436"` + } `json:"fields"` + } `json:"issuetypes"` + } `json:"projects"` +} + +type jiraIssue struct { + Fields struct { + Issuetype struct { + Id string `json:"id"` + } `json:"issuetype"` + EpicLink string `json:"customfield_10014"` + } `json:"fields"` +} + +type jiraCreateIssueMetaFixVersionsAllowedValue struct { + Self string `json:"self"` + Id string `json:"id"` + Description string `json:"description,omitempty"` + Name string `json:"name"` + Archived bool `json:"archived"` + Released bool `json:"released"` + ProjectId int `json:"projectId"` + StartDate string `json:"startDate,omitempty"` + ReleaseDate string `json:"releaseDate,omitempty"` + Overdue bool `json:"overdue,omitempty"` + UserStartDate string `json:"userStartDate,omitempty"` + UserReleaseDate string `json:"userReleaseDate,omitempty"` +} + +type cockroachPR struct { + Title string `json:"title"` + Number int `json:"number"` + Body string `json:"body"` + BaseRefName string `json:"baseRefName"` + Commits []cockroachCommit +} + +type cockroachCommit struct { + Sha string `json:"oid"` + MessageHeadline string `json:"messageHeadline"` + MessageBody string `json:"messageBody"` +} + +type epicIssueRefInfo struct { + epicRefs map[string]int + epicNone bool + issueCloseRefs map[string]int + issueInformRefs map[string]int +} + +type adfRoot struct { + Version int `json:"version"` // 1 + Type string `json:"type"` // doc + Content []adfNode `json:"content"` +} + +type adfNode struct { + Type string `json:"type"` + Content []adfNode `json:"content,omitempty"` // block nodes only, not inline nodes + Text string `json:"text,omitempty"` + Marks []adfMark `json:"marks,omitempty"` // inline nodes only +} + +type adfMark struct { + Type string `json:"type"` + Attrs map[string]string `json:"attrs"` +} + +type httpReqSource int diff --git a/pkg/cmd/roachtest/tests/cluster_to_cluster.go b/pkg/cmd/roachtest/tests/cluster_to_cluster.go index 197beda00885..6d29fe4d63f9 100644 --- a/pkg/cmd/roachtest/tests/cluster_to_cluster.go +++ b/pkg/cmd/roachtest/tests/cluster_to_cluster.go @@ -34,6 +34,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test" "github.com/cockroachdb/cockroach/pkg/jobs" "github.com/cockroachdb/cockroach/pkg/jobs/jobspb" + "github.com/cockroachdb/cockroach/pkg/keys" "github.com/cockroachdb/cockroach/pkg/multitenant/mtinfopb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/roachprod/install" @@ -245,6 +246,15 @@ type replicateKV struct { // max size of raw data written during each insertion maxBlockBytes int + + // partitionKVDatabaseInRegion constrains the kv database in the specified + // region and asserts, before cutover, that the replicated span configuration + // correctly enforces the regional constraint in the destination tenant. + partitionKVDatabaseInRegion string + + // antiRegion is the region we do not expect any kv data to reside in if + // partitionKVDatabaseInRegion is set. + antiRegion string } func (kv replicateKV) sourceInitCmd(tenantName string, nodes option.NodeListOption) string { @@ -270,9 +280,47 @@ func (kv replicateKV) sourceRunCmd(tenantName string, nodes option.NodeListOptio func (kv replicateKV) runDriver( workloadCtx context.Context, c cluster.Cluster, t test.Test, setup *c2cSetup, ) error { + if kv.partitionKVDatabaseInRegion != "" { + require.NotEqual(t, "", kv.antiRegion, "if partitionKVDatabaseInRegion is set, then antiRegion must be set") + t.L().Printf("constrain the kv database to region %s", kv.partitionKVDatabaseInRegion) + alterStmt := fmt.Sprintf("ALTER DATABASE kv CONFIGURE ZONE USING constraints = '[+region=%s]'", kv.partitionKVDatabaseInRegion) + srcTenantConn := c.Conn(workloadCtx, t.L(), setup.src.nodes.RandNode()[0], option.TenantName(setup.src.name)) + srcTenantSQL := sqlutils.MakeSQLRunner(srcTenantConn) + srcTenantSQL.Exec(t, alterStmt) + defer kv.checkRegionalConstraints(t, setup, srcTenantSQL) + } return defaultWorkloadDriver(workloadCtx, setup, c, kv) } +// checkRegionalConstraints checks that the kv table is constrained to the +// expected locality. +func (kv replicateKV) checkRegionalConstraints( + t test.Test, setup *c2cSetup, srcTenantSQL *sqlutils.SQLRunner, +) { + + var kvTableID uint32 + srcTenantSQL.QueryRow(t, `SELECT id FROM system.namespace WHERE name ='kv' AND "parentID" != 0`).Scan(&kvTableID) + + dstTenantCodec := keys.MakeSQLCodec(roachpb.MustMakeTenantID(uint64(setup.dst.ID))) + tablePrefix := dstTenantCodec.TablePrefix(kvTableID) + t.L().Printf("Checking replica localities in destination side kv table, id %d and table prefix %s", kvTableID, tablePrefix) + + distinctQuery := fmt.Sprintf(` +SELECT + DISTINCT replica_localities +FROM + [SHOW CLUSTER RANGES] +WHERE + start_key ~ '%s' +`, tablePrefix) + + res := setup.dst.sysSQL.QueryStr(t, distinctQuery) + require.Equal(t, 1, len(res), "expected only one distinct locality") + locality := res[0][0] + require.Contains(t, locality, kv.partitionKVDatabaseInRegion) + require.False(t, strings.Contains(locality, kv.antiRegion), "region %s is in locality %s", kv.antiRegion, locality) +} + type replicateBulkOps struct { // short uses less data during the import and rollback steps. Also only runs one rollback. short bool @@ -325,6 +373,9 @@ type replicationSpec struct { // workload specifies the streaming workload. workload streamingWorkload + // multiregion specifies multiregion cluster specs + multiregion multiRegionSpecs + // additionalDuration specifies how long the workload will run after the initial scan //completes. If the time out is set to 0, it will run until completion. additionalDuration time.Duration @@ -360,6 +411,17 @@ type replicationSpec struct { tags map[string]struct{} } +type multiRegionSpecs struct { + // srcLocalities specifies the zones each src node should live. The length of this array must match the number of src nodes. + srcLocalities []string + + // destLocalities specifies the zones each src node should live. The length of this array must match the number of dest nodes. + destLocalities []string + + // workloadNodeZone specifies the zone that the workload node should live + workloadNodeZone string +} + // replicationDriver manages c2c roachtest execution. type replicationDriver struct { rs replicationSpec @@ -392,6 +454,13 @@ func makeReplicationDriver(t test.Test, c cluster.Cluster, rs replicationSpec) * } func (rd *replicationDriver) setupC2C(ctx context.Context, t test.Test, c cluster.Cluster) { + if len(rd.rs.multiregion.srcLocalities) != 0 { + nodeCount := rd.rs.srcNodes + rd.rs.dstNodes + localityCount := len(rd.rs.multiregion.srcLocalities) + len(rd.rs.multiregion.destLocalities) + require.Equal(t, nodeCount, localityCount) + require.NotEqual(t, "", rd.rs.multiregion.workloadNodeZone) + } + c.Put(ctx, t.Cockroach(), "./cockroach") srcCluster := c.Range(1, rd.rs.srcNodes) dstCluster := c.Range(rd.rs.srcNodes+1, rd.rs.srcNodes+rd.rs.dstNodes) @@ -706,7 +775,7 @@ func (rd *replicationDriver) backupAfterFingerprintMismatch( if rd.c.Spec().Cloud == spec.AWS { prefix = "s3" } - collection := fmt.Sprintf("%s://%s/c2c-fingerprint-mismatch/%s/%s/%s?AUTH=implicit", prefix, testutils.BackupTestingBucket(), rd.rs.name, rd.c.Name(), tenantName) + collection := fmt.Sprintf("%s://%s/c2c-fingerprint-mismatch/%s/%s/%s?AUTH=implicit", prefix, testutils.BackupTestingBucketLongTTL(), rd.rs.name, rd.c.Name(), tenantName) fullBackupQuery := fmt.Sprintf("BACKUP INTO '%s' AS OF SYSTEM TIME '%s' with revision_history", collection, startTime.AsOfSystemTime()) _, err := conn.ExecContext(ctx, fullBackupQuery) if err != nil { @@ -881,6 +950,15 @@ func c2cRegisterWrapper( clusterOps = append(clusterOps, spec.VolumeSize(sp.pdSize)) } + if len(sp.multiregion.srcLocalities) > 0 { + allZones := make([]string, 0, sp.srcNodes+sp.dstNodes+1) + allZones = append(allZones, sp.multiregion.srcLocalities...) + allZones = append(allZones, sp.multiregion.destLocalities...) + allZones = append(allZones, sp.multiregion.workloadNodeZone) + clusterOps = append(clusterOps, spec.Zones(strings.Join(allZones, ","))) + clusterOps = append(clusterOps, spec.Geo()) + } + r.Add(registry.TestSpec{ Name: sp.name, Owner: registry.OwnerDisasterRecovery, @@ -986,6 +1064,29 @@ func registerClusterToCluster(r registry.Registry) { additionalDuration: 1 * time.Minute, cutover: 0, }, + { + name: "c2c/MultiRegion/SameRegions/kv0", + benchmark: true, + srcNodes: 4, + dstNodes: 4, + cpus: 8, + pdSize: 100, + workload: replicateKV{ + readPercent: 0, + maxBlockBytes: 1024, + partitionKVDatabaseInRegion: "us-west1", + antiRegion: "us-central1", + }, + timeout: 1 * time.Hour, + additionalDuration: 10 * time.Minute, + cutover: 1 * time.Minute, + multiregion: multiRegionSpecs{ + // gcp specific + srcLocalities: []string{"us-west1-b", "us-west1-b", "us-west1-b", "us-central1-b"}, + destLocalities: []string{"us-central1-b", "us-west1-b", "us-west1-b", "us-west1-b"}, + workloadNodeZone: "us-west1-b", + }, + }, { name: "c2c/UnitTest", srcNodes: 1, diff --git a/pkg/cmd/roachtest/tests/disagg_rebalance.go b/pkg/cmd/roachtest/tests/disagg_rebalance.go index ecdd199c8826..fd2bab6dd25d 100644 --- a/pkg/cmd/roachtest/tests/disagg_rebalance.go +++ b/pkg/cmd/roachtest/tests/disagg_rebalance.go @@ -39,7 +39,7 @@ func registerDisaggRebalance(r registry.Registry) { t.Skip("disagg-rebalance is only configured to run on AWS") } c.Put(ctx, t.Cockroach(), "./cockroach") - s3dir := fmt.Sprintf("s3://%s/disagg-rebalance/%s?AUTH=implicit", testutils.BackupTestingBucket(), c.Name()) + s3dir := fmt.Sprintf("s3://%s/disagg-rebalance/%s?AUTH=implicit", testutils.BackupTestingBucketLongTTL(), c.Name()) startOpts := option.DefaultStartOptsNoBackups() startOpts.RoachprodOpts.ExtraArgs = append(startOpts.RoachprodOpts.ExtraArgs, fmt.Sprintf("--experimental-shared-storage=%s", s3dir)) c.Start(ctx, t.L(), startOpts, install.MakeClusterSettings(), c.Range(1, 3)) diff --git a/pkg/cmd/roachtest/tests/mixed_version_backup.go b/pkg/cmd/roachtest/tests/mixed_version_backup.go index 698c2d08c5df..6440c9da7295 100644 --- a/pkg/cmd/roachtest/tests/mixed_version_backup.go +++ b/pkg/cmd/roachtest/tests/mixed_version_backup.go @@ -947,11 +947,7 @@ func (bc *backupCollection) uri() string { // global namespace represented by the BACKUP_TESTING_BUCKET // bucket. The nonce allows multiple people (or TeamCity builds) to // be running this test without interfering with one another. - gcsBackupTestingBucket := os.Getenv("BACKUP_TESTING_BUCKET") - if gcsBackupTestingBucket == "" { - gcsBackupTestingBucket = "cockroachdb-backup-testing" - } - return fmt.Sprintf("gs://"+gcsBackupTestingBucket+"/mixed-version/%s_%s?AUTH=implicit", bc.name, bc.nonce) + return fmt.Sprintf("gs://%s/mixed-version/%s_%s?AUTH=implicit", testutils.BackupTestingBucketLongTTL(), bc.name, bc.nonce) } func (bc *backupCollection) encryptionOption() *encryptionPassphrase { @@ -2157,7 +2153,7 @@ func registerBackupMixedVersion(r registry.Registry) { RequiresLicense: true, Run: func(ctx context.Context, t test.Test, c cluster.Cluster) { if c.Spec().Cloud != spec.GCE && !c.IsLocal() { - t.Skip("uses gs://cockroachdb-backup-testing; see https://github.com/cockroachdb/cockroach/issues/105968") + t.Skip("uses gs://cockroachdb-backup-testing-long-ttl; see https://github.com/cockroachdb/cockroach/issues/105968") } roachNodes := c.Range(1, c.Spec().NodeCount-1) diff --git a/pkg/cmd/roachtest/tests/multitenant_utils.go b/pkg/cmd/roachtest/tests/multitenant_utils.go index 9115b28cdec8..0d7dab42ab95 100644 --- a/pkg/cmd/roachtest/tests/multitenant_utils.go +++ b/pkg/cmd/roachtest/tests/multitenant_utils.go @@ -366,6 +366,7 @@ func startInMemoryTenant( sysSQL.Exec(t, `ALTER TENANT $1 SET CLUSTER SETTING sql.split_at.allow_for_secondary_tenant.enabled=true`, tenantName) sysSQL.Exec(t, `ALTER TENANT $1 SET CLUSTER SETTING sql.scatter.allow_for_secondary_tenant.enabled=true`, tenantName) sysSQL.Exec(t, `ALTER TENANT $1 SET CLUSTER SETTING sql.zone_configs.allow_for_secondary_tenant.enabled=true`, tenantName) + sysSQL.Exec(t, `ALTER TENANT $1 SET CLUSTER SETTING sql.virtual_cluster.feature_access.multiregion.enabled=true`, tenantName) sysSQL.Exec(t, `ALTER TENANT $1 SET CLUSTER SETTING enterprise.license = $2`, tenantName, config.CockroachDevLicense) sysSQL.Exec(t, `ALTER TENANT $1 SET CLUSTER SETTING cluster.organization = 'Cockroach Labs - Production Testing'`, tenantName) removeTenantRateLimiters(t, sysSQL, tenantName) diff --git a/pkg/sql/parser/testdata/backup_restore b/pkg/sql/parser/testdata/backup_restore index 9078505562a2..31bd0b31c0f9 100644 --- a/pkg/sql/parser/testdata/backup_restore +++ b/pkg/sql/parser/testdata/backup_restore @@ -386,10 +386,10 @@ BACKUP TABLE _ TO 'bar' WITH OPTIONS (revision_history = true, detached, kms = ( parse BACKUP foo TO 'bar' WITH OPTIONS (detached = false) ---- -BACKUP TABLE foo TO 'bar' WITH OPTIONS (detached = FALSE) -- normalized! -BACKUP TABLE (foo) TO ('bar') WITH OPTIONS (detached = FALSE) -- fully parenthesized -BACKUP TABLE foo TO '_' WITH OPTIONS (detached = FALSE) -- literals removed -BACKUP TABLE _ TO 'bar' WITH OPTIONS (detached = FALSE) -- identifiers removed +BACKUP TABLE foo TO 'bar' -- normalized! +BACKUP TABLE (foo) TO ('bar') -- fully parenthesized +BACKUP TABLE foo TO '_' -- literals removed +BACKUP TABLE _ TO 'bar' -- identifiers removed parse BACKUP VIRTUAL CLUSTER 36 TO 'bar' @@ -1024,7 +1024,7 @@ HINT: try \h RESTORE parse BACKUP INTO LATEST IN UNLOGGED WITH OPTIONS ( DETACHED = FALSE ) ---- -BACKUP INTO LATEST IN 'unlogged' WITH OPTIONS (detached = FALSE) -- normalized! -BACKUP INTO LATEST IN ('unlogged') WITH OPTIONS (detached = FALSE) -- fully parenthesized -BACKUP INTO LATEST IN '_' WITH OPTIONS (detached = FALSE) -- literals removed -BACKUP INTO LATEST IN 'unlogged' WITH OPTIONS (detached = FALSE) -- identifiers removed +BACKUP INTO LATEST IN 'unlogged' -- normalized! +BACKUP INTO LATEST IN ('unlogged') -- fully parenthesized +BACKUP INTO LATEST IN '_' -- literals removed +BACKUP INTO LATEST IN 'unlogged' -- identifiers removed diff --git a/pkg/sql/sem/tree/backup.go b/pkg/sql/sem/tree/backup.go index 7092d3fe26f3..aa3c7b692808 100644 --- a/pkg/sql/sem/tree/backup.go +++ b/pkg/sql/sem/tree/backup.go @@ -379,7 +379,7 @@ func (o *BackupOptions) CombineWith(other *BackupOptions) error { func (o BackupOptions) IsDefault() bool { options := BackupOptions{} return o.CaptureRevisionHistory == options.CaptureRevisionHistory && - o.Detached == options.Detached && + (o.Detached == nil || o.Detached == DBoolFalse) && cmp.Equal(o.EncryptionKMSURI, options.EncryptionKMSURI) && o.EncryptionPassphrase == options.EncryptionPassphrase && cmp.Equal(o.IncrementalStorage, options.IncrementalStorage) && diff --git a/pkg/testutils/backup.go b/pkg/testutils/backup.go index d51dc2fb9294..399b2eeec5e1 100644 --- a/pkg/testutils/backup.go +++ b/pkg/testutils/backup.go @@ -13,14 +13,16 @@ package testutils import "os" const ( - defaultBackupBucket = "cockroachdb-backup-testing" - backupTestingBucketEnvVar = "BACKUP_TESTING_BUCKET" + defaultBackupBucket = "cockroachdb-backup-testing" + longTTLBackupTestingBucket = "cockroachdb-backup-testing-long-ttl" + backupTestingBucketEnvVar = "BACKUP_TESTING_BUCKET" + backupTestingBucketLongTTLEnvVar = "BACKUP_TESTING_BUCKET_LONG_TTL" ) -// BackupTestingBucket returns the name of the GCS bucket that should -// be used in a test run. Most times, this will be the regular public -// bucket. In private test runs, the name of the bucket is passed -// through an environment variable. +// BackupTestingBucket returns the name of the external storage bucket that +// should be used in a test run. Most times, this will be the regular public +// bucket. In private test runs, the name of the bucket is passed through an +// environment variable. func BackupTestingBucket() string { if bucket := os.Getenv(backupTestingBucketEnvVar); bucket != "" { return bucket @@ -28,3 +30,18 @@ func BackupTestingBucket() string { return defaultBackupBucket } + +// BackupTestingBucketLongTTL returns the name of the external storage bucket +// that should be used in a test run where the bucket's content may inform a +// debugging investigation. At the time of this comment, the ttl for the s3 and +// gcs buckets is 20 days. +// +// In private test runs, the name of the bucket is passed through an environment +// variable. +func BackupTestingBucketLongTTL() string { + if bucket := os.Getenv(backupTestingBucketLongTTLEnvVar); bucket != "" { + return bucket + } + + return longTTLBackupTestingBucket +} diff --git a/pkg/ui/workspaces/cluster-ui/src/databaseDetailsPage/databaseDetailsPage.tsx b/pkg/ui/workspaces/cluster-ui/src/databaseDetailsPage/databaseDetailsPage.tsx index aad2ce5ce0d2..021bb23cbc30 100644 --- a/pkg/ui/workspaces/cluster-ui/src/databaseDetailsPage/databaseDetailsPage.tsx +++ b/pkg/ui/workspaces/cluster-ui/src/databaseDetailsPage/databaseDetailsPage.tsx @@ -622,7 +622,10 @@ export class DatabaseDetailsPage extends React.Component< checkInfoAvailable( table.requestError, null, - table.details.nodesByRegionString || "None", + table.details.nodesByRegionString && + table.details.nodesByRegionString.length > 0 + ? table.details.nodesByRegionString + : null, ), sort: table => table.details.nodesByRegionString, className: cx("database-table__col--regions"), @@ -669,13 +672,16 @@ export class DatabaseDetailsPage extends React.Component< Table Stats Last Updated ), - cell: table => ( - - ), + cell: table => + checkInfoAvailable( + table.requestError, + table.details.statsLastUpdated?.error, + , + ), sort: table => table.details.statsLastUpdated, className: cx("database-table__col--table-stats"), name: "tableStatsUpdated", diff --git a/pkg/ui/workspaces/cluster-ui/src/databaseTablePage/databaseTablePage.tsx b/pkg/ui/workspaces/cluster-ui/src/databaseTablePage/databaseTablePage.tsx index 21ed05dc6f27..bfaa4b3cb30e 100644 --- a/pkg/ui/workspaces/cluster-ui/src/databaseTablePage/databaseTablePage.tsx +++ b/pkg/ui/workspaces/cluster-ui/src/databaseTablePage/databaseTablePage.tsx @@ -53,11 +53,22 @@ import { ActionCell, DbTablesBreadcrumbs, FormatMVCCInfo, + getCreateStmt, IndexRecCell, LastReset, LastUsed, NameCell, } from "./helperComponents"; +import { + SqlApiQueryResponse, + SqlExecutionErrorMessage, + TableCreateStatementRow, + TableHeuristicDetailsRow, + TableReplicaData, + TableSchemaDetailsRow, + TableSpanStatsRow, +} from "../api"; +import { checkInfoAvailable } from "../databases"; const cx = classNames.bind(styles); const booleanSettingCx = classnames.bind(booleanSettingStyles); @@ -122,17 +133,16 @@ export interface DatabaseTablePageData { export interface DatabaseTablePageDataDetails { loading: boolean; loaded: boolean; - lastError: Error; - createStatement: string; - replicaCount: number; - indexNames: string[]; - grants: Grant[]; - statsLastUpdated?: Moment; - totalBytes: number; - liveBytes: number; - livePercentage: number; - sizeInBytes: number; - rangeCount: number; + // Request error getting table details + requestError: Error; + // Query error getting table details + queryError: SqlExecutionErrorMessage; + createStatement: SqlApiQueryResponse; + replicaData: SqlApiQueryResponse; + spanStats: SqlApiQueryResponse; + indexData: SqlApiQueryResponse; + grants: SqlApiQueryResponse; + statsLastUpdated?: SqlApiQueryResponse; nodesByRegionString?: string; } @@ -157,6 +167,10 @@ interface IndexRecommendation { reason: string; } +interface AllGrants { + all: Grant[]; +} + interface Grant { user: string; privileges: string[]; @@ -274,7 +288,7 @@ export class DatabaseTablePage extends React.Component< if ( !this.props.details.loaded && !this.props.details.loading && - this.props.details.lastError === undefined + this.props.details.requestError === undefined ) { return this.props.refreshTableDetails( this.props.databaseName, @@ -412,6 +426,7 @@ export class DatabaseTablePage extends React.Component< render(): React.ReactElement { const { hasAdminRole } = this.props; + const details: DatabaseTablePageDataDetails = this.props.details; return (
@@ -442,47 +457,70 @@ export class DatabaseTablePage extends React.Component< className={cx("tab-pane")} > ( <> - + - - } + value={checkInfoAvailable( + details.requestError, + details.spanStats?.error, + , + )} /> - {this.props.details.statsLastUpdated && ( + {details.statsLastUpdated && ( - } + fallback={"No table statistics found"} + />, + )} /> )} {this.props.automaticStatsCollectionEnabled != @@ -517,7 +555,14 @@ export class DatabaseTablePage extends React.Component< {this.props.showNodeRegionsSection && ( )} LoadingError({ statsType: "databases", - error: this.props.details.lastError, + error: details.requestError, }) } /> ( )} renderError={() => LoadingError({ statsType: "databases", - error: this.props.details.lastError, + error: details.requestError, }) } /> diff --git a/pkg/ui/workspaces/cluster-ui/src/databaseTablePage/helperComponents.tsx b/pkg/ui/workspaces/cluster-ui/src/databaseTablePage/helperComponents.tsx index 882da233c86d..e0c405c3d034 100644 --- a/pkg/ui/workspaces/cluster-ui/src/databaseTablePage/helperComponents.tsx +++ b/pkg/ui/workspaces/cluster-ui/src/databaseTablePage/helperComponents.tsx @@ -35,6 +35,7 @@ import { Search as IndexIcon } from "@cockroachlabs/icons"; import { Breadcrumbs } from "../breadcrumbs"; import { CaretRight } from "../icon/caretRight"; import { CockroachCloudContext } from "../contexts"; +import { sqlApiErrorMessage } from "../api"; const cx = classNames.bind(styles); export const NameCell = ({ @@ -213,12 +214,25 @@ export const FormatMVCCInfo = ({ }): JSX.Element => { return ( <> - {format.Percentage(details.livePercentage, 1, 1)} + {format.Percentage(details.spanStats?.live_percentage, 1, 1)} {" ("} - {format.Bytes(details.liveBytes)} live - data /{" "} - {format.Bytes(details.totalBytes)} + + {format.Bytes(details.spanStats?.live_bytes)} + {" "} + live data /{" "} + + {format.Bytes(details.spanStats?.total_bytes)} + {" total)"} ); }; + +export const getCreateStmt = ({ + createStatement, +}: DatabaseTablePageDataDetails): string => { + return createStatement?.create_statement + ? createStatement?.create_statement + : "(unavailable)\n" + + sqlApiErrorMessage(createStatement?.error?.message || ""); +}; diff --git a/pkg/ui/workspaces/cluster-ui/src/databases/combiners.ts b/pkg/ui/workspaces/cluster-ui/src/databases/combiners.ts index 3379ce9ba580..117cff6fa0af 100644 --- a/pkg/ui/workspaces/cluster-ui/src/databases/combiners.ts +++ b/pkg/ui/workspaces/cluster-ui/src/databases/combiners.ts @@ -179,18 +179,14 @@ export const deriveTablePageDetailsMemoized = createSelector( return { loading: !!details?.inFlight, loaded: !!details?.valid, - lastError: details?.lastError, - createStatement: results?.createStmtResp.create_statement || "", - replicaCount: results?.stats.replicaData.replicaCount || 0, - indexNames: results?.schemaDetails.indexes || [], - grants: normalizedGrants, - statsLastUpdated: - results?.heuristicsDetails.stats_last_created_at || null, - totalBytes: results?.stats.spanStats.total_bytes || 0, - liveBytes: results?.stats.spanStats.live_bytes || 0, - livePercentage: results?.stats.spanStats.live_percentage || 0, - sizeInBytes: results?.stats.spanStats.approximate_disk_bytes || 0, - rangeCount: results?.stats.spanStats.range_count || 0, + requestError: details?.lastError, + queryError: results?.error, + createStatement: results?.createStmtResp, + replicaData: results?.stats?.replicaData, + indexData: results?.schemaDetails, + grants: { all: normalizedGrants, error: results?.grantsResp?.error }, + statsLastUpdated: results?.heuristicsDetails, + spanStats: results?.stats?.spanStats, nodesByRegionString: getNodesByRegionString(nodes, nodeRegions, isTenant), }; }, diff --git a/pkg/ui/workspaces/db-console/src/views/databases/databaseTablePage/redux.spec.ts b/pkg/ui/workspaces/db-console/src/views/databases/databaseTablePage/redux.spec.ts index b482367f61e2..7b1b2f05cf5a 100644 --- a/pkg/ui/workspaces/db-console/src/views/databases/databaseTablePage/redux.spec.ts +++ b/pkg/ui/workspaces/db-console/src/views/databases/databaseTablePage/redux.spec.ts @@ -105,10 +105,13 @@ class TestDriver { expect(rest).toEqual(expectedRest); expect( // Moments are the same - moment(statsLastUpdated).isSame(expectedStatsLastUpdated) || + moment(statsLastUpdated.stats_last_created_at).isSame( + expectedStatsLastUpdated.stats_last_created_at, + ) || // Moments are null. - (statsLastUpdated === expectedStatsLastUpdated && - statsLastUpdated === null), + (statsLastUpdated.stats_last_created_at === + expectedStatsLastUpdated.stats_last_created_at && + statsLastUpdated.stats_last_created_at === null), ).toBe(true); } @@ -188,17 +191,17 @@ describe("Database Table Page", function () { details: { loading: false, loaded: false, - lastError: undefined, - createStatement: "", - replicaCount: 0, - indexNames: [], - grants: [], - statsLastUpdated: null, - livePercentage: 0, - liveBytes: 0, - totalBytes: 0, - sizeInBytes: 0, - rangeCount: 0, + requestError: undefined, + queryError: undefined, + createStatement: undefined, + replicaData: undefined, + spanStats: undefined, + indexData: undefined, + grants: { + all: [], + error: undefined, + }, + statsLastUpdated: undefined, nodesByRegionString: "", }, automaticStatsCollectionEnabled: true, @@ -286,20 +289,31 @@ describe("Database Table Page", function () { driver.assertTableDetails({ loading: false, loaded: true, - lastError: null, - createStatement: "CREATE TABLE foo", - replicaCount: 5, - indexNames: ["primary", "anotha", "one"], - grants: [ - { user: "admin", privileges: ["CREATE", "DROP"] }, - { user: "public", privileges: ["SELECT"] }, - ], - statsLastUpdated: mockStatsLastCreatedTimestamp, - livePercentage: 1, - liveBytes: 45, - totalBytes: 45, - sizeInBytes: 23, - rangeCount: 56, + requestError: null, + queryError: undefined, + createStatement: { create_statement: "CREATE TABLE foo" }, + replicaData: { replicaCount: 5, nodeCount: 5, nodeIDs: [1, 2, 3, 4, 5] }, + spanStats: { + approximate_disk_bytes: 23, + live_bytes: 45, + total_bytes: 45, + range_count: 56, + live_percentage: 1, + }, + indexData: { + columns: ["colA", "colB", "c"], + indexes: ["primary", "anotha", "one"], + }, + grants: { + all: [ + { user: "admin", privileges: ["CREATE", "DROP"] }, + { user: "public", privileges: ["SELECT"] }, + ], + error: undefined, + }, + statsLastUpdated: { + stats_last_created_at: mockStatsLastCreatedTimestamp, + }, nodesByRegionString: "", }); });