From c472b28d910316ca2f49b5bc4b3104d3bcc9cdb3 Mon Sep 17 00:00:00 2001 From: Guillermo Gaston Date: Wed, 7 Jun 2023 15:25:35 +0000 Subject: [PATCH] Make release notes generation concurrent Now that we retrieve the area labels from the PRs, the release notes generation can take up to 2 minutes for big releases like 1.5. Although in general this is not a problem since it should be a one time process, it does slow down the comms release team when iterating in the notes, adding missing labels, etc. It also makes development of the script painful when testing it manually. The default is 10 concurrent routines, but this can be changed with command flag. The default value of 10 doesn't seem to run into rate limiting problems. I had to bump it to a 100 and run the command multiple times in a row to be able to get rate limited. --- hack/tools/release/notes.go | 149 +++++++++++++++++++++++++----------- 1 file changed, 105 insertions(+), 44 deletions(-) diff --git a/hack/tools/release/notes.go b/hack/tools/release/notes.go index f5812502e7e8..2e825caf2786 100644 --- a/hack/tools/release/notes.go +++ b/hack/tools/release/notes.go @@ -31,6 +31,7 @@ import ( "regexp" "sort" "strings" + "sync" "time" ) @@ -64,8 +65,9 @@ var ( fromTag = flag.String("from", "", "The tag or commit to start from.") - since = flag.String("since", "", "Include commits starting from and including this date. Accepts format: YYYY-MM-DD") - until = flag.String("until", "", "Include commits up to and including this date. Accepts format: YYYY-MM-DD") + since = flag.String("since", "", "Include commits starting from and including this date. Accepts format: YYYY-MM-DD") + until = flag.String("until", "", "Include commits up to and including this date. Accepts format: YYYY-MM-DD") + numWorkers = flag.Int("workers", 10, "Number of concurrent routines to process PR entries. If running into GitHub rate limiting, use 1.") tagRegex = regexp.MustCompile(`^\[release-[\w-\.]*\]`) ) @@ -129,7 +131,7 @@ func getAreaLabel(merge string) (string, error) { out, err := cmd.CombinedOutput() if err != nil { - return "", err + return "", fmt.Errorf("%s: %v", string(out), err) } pr := &githubPullRequest{} @@ -223,53 +225,49 @@ func run() int { } } - for _, c := range commits { - body := trimTitle(c.body) - var key, prNumber, fork string - prefix, err := getAreaLabel(c.merge) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - switch { - case strings.HasPrefix(body, ":sparkles:"), strings.HasPrefix(body, "✨"): - key = features - body = strings.TrimPrefix(body, ":sparkles:") - body = strings.TrimPrefix(body, "✨") - case strings.HasPrefix(body, ":bug:"), strings.HasPrefix(body, "🐛"): - key = bugs - body = strings.TrimPrefix(body, ":bug:") - body = strings.TrimPrefix(body, "🐛") - case strings.HasPrefix(body, ":book:"), strings.HasPrefix(body, "📖"): - key = documentation - body = strings.TrimPrefix(body, ":book:") - body = strings.TrimPrefix(body, "📖") - if strings.Contains(body, "CAEP") || strings.Contains(body, "proposal") { - key = proposals + results := make(chan releaseNoteEntryResult) + commitCh := make(chan *commit) + var wg sync.WaitGroup + + wg.Add(*numWorkers) + for i := 0; i < *numWorkers; i++ { + go func() { + for commit := range commitCh { + processed := releaseNoteEntryResult{} + processed.prEntry, processed.err = generateReleaseNoteEntry(commit) + results <- processed } - case strings.HasPrefix(body, ":seedling:"), strings.HasPrefix(body, "🌱"): - key = other - body = strings.TrimPrefix(body, ":seedling:") - body = strings.TrimPrefix(body, "🌱") - case strings.HasPrefix(body, ":warning:"), strings.HasPrefix(body, "⚠️"): - key = warning - body = strings.TrimPrefix(body, ":warning:") - body = strings.TrimPrefix(body, "⚠️") - default: - key = unknown + wg.Done() + }() + } + + go func() { + for _, c := range commits { + commitCh <- c + } + close(commitCh) + }() + + go func() { + wg.Wait() + close(results) + }() + + for result := range results { + if result.err != nil { + fmt.Println(result.err) + os.Exit(0) } - body = strings.TrimSpace(body) - if body == "" { + if result.prEntry.title == "" { continue } - body = fmt.Sprintf("- %s: %s", prefix, body) - _, _ = fmt.Sscanf(c.merge, "Merge pull request %s from %s", &prNumber, &fork) - if key == documentation { - merges[key] = append(merges[key], prNumber) - continue + + if result.prEntry.section == documentation { + merges[result.prEntry.section] = append(merges[result.prEntry.section], result.prEntry.prNumber) + } else { + merges[result.prEntry.section] = append(merges[result.prEntry.section], result.prEntry.title) } - merges[key] = append(merges[key], formatMerge(body, prNumber)) } // TODO Turn this into a link (requires knowing the project name + organization) @@ -346,3 +344,66 @@ func commandExists(cmd string) bool { _, err := exec.LookPath(cmd) return err == nil } + +// releaseNoteEntryResult is the result of processing a PR to create a release note item. +// Used to aggregate the line item and error when processing concurrently. +type releaseNoteEntryResult struct { + prEntry *releaseNoteEntry + err error +} + +// releaseNoteEntry represents a line item in the release notes. +type releaseNoteEntry struct { + title string + section string + prNumber string +} + +// generateReleaseNoteEntry processes a commit into a PR line item for the release notes. +func generateReleaseNoteEntry(c *commit) (*releaseNoteEntry, error) { + entry := &releaseNoteEntry{} + entry.title = trimTitle(c.body) + var fork string + prefix, err := getAreaLabel(c.merge) + if err != nil { + return nil, err + } + + switch { + case strings.HasPrefix(entry.title, ":sparkles:"), strings.HasPrefix(entry.title, "✨"): + entry.section = features + entry.title = strings.TrimPrefix(entry.title, ":sparkles:") + entry.title = strings.TrimPrefix(entry.title, "✨") + case strings.HasPrefix(entry.title, ":bug:"), strings.HasPrefix(entry.title, "🐛"): + entry.section = bugs + entry.title = strings.TrimPrefix(entry.title, ":bug:") + entry.title = strings.TrimPrefix(entry.title, "🐛") + case strings.HasPrefix(entry.title, ":book:"), strings.HasPrefix(entry.title, "📖"): + entry.section = documentation + entry.title = strings.TrimPrefix(entry.title, ":book:") + entry.title = strings.TrimPrefix(entry.title, "📖") + if strings.Contains(entry.title, "CAEP") || strings.Contains(entry.title, "proposal") { + entry.section = proposals + } + case strings.HasPrefix(entry.title, ":seedling:"), strings.HasPrefix(entry.title, "🌱"): + entry.section = other + entry.title = strings.TrimPrefix(entry.title, ":seedling:") + entry.title = strings.TrimPrefix(entry.title, "🌱") + case strings.HasPrefix(entry.title, ":warning:"), strings.HasPrefix(entry.title, "⚠️"): + entry.section = warning + entry.title = strings.TrimPrefix(entry.title, ":warning:") + entry.title = strings.TrimPrefix(entry.title, "⚠️") + default: + entry.section = unknown + } + + entry.title = strings.TrimSpace(entry.title) + if entry.title == "" { + return entry, nil + } + entry.title = fmt.Sprintf("- %s: %s", prefix, entry.title) + _, _ = fmt.Sscanf(c.merge, "Merge pull request %s from %s", &entry.prNumber, &fork) + entry.title = formatMerge(entry.title, entry.prNumber) + + return entry, nil +}