Merge pull request #8817 from g-gaston/release-notes-concurrency

🌱 Make release notes generation concurrent

hack/tools/release/notes.go

@@ -31,6 +31,7 @@ import (
 	"regexp"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 )
 
@@ -64,8 +65,9 @@ var (
 
 	fromTag = flag.String("from", "", "The tag or commit to start from.")
 
-	since = flag.String("since", "", "Include commits starting from and including this date. Accepts format: YYYY-MM-DD")
-	until = flag.String("until", "", "Include commits up to and including this date. Accepts format: YYYY-MM-DD")
+	since      = flag.String("since", "", "Include commits starting from and including this date. Accepts format: YYYY-MM-DD")
+	until      = flag.String("until", "", "Include commits up to and including this date. Accepts format: YYYY-MM-DD")
+	numWorkers = flag.Int("workers", 10, "Number of concurrent routines to process PR entries. If running into GitHub rate limiting, use 1.")
 
 	tagRegex = regexp.MustCompile(`^\[release-[\w-\.]*\]`)
 )
@@ -129,7 +131,7 @@ func getAreaLabel(merge string) (string, error) {
 
 	out, err := cmd.CombinedOutput()
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("%s: %v", string(out), err)
 	}
 
 	pr := &githubPullRequest{}
@@ -223,53 +225,49 @@ func run() int {
 		}
 	}
 
-	for _, c := range commits {
-		body := trimTitle(c.body)
-		var key, prNumber, fork string
-		prefix, err := getAreaLabel(c.merge)
-		if err != nil {
-			fmt.Println(err)
-			os.Exit(1)
-		}
-		switch {
-		case strings.HasPrefix(body, ":sparkles:"), strings.HasPrefix(body, "✨"):
-			key = features
-			body = strings.TrimPrefix(body, ":sparkles:")
-			body = strings.TrimPrefix(body, "✨")
-		case strings.HasPrefix(body, ":bug:"), strings.HasPrefix(body, "🐛"):
-			key = bugs
-			body = strings.TrimPrefix(body, ":bug:")
-			body = strings.TrimPrefix(body, "🐛")
-		case strings.HasPrefix(body, ":book:"), strings.HasPrefix(body, "📖"):
-			key = documentation
-			body = strings.TrimPrefix(body, ":book:")
-			body = strings.TrimPrefix(body, "📖")
-			if strings.Contains(body, "CAEP") || strings.Contains(body, "proposal") {
-				key = proposals
+	results := make(chan releaseNoteEntryResult)
+	commitCh := make(chan *commit)
+	var wg sync.WaitGroup
+
+	wg.Add(*numWorkers)
+	for i := 0; i < *numWorkers; i++ {
+		go func() {
+			for commit := range commitCh {
+				processed := releaseNoteEntryResult{}
+				processed.prEntry, processed.err = generateReleaseNoteEntry(commit)
+				results <- processed
 			}
-		case strings.HasPrefix(body, ":seedling:"), strings.HasPrefix(body, "🌱"):
-			key = other
-			body = strings.TrimPrefix(body, ":seedling:")
-			body = strings.TrimPrefix(body, "🌱")
-		case strings.HasPrefix(body, ":warning:"), strings.HasPrefix(body, "⚠️"):
-			key = warning
-			body = strings.TrimPrefix(body, ":warning:")
-			body = strings.TrimPrefix(body, "⚠️")
-		default:
-			key = unknown
+			wg.Done()
+		}()
+	}
+
+	go func() {
+		for _, c := range commits {
+			commitCh <- c
+		}
+		close(commitCh)
+	}()
+
+	go func() {
+		wg.Wait()
+		close(results)
+	}()
+
+	for result := range results {
+		if result.err != nil {
+			fmt.Println(result.err)
+			os.Exit(0)
 		}
 
-		body = strings.TrimSpace(body)
-		if body == "" {
+		if result.prEntry.title == "" {
 			continue
 		}
-		body = fmt.Sprintf("- %s: %s", prefix, body)
-		_, _ = fmt.Sscanf(c.merge, "Merge pull request %s from %s", &prNumber, &fork)
-		if key == documentation {
-			merges[key] = append(merges[key], prNumber)
-			continue
+
+		if result.prEntry.section == documentation {
+			merges[result.prEntry.section] = append(merges[result.prEntry.section], result.prEntry.prNumber)
+		} else {
+			merges[result.prEntry.section] = append(merges[result.prEntry.section], result.prEntry.title)
 		}
-		merges[key] = append(merges[key], formatMerge(body, prNumber))
 	}
 
 	// TODO Turn this into a link (requires knowing the project name + organization)
@@ -346,3 +344,66 @@ func commandExists(cmd string) bool {
 	_, err := exec.LookPath(cmd)
 	return err == nil
 }
+
+// releaseNoteEntryResult is the result of processing a PR to create a release note item.
+// Used to aggregate the line item and error when processing concurrently.
+type releaseNoteEntryResult struct {
+	prEntry *releaseNoteEntry
+	err     error
+}
+
+// releaseNoteEntry represents a line item in the release notes.
+type releaseNoteEntry struct {
+	title    string
+	section  string
+	prNumber string
+}
+
+// generateReleaseNoteEntry processes a commit into a PR line item for the release notes.
+func generateReleaseNoteEntry(c *commit) (*releaseNoteEntry, error) {
+	entry := &releaseNoteEntry{}
+	entry.title = trimTitle(c.body)
+	var fork string
+	prefix, err := getAreaLabel(c.merge)
+	if err != nil {
+		return nil, err
+	}
+
+	switch {
+	case strings.HasPrefix(entry.title, ":sparkles:"), strings.HasPrefix(entry.title, "✨"):
+		entry.section = features
+		entry.title = strings.TrimPrefix(entry.title, ":sparkles:")
+		entry.title = strings.TrimPrefix(entry.title, "✨")
+	case strings.HasPrefix(entry.title, ":bug:"), strings.HasPrefix(entry.title, "🐛"):
+		entry.section = bugs
+		entry.title = strings.TrimPrefix(entry.title, ":bug:")
+		entry.title = strings.TrimPrefix(entry.title, "🐛")
+	case strings.HasPrefix(entry.title, ":book:"), strings.HasPrefix(entry.title, "📖"):
+		entry.section = documentation
+		entry.title = strings.TrimPrefix(entry.title, ":book:")
+		entry.title = strings.TrimPrefix(entry.title, "📖")
+		if strings.Contains(entry.title, "CAEP") || strings.Contains(entry.title, "proposal") {
+			entry.section = proposals
+		}
+	case strings.HasPrefix(entry.title, ":seedling:"), strings.HasPrefix(entry.title, "🌱"):
+		entry.section = other
+		entry.title = strings.TrimPrefix(entry.title, ":seedling:")
+		entry.title = strings.TrimPrefix(entry.title, "🌱")
+	case strings.HasPrefix(entry.title, ":warning:"), strings.HasPrefix(entry.title, "⚠️"):
+		entry.section = warning
+		entry.title = strings.TrimPrefix(entry.title, ":warning:")
+		entry.title = strings.TrimPrefix(entry.title, "⚠️")
+	default:
+		entry.section = unknown
+	}
+
+	entry.title = strings.TrimSpace(entry.title)
+	if entry.title == "" {
+		return entry, nil
+	}
+	entry.title = fmt.Sprintf("- %s: %s", prefix, entry.title)
+	_, _ = fmt.Sscanf(c.merge, "Merge pull request %s from %s", &entry.prNumber, &fork)
+	entry.title = formatMerge(entry.title, entry.prNumber)
+
+	return entry, nil
+}