From 2c57f51fd5f459d4895c523134cd4e90bf54eb15 Mon Sep 17 00:00:00 2001
From: Ryan King <ryan.king@solo.io>
Date: Tue, 31 Aug 2021 16:02:27 -0400
Subject: [PATCH] Limit concurrent protoc instances (#290)

* limit concurrent protoc instances
* disable by default
* add changelog
* fix default
---
 changelog/v0.19.7/limit-protoc.yaml |  3 +++
 codegen/collector/compiler.go       | 29 +++++++++++++++++++++++++++--
 2 files changed, 30 insertions(+), 2 deletions(-)
 create mode 100644 changelog/v0.19.7/limit-protoc.yaml

diff --git a/changelog/v0.19.7/limit-protoc.yaml b/changelog/v0.19.7/limit-protoc.yaml
new file mode 100644
index 000000000..2567f520d
--- /dev/null
+++ b/changelog/v0.19.7/limit-protoc.yaml
@@ -0,0 +1,3 @@
+changelog:
+- type: NON_USER_FACING
+  description: Adds env var to limit concurrent protoc processes.
diff --git a/codegen/collector/compiler.go b/codegen/collector/compiler.go
index acb7dcaf9..113be8150 100644
--- a/codegen/collector/compiler.go
+++ b/codegen/collector/compiler.go
@@ -7,6 +7,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -77,7 +78,19 @@ func (p *protoCompiler) CompileDescriptorsFromRoot(root string, skipDirs []strin
 		defer mutex.Unlock()
 		descriptors = append(descriptors, &f)
 	}
-	var group errgroup.Group
+	var (
+		group            errgroup.Group
+		sem              chan struct{}
+		limitConcurrency bool
+	)
+	if s := os.Getenv("MAX_CONCURRENT_PROTOCS"); s != "" {
+		maxProtocs, err := strconv.Atoi(s)
+		if err != nil {
+			return nil, eris.Wrapf(err, "invalid value for MAX_CONCURRENT_PROTOCS: %s", s)
+		}
+		sem = make(chan struct{}, maxProtocs)
+		limitConcurrency = true
+	}
 	for _, dir := range append([]string{root}) {
 		absoluteDir, err := filepath.Abs(dir)
 		if err != nil {
@@ -97,6 +110,14 @@ func (p *protoCompiler) CompileDescriptorsFromRoot(root string, skipDirs []strin
 
 			// parallelize parsing the descriptors as each one requires file i/o and is slow
 			group.Go(func() error {
+				if limitConcurrency {
+					sem <- struct{}{}
+				}
+				defer func() {
+					if limitConcurrency {
+						<-sem
+					}
+				}()
 				imports, err := p.collector.CollectImportsForFile(absoluteDir, protoFile)
 				if err != nil {
 					return err
@@ -123,7 +144,11 @@ func (p *protoCompiler) CompileDescriptorsFromRoot(root string, skipDirs []strin
 	return filterDuplicateDescriptors(descriptors), nil
 }
 
-func (p *protoCompiler) addDescriptorsForFile(addDescriptor func(f DescriptorWithPath), imports []string, protoFile string) error {
+func (p *protoCompiler) addDescriptorsForFile(
+	addDescriptor func(f DescriptorWithPath),
+	imports []string,
+	protoFile string,
+) error {
 	log.Printf("processing proto file input %v", protoFile)
 	// don't generate protos for non-project files
 	compile := p.wantCompile(protoFile)