From 45c25ddf5b7d3f1de70cc76bdb42ddc34ffe12ff Mon Sep 17 00:00:00 2001
From: Yevgeniy Miretskiy <yevgeniy@cockroachlabs.com>
Date: Fri, 4 Feb 2022 13:39:26 -0500
Subject: [PATCH] roachprod: Make it possible to specify file descriptor limit.

Add a `--num-files-limit` flag to roachprod start command
to change the default file descriptor limit for cockroach process.

Release Notes: None
---
 pkg/cmd/roachprod/flags.go             |  2 ++
 pkg/roachprod/config/config.go         |  4 ++++
 pkg/roachprod/install/cockroach.go     | 27 +++++++++++++++-----------
 pkg/roachprod/install/scripts/start.sh |  3 ++-
 pkg/roachprod/roachprod.go             |  1 +
 5 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/pkg/cmd/roachprod/flags.go b/pkg/cmd/roachprod/flags.go
index 37117039b692..affb6ee709e3 100644
--- a/pkg/cmd/roachprod/flags.go
+++ b/pkg/cmd/roachprod/flags.go
@@ -256,6 +256,8 @@ func initFlags() {
 	for _, cmd := range []*cobra.Command{startCmd, startTenantCmd} {
 		cmd.Flags().BoolVar(&startOpts.Sequential,
 			"sequential", startOpts.Sequential, "start nodes sequentially so node IDs match hostnames")
+		cmd.Flags().Int64Var(&startOpts.NumFilesLimit, "num-files-limit", startOpts.NumFilesLimit,
+			"limit the number of files that can be created by the cockroach process")
 	}
 
 	for _, cmd := range []*cobra.Command{
diff --git a/pkg/roachprod/config/config.go b/pkg/roachprod/config/config.go
index 4391b90f4198..97e1f4881d0c 100644
--- a/pkg/roachprod/config/config.go
+++ b/pkg/roachprod/config/config.go
@@ -71,6 +71,10 @@ const (
 	// DefaultAdminUIPort is the default port on which the cockroach process is
 	// listening for HTTP connections for the Admin UI.
 	DefaultAdminUIPort = 26258
+
+	// DefaultNumFilesLimit is the default limit on the number of files that can
+	// be opened by the process.
+	DefaultNumFilesLimit = 65 << 10
 )
 
 // IsLocalClusterName returns true if the given name is a valid name for a local
diff --git a/pkg/roachprod/install/cockroach.go b/pkg/roachprod/install/cockroach.go
index 41af92237248..c327d06011f2 100644
--- a/pkg/roachprod/install/cockroach.go
+++ b/pkg/roachprod/install/cockroach.go
@@ -107,6 +107,9 @@ type StartOpts struct {
 	Sequential bool
 	ExtraArgs  []string
 
+	// systemd limits on resources.
+	NumFilesLimit int64
+
 	// -- Options that apply only to StartDefault target --
 
 	SkipInit        bool
@@ -420,21 +423,23 @@ func (c *SyncedCluster) generateStartCmd(
 			"GOTRACEBACK=crash",
 			"COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING=1",
 		}, c.Env...), getEnvVars()...),
-		Binary:    cockroachNodeBinary(c, node),
-		Args:      args,
-		MemoryMax: config.MemoryMax,
-		Local:     c.IsLocal(),
+		Binary:        cockroachNodeBinary(c, node),
+		Args:          args,
+		MemoryMax:     config.MemoryMax,
+		NumFilesLimit: startOpts.NumFilesLimit,
+		Local:         c.IsLocal(),
 	})
 }
 
 type startTemplateData struct {
-	Local     bool
-	LogDir    string
-	Binary    string
-	KeyCmd    string
-	MemoryMax string
-	Args      []string
-	EnvVars   []string
+	Local         bool
+	LogDir        string
+	Binary        string
+	KeyCmd        string
+	MemoryMax     string
+	NumFilesLimit int64
+	Args          []string
+	EnvVars       []string
 }
 
 func execStartTemplate(data startTemplateData) (string, error) {
diff --git a/pkg/roachprod/install/scripts/start.sh b/pkg/roachprod/install/scripts/start.sh
index 8a2a7fb6f8a7..8f87f9e67735 100644
--- a/pkg/roachprod/install/scripts/start.sh
+++ b/pkg/roachprod/install/scripts/start.sh
@@ -19,6 +19,7 @@ LOG_DIR=#{shesc .LogDir#}
 BINARY=#{shesc .Binary#}
 KEY_CMD=#{.KeyCmd#}
 MEMORY_MAX=#{.MemoryMax#}
+NUM_FILES_LIMIT=#{.NumFilesLimit#}
 ARGS=(
 #{range .Args -#}
 #{shesc .#}
@@ -93,5 +94,5 @@ sudo systemd-run --unit cockroach \
   --service-type=notify -p NotifyAccess=all \
   -p "MemoryMax=${MEMORY_MAX}" \
   -p LimitCORE=infinity \
-  -p LimitNOFILE=65536 \
+  -p "LimitNOFILE=${NUM_FILES_LIMIT}" \
   bash "${0}" run
diff --git a/pkg/roachprod/roachprod.go b/pkg/roachprod/roachprod.go
index 11b2ebdcddab..f4377ffaa3df 100644
--- a/pkg/roachprod/roachprod.go
+++ b/pkg/roachprod/roachprod.go
@@ -635,6 +635,7 @@ func DefaultStartOpts() install.StartOpts {
 	return install.StartOpts{
 		Sequential:      true,
 		EncryptedStores: false,
+		NumFilesLimit:   config.DefaultNumFilesLimit,
 		SkipInit:        false,
 		StoreCount:      1,
 		TenantID:        2,