diff --git a/pkg/cmd/roachprod/flags.go b/pkg/cmd/roachprod/flags.go index 37117039b692..affb6ee709e3 100644 --- a/pkg/cmd/roachprod/flags.go +++ b/pkg/cmd/roachprod/flags.go @@ -256,6 +256,8 @@ func initFlags() { for _, cmd := range []*cobra.Command{startCmd, startTenantCmd} { cmd.Flags().BoolVar(&startOpts.Sequential, "sequential", startOpts.Sequential, "start nodes sequentially so node IDs match hostnames") + cmd.Flags().Int64Var(&startOpts.NumFilesLimit, "num-files-limit", startOpts.NumFilesLimit, + "limit the number of files that can be created by the cockroach process") } for _, cmd := range []*cobra.Command{ diff --git a/pkg/roachprod/config/config.go b/pkg/roachprod/config/config.go index 4391b90f4198..33d8c4dcfcde 100644 --- a/pkg/roachprod/config/config.go +++ b/pkg/roachprod/config/config.go @@ -71,6 +71,10 @@ const ( // DefaultAdminUIPort is the default port on which the cockroach process is // listening for HTTP connections for the Admin UI. DefaultAdminUIPort = 26258 + + // DefaultNumFilesLimit is the default limit on the number of files that can + // be opened by the process. + DefaultNumFilesLimit = 65 << 13 ) // IsLocalClusterName returns true if the given name is a valid name for a local diff --git a/pkg/roachprod/install/cockroach.go b/pkg/roachprod/install/cockroach.go index 41af92237248..c327d06011f2 100644 --- a/pkg/roachprod/install/cockroach.go +++ b/pkg/roachprod/install/cockroach.go @@ -107,6 +107,9 @@ type StartOpts struct { Sequential bool ExtraArgs []string + // systemd limits on resources. + NumFilesLimit int64 + // -- Options that apply only to StartDefault target -- SkipInit bool @@ -420,21 +423,23 @@ func (c *SyncedCluster) generateStartCmd( "GOTRACEBACK=crash", "COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING=1", }, c.Env...), getEnvVars()...), - Binary: cockroachNodeBinary(c, node), - Args: args, - MemoryMax: config.MemoryMax, - Local: c.IsLocal(), + Binary: cockroachNodeBinary(c, node), + Args: args, + MemoryMax: config.MemoryMax, + NumFilesLimit: startOpts.NumFilesLimit, + Local: c.IsLocal(), }) } type startTemplateData struct { - Local bool - LogDir string - Binary string - KeyCmd string - MemoryMax string - Args []string - EnvVars []string + Local bool + LogDir string + Binary string + KeyCmd string + MemoryMax string + NumFilesLimit int64 + Args []string + EnvVars []string } func execStartTemplate(data startTemplateData) (string, error) { diff --git a/pkg/roachprod/install/scripts/start.sh b/pkg/roachprod/install/scripts/start.sh index 8a2a7fb6f8a7..8f87f9e67735 100644 --- a/pkg/roachprod/install/scripts/start.sh +++ b/pkg/roachprod/install/scripts/start.sh @@ -19,6 +19,7 @@ LOG_DIR=#{shesc .LogDir#} BINARY=#{shesc .Binary#} KEY_CMD=#{.KeyCmd#} MEMORY_MAX=#{.MemoryMax#} +NUM_FILES_LIMIT=#{.NumFilesLimit#} ARGS=( #{range .Args -#} #{shesc .#} @@ -93,5 +94,5 @@ sudo systemd-run --unit cockroach \ --service-type=notify -p NotifyAccess=all \ -p "MemoryMax=${MEMORY_MAX}" \ -p LimitCORE=infinity \ - -p LimitNOFILE=65536 \ + -p "LimitNOFILE=${NUM_FILES_LIMIT}" \ bash "${0}" run diff --git a/pkg/roachprod/install/testdata/start/start.txt b/pkg/roachprod/install/testdata/start/start.txt index 35d6040b792c..8ce8b957ffb6 100644 --- a/pkg/roachprod/install/testdata/start/start.txt +++ b/pkg/roachprod/install/testdata/start/start.txt @@ -23,6 +23,7 @@ BINARY=./cockroach KEY_CMD=echo foo && \ echo bar $HOME MEMORY_MAX=81% +NUM_FILES_LIMIT=0 ARGS=( start --log @@ -97,7 +98,7 @@ sudo systemd-run --unit cockroach \ --service-type=notify -p NotifyAccess=all \ -p "MemoryMax=${MEMORY_MAX}" \ -p LimitCORE=infinity \ - -p LimitNOFILE=65536 \ + -p "LimitNOFILE=${NUM_FILES_LIMIT}" \ bash "${0}" run ---- ---- diff --git a/pkg/roachprod/roachprod.go b/pkg/roachprod/roachprod.go index 11b2ebdcddab..f4377ffaa3df 100644 --- a/pkg/roachprod/roachprod.go +++ b/pkg/roachprod/roachprod.go @@ -635,6 +635,7 @@ func DefaultStartOpts() install.StartOpts { return install.StartOpts{ Sequential: true, EncryptedStores: false, + NumFilesLimit: config.DefaultNumFilesLimit, SkipInit: false, StoreCount: 1, TenantID: 2,