Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
76076: roachprod: Make it possible to specify file descriptor limit. r=miretskiy a=miretskiy

Add a `--num-files-limit` flag to roachprod start command
to change the default file descriptor limit for cockroach process.

Release Notes: None

Co-authored-by: Yevgeniy Miretskiy <[email protected]>
  • Loading branch information
craig[bot] and Yevgeniy Miretskiy committed Feb 6, 2022
2 parents 4ea464d + 5af48f0 commit 9a7e652
Show file tree
Hide file tree
Showing 6 changed files with 27 additions and 13 deletions.
2 changes: 2 additions & 0 deletions pkg/cmd/roachprod/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,8 @@ func initFlags() {
for _, cmd := range []*cobra.Command{startCmd, startTenantCmd} {
cmd.Flags().BoolVar(&startOpts.Sequential,
"sequential", startOpts.Sequential, "start nodes sequentially so node IDs match hostnames")
cmd.Flags().Int64Var(&startOpts.NumFilesLimit, "num-files-limit", startOpts.NumFilesLimit,
"limit the number of files that can be created by the cockroach process")
}

for _, cmd := range []*cobra.Command{
Expand Down
4 changes: 4 additions & 0 deletions pkg/roachprod/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,10 @@ const (
// DefaultAdminUIPort is the default port on which the cockroach process is
// listening for HTTP connections for the Admin UI.
DefaultAdminUIPort = 26258

// DefaultNumFilesLimit is the default limit on the number of files that can
// be opened by the process.
DefaultNumFilesLimit = 65 << 13
)

// IsLocalClusterName returns true if the given name is a valid name for a local
Expand Down
27 changes: 16 additions & 11 deletions pkg/roachprod/install/cockroach.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,9 @@ type StartOpts struct {
Sequential bool
ExtraArgs []string

// systemd limits on resources.
NumFilesLimit int64

// -- Options that apply only to StartDefault target --

SkipInit bool
Expand Down Expand Up @@ -420,21 +423,23 @@ func (c *SyncedCluster) generateStartCmd(
"GOTRACEBACK=crash",
"COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING=1",
}, c.Env...), getEnvVars()...),
Binary: cockroachNodeBinary(c, node),
Args: args,
MemoryMax: config.MemoryMax,
Local: c.IsLocal(),
Binary: cockroachNodeBinary(c, node),
Args: args,
MemoryMax: config.MemoryMax,
NumFilesLimit: startOpts.NumFilesLimit,
Local: c.IsLocal(),
})
}

type startTemplateData struct {
Local bool
LogDir string
Binary string
KeyCmd string
MemoryMax string
Args []string
EnvVars []string
Local bool
LogDir string
Binary string
KeyCmd string
MemoryMax string
NumFilesLimit int64
Args []string
EnvVars []string
}

func execStartTemplate(data startTemplateData) (string, error) {
Expand Down
3 changes: 2 additions & 1 deletion pkg/roachprod/install/scripts/start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ LOG_DIR=#{shesc .LogDir#}
BINARY=#{shesc .Binary#}
KEY_CMD=#{.KeyCmd#}
MEMORY_MAX=#{.MemoryMax#}
NUM_FILES_LIMIT=#{.NumFilesLimit#}
ARGS=(
#{range .Args -#}
#{shesc .#}
Expand Down Expand Up @@ -93,5 +94,5 @@ sudo systemd-run --unit cockroach \
--service-type=notify -p NotifyAccess=all \
-p "MemoryMax=${MEMORY_MAX}" \
-p LimitCORE=infinity \
-p LimitNOFILE=65536 \
-p "LimitNOFILE=${NUM_FILES_LIMIT}" \
bash "${0}" run
3 changes: 2 additions & 1 deletion pkg/roachprod/install/testdata/start/start.txt
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ BINARY=./cockroach
KEY_CMD=echo foo && \
echo bar $HOME
MEMORY_MAX=81%
NUM_FILES_LIMIT=0
ARGS=(
start
--log
Expand Down Expand Up @@ -97,7 +98,7 @@ sudo systemd-run --unit cockroach \
--service-type=notify -p NotifyAccess=all \
-p "MemoryMax=${MEMORY_MAX}" \
-p LimitCORE=infinity \
-p LimitNOFILE=65536 \
-p "LimitNOFILE=${NUM_FILES_LIMIT}" \
bash "${0}" run
----
----
1 change: 1 addition & 0 deletions pkg/roachprod/roachprod.go
Original file line number Diff line number Diff line change
Expand Up @@ -635,6 +635,7 @@ func DefaultStartOpts() install.StartOpts {
return install.StartOpts{
Sequential: true,
EncryptedStores: false,
NumFilesLimit: config.DefaultNumFilesLimit,
SkipInit: false,
StoreCount: 1,
TenantID: 2,
Expand Down

0 comments on commit 9a7e652

Please sign in to comment.