[dbode] Call syscall.Setrlimit to set num files open hard limit with setcap for DB docker image

docker/m3dbnode/Dockerfile

@@ -26,5 +26,9 @@ COPY --from=builder /go/src/github.com/m3db/m3/bin/m3dbnode /bin/
 COPY --from=builder /go/src/github.com/m3db/m3/src/dbnode/config/m3dbnode-local-etcd.yml /etc/m3dbnode/m3dbnode.yml
 COPY --from=builder /go/src/github.com/m3db/m3/scripts/m3dbnode_bootstrapped.sh /bin/
 
+# Use setcap and run as specific user
+RUN apk add libcap && \
+    setcap cap_sys_resource=+ep /bin/m3dbnode
+
 ENTRYPOINT [ "/bin/m3dbnode" ]
 CMD [ "-f", "/etc/m3dbnode/m3dbnode.yml" ]

src/dbnode/server/limits.go

@@ -23,8 +23,8 @@ package server
 import (
 	"fmt"
 
-	xos "github.com/m3db/m3/src/x/os"
 	xerror "github.com/m3db/m3/src/x/errors"
+	xos "github.com/m3db/m3/src/x/os"
 )
 
 const (

src/dbnode/server/server.go

@@ -75,6 +75,7 @@ import (
 	"github.com/m3db/m3/src/x/instrument"
 	"github.com/m3db/m3/src/x/lockfile"
 	"github.com/m3db/m3/src/x/mmap"
+	xos "github.com/m3db/m3/src/x/os"
 	"github.com/m3db/m3/src/x/pool"
 	"github.com/m3db/m3/src/x/serialize"
 	xsync "github.com/m3db/m3/src/x/sync"
@@ -150,6 +151,19 @@ func Run(runOpts RunOptions) {
 	}
 	defer logger.Sync()
 
+	// Raise fd limits to nr_open system limit
+	result, err := xos.RaiseProcessNoFileToNROpen()
+	if err != nil {
+		logger.Warn("unable to raise rlimit", zap.Error(err))
+	} else {
+		logger.Info("raised rlimit no file fds limit",
+			zap.Bool("required", result.RaiseRequired),
+			zap.Uint64("sysNROpenValue", result.NROpenValue),
+			zap.Uint64("noFileMaxValue", result.NoFileMaxValue),
+			zap.Uint64("noFileCurrValue", result.NoFileCurrValue))
+	}
+
+	// Parse file and directory modes
 	newFileMode, err := cfg.Filesystem.ParseNewFileMode()
 	if err != nil {
 		logger.Fatal("could not parse new file mode", zap.Error(err))

src/x/os/limits.go

@@ -27,3 +27,12 @@ type ProcessLimits struct {
 	VMMaxMapCount int64  // corresponds to /proc/sys/vm/max_map_count
 	VMSwappiness  int64  // corresponds to /proc/sys/vm/swappiness
 }
+
+// RaiseProcessNoFileToNROpenResult captures the result of trying to
+// raise the process num files open limit to the nr_open system value.
+type RaiseProcessNoFileToNROpenResult struct {
+	RaiseRequired   bool
+	NROpenValue     uint64
+	NoFileMaxValue  uint64
+	NoFileCurrValue uint64
+}

src/x/os/limits_linux.go

@@ -32,6 +32,7 @@ const (
 	sysctlDir        = "/proc/sys/"
 	vmMaxMapCountKey = "vm.max_map_count"
 	vmSwappinessKey  = "vm.swappiness"
+	fsNROpenKey      = "fs.nr_open"
 )
 
 // CanGetProcessLimits returns a boolean to signify if it can return limits,
@@ -66,6 +67,50 @@ func GetProcessLimits() (ProcessLimits, error) {
 	}, nil
 }
 
+// RaiseProcessNoFileToNROpen first determines the NROpen limit by reading
+// the corresponding proc sys file and then if the hard or soft limits
+// are below this number, the limits are raised using a call to setrlimit.
+func RaiseProcessNoFileToNROpen() (RaiseProcessNoFileToNROpenResult, error) {
+	value, err := sysctlInt64(fsNROpenKey)
+	if err != nil {
+		return RaiseProcessNoFileToNROpenResult{}, fmt.Errorf(
+			"unable to raise nofile limits: nr_open_parse_err=%v", err)
+	}
+
+	limit := uint64(value)
+
+	var limits syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limits); err != nil {
+		return RaiseProcessNoFileToNROpenResult{}, fmt.Errorf(
+			"unable to raise nofile limits: rlimit_get_err=%v", err)
+	}
+
+	if limits.Max >= limit && limits.Cur >= limit {
+		// Limit already set correctly
+		return RaiseProcessNoFileToNROpenResult{
+			RaiseRequired:   false,
+			NROpenValue:     limit,
+			NoFileMaxValue:  limits.Max,
+			NoFileCurrValue: limits.Cur,
+		}, nil
+	}
+
+	limits.Max = limit
+	limits.Cur = limit
+
+	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limits); err != nil {
+		return RaiseProcessNoFileToNROpenResult{}, fmt.Errorf(
+			"unable to raise nofile limits: rlimit_set_err=%v", err)
+	}
+
+	return RaiseProcessNoFileToNROpenResult{
+		RaiseRequired:   true,
+		NROpenValue:     limit,
+		NoFileMaxValue:  limits.Max,
+		NoFileCurrValue: limits.Cur,
+	}, nil
+}
+
 func sysctlInt64(key string) (int64, error) {
 	str, err := sysctl(key)
 	if err != nil {

src/x/os/limits_other.go

@@ -22,14 +22,17 @@
 
 package xos
 
-import "errors"
+import (
+	"errors"
+)
 
 const (
 	nonLinuxWarning = "unable to determine process limits on non-linux os"
 )
 
 var (
-	errUnableToDetermineProcessLimits = errors.New(nonLinuxWarning)
+	errUnableToDetermineProcessLimits     = errors.New(nonLinuxWarning)
+	errUnableToRaiseProcessNoFileNonLinux = errors.New("unable to raise no file limits on non-linux os")
 )
 
 // CanGetProcessLimits returns a boolean to signify if it can return limits,
@@ -42,3 +45,9 @@ func CanGetProcessLimits() (bool, string) {
 func GetProcessLimits() (ProcessLimits, error) {
 	return ProcessLimits{}, errUnableToDetermineProcessLimits
 }
+
+// RaiseProcessNoFileToNROpen attempts to raise the process num files
+// open limit to the nr_open system value.
+func RaiseProcessNoFileToNROpen() (RaiseProcessNoFileToNROpenResult, error) {
+	return RaiseProcessNoFileToNROpenResult{}, errUnableToRaiseProcessNoFileNonLinux
+}