[dbode] Call syscall.Setrlimit to set num files open hard limit with setcap for DB docker image

docker/m3dbnode/Dockerfile

@@ -26,5 +26,18 @@ COPY --from=builder /go/src/github.com/m3db/m3/bin/m3dbnode /bin/
 COPY --from=builder /go/src/github.com/m3db/m3/src/dbnode/config/m3dbnode-local-etcd.yml /etc/m3dbnode/m3dbnode.yml
 COPY --from=builder /go/src/github.com/m3db/m3/scripts/m3dbnode_bootstrapped.sh /bin/
 
+# Use setcap and run as specific user
+RUN apk add libcap && \
+    mkdir -p /home/m3dbnode-user && \
+    addgroup -S m3dbnode-group && \
+    adduser -u 1000 -S -h /home/m3dbnode-user -G m3dbnode-group m3dbnode-user && \
+    chown m3dbnode-user:m3dbnode-group /bin/m3dbnode && \
+    setcap cap_ipc_lock=+ep /bin/m3dbnode && \
+    setcap cap_sys_resource=+ep /bin/m3dbnode && \
+    mkdir -p /var/lib && \
+    chown -R m3dbnode-user:m3dbnode-group /var/lib
+
+USER m3dbnode-user
+
 ENTRYPOINT [ "/bin/m3dbnode" ]
 CMD [ "-f", "/etc/m3dbnode/m3dbnode.yml" ]

src/dbnode/server/limits.go

@@ -21,10 +21,15 @@
 package server
 
 import (
+	"bufio"
 	"fmt"
+	"os/exec"
+	"strconv"
+	"strings"
+	"syscall"
 
-	xos "github.com/m3db/m3/src/x/os"
 	xerror "github.com/m3db/m3/src/x/errors"
+	xos "github.com/m3db/m3/src/x/os"
 )
 
 const (
@@ -78,3 +83,75 @@ func validateProcessLimits() error {
 
 	return multiErr.FinalError()
 }
+
+func raiseRlimitToNROpen() error {
+	cmd := exec.Command("sysctl", "-a")
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return fmt.Errorf(
+			"unable to raise nofile limits: sysctl_stdout_err=%v", err)
+	}
+
+	defer stdout.Close()
+
+	if err := cmd.Start(); err != nil {
+		return fmt.Errorf(
+			"unable to raise nofile limits: sysctl_start_err=%v", err)
+	}
+
+	var (
+		scanner = bufio.NewScanner(stdout)
+		limit   uint64
+	)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if !strings.Contains(line, "nr_open") {
+			continue
+		}
+		equalsIdx := strings.LastIndex(line, "=")
+		if equalsIdx < 0 {
+			return fmt.Errorf(
+				"unable to raise nofile limits: sysctl_parse_stdout_err=%v", err)
+		}
+		value := strings.TrimSpace(line[equalsIdx+1:])
+		n, err := strconv.Atoi(value)
+		if err != nil {
+			return fmt.Errorf(
+				"unable to raise nofile limits: sysctl_eval_stdout_err=%v", err)
+		}
+
+		limit = uint64(n)
+		break
+	}
+
+	if err := scanner.Err(); err != nil {
+		return fmt.Errorf(
+			"unable to raise nofile limits: sysctl_read_stdout_err=%v", err)
+	}
+
+	if err := cmd.Wait(); err != nil {
+		return fmt.Errorf(
+			"unable to raise nofile limits: sysctl_exec_err=%v", err)
+	}
+
+	var limits syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limits); err != nil {
+		return fmt.Errorf(
+			"unable to raise nofile limits: rlimit_get_err=%v", err)
+	}
+
+	if limits.Max >= limit && limits.Cur >= limit {
+		// Limit already set correctly
+		return nil
+	}
+
+	limits.Max = limit
+	limits.Cur = limit
+
+	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limits); err != nil {
+		return fmt.Errorf(
+			"unable to raise nofile limits: rlimit_set_err=%v", err)
+	}
+
+	return nil
+}

src/dbnode/server/server.go

@@ -150,6 +150,12 @@ func Run(runOpts RunOptions) {
 	}
 	defer logger.Sync()
 
+	// Raise soft fd limit to hard limit
+	if err := raiseRlimitToNROpen(); err != nil {
+		logger.Warn("unable to raise rlimit", zap.Error(err))
+	}
+
+	// Parse file and directory modes
 	newFileMode, err := cfg.Filesystem.ParseNewFileMode()
 	if err != nil {
 		logger.Fatal("could not parse new file mode", zap.Error(err))