[Heartbeat] Setuid to regular user / lower capabilities when possible

CHANGELOG.next.asciidoc

@@ -325,6 +325,7 @@ for a few releases. Please use other tools provided by Elastic to fetch data fro
 
 - Fixed excessive memory usage introduced in 7.5 due to over-allocating memory for HTTP checks. {pull}15639[15639]
 - Fixed TCP TLS checks to properly validate hostnames, this broke in 7.x and only worked for IP SANs. {pull}17549[17549]
+- Fix broken seccomp filtering and improve security via `setcap` and `setuid` when running as root on linux in containers. {pull}27878[27878]
 
 *Journalbeat*
 

dev-tools/notice/overrides.json

@@ -11,3 +11,5 @@
 {"name": "github.com/munnerz/goautoneg", "licenceType": "BSD-3-Clause"}
 {"name": "github.com/pelletier/go-buffruneio", "licenceType": "MIT"}
 {"name": "github.com/urso/magetools", "licenceType": "Apache-2.0"}
+{"name": "kernel.org/pub/linux/libs/security/libcap/cap", "licenceType": "BSD-3-Clause", "note": "dual licensed as GPL-v2 and BSD"}
+{"name": "kernel.org/pub/linux/libs/security/libcap/psx", "licenceType": "BSD-3-Clause", "note": "dual licensed as GPL-v2 and BSD"}

dev-tools/packaging/packages.yml

@@ -432,7 +432,7 @@ shared:
       dockerfile: 'Dockerfile.elastic-agent.tmpl'
       docker_entrypoint: 'docker-entrypoint.elastic-agent.tmpl'
       user: '{{ .BeatName }}'
-      linux_capabilities: ''
+      linux_capabilities: 'cap_net_raw+eip'
       image_name: ''
     files:
       'elastic-agent.yml':

dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

@@ -17,9 +17,6 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_s
     rm {{ $beatBinary }} && \
     ln -s {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/elastic-agent {{ $beatBinary }} && \
     chmod 0755 {{ $beatHome }}/data/elastic-agent-*/elastic-agent && \
-{{- if .linux_capabilities }}
-    setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
-{{- end }}
 {{- range $i, $modulesd := .ModulesDirs }}
     chmod 0775 {{ $beatHome}}/{{ $modulesd }} && \
 {{- end }}
@@ -30,11 +27,20 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_s
 {{- end }}
     true
 
+{{- if .linux_capabilities }}
+# Since the beat is stored at the other end of a symlink we must follow the symlink first
+# For security reasons setcap does not support symlinks. This is smart in the general case
+# but in our specific case since we're building a trusted image from trusted binaries this is
+# fine. Thus, we use readlink to follow the link and setcap on the actual binary
+RUN readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }}
+{{- end }}
+
 FROM {{ .from }}
 
 # Contains the elastic agent image variant, an empty string for the standard variant
 # or "complete" for the bigger one.
 ENV ELASTIC_AGENT_IMAGE_VARIANT={{.Variant}}
+ENV BEAT_SETUID_AS={{ .user }}
 
 {{- if contains .from "ubi-minimal" }}
 RUN for iter in {1..10}; do microdnf update -y && microdnf install -y shadow-utils jq &&  microdnf clean all && exit_code=0 && break || exit_code=$? && echo "microdnf error: retry $iter in 10s" && sleep 10; done; (exit $exit_code)

dev-tools/packaging/templates/docker/Dockerfile.tmpl

@@ -13,14 +13,19 @@ RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \
     find {{ $beatHome }} -type d -exec chmod 0755 {} \; && \
     find {{ $beatHome }} -type f -exec chmod 0644 {} \; && \
     chmod 0755 {{ $beatBinary }} && \
-{{- if .linux_capabilities }}
-    setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
-{{- end }}
 {{- range $i, $modulesd := .ModulesDirs }}
     chmod 0775 {{ $beatHome}}/{{ $modulesd }} && \
 {{- end }}
     chmod 0775 {{ $beatHome }}/data {{ $beatHome }}/logs
 
+{{- if .linux_capabilities }}
+# Since the beat is stored at the other end of a symlink we must follow the symlink first
+# For security reasons setcap does not support symlinks. This is smart in the general case
+# but in our specific case since we're building a trusted image from trusted binaries this is
+# fine. Thus, we use readlink to follow the link and setcap on the actual binary
+RUN readlink -f {{ $beatBinary }} | xargs setcap {{ .linux_capabilities }}
+{{- end }}
+
 FROM {{ .from }}
 
 {{- if contains .from "ubi-minimal" }}
@@ -127,6 +132,7 @@ USER {{ .user }}
 {{- if (and (eq .BeatName "heartbeat") (not (contains .from "ubi-minimal")))  }}
 # Setup synthetics env vars
 ENV ELASTIC_SYNTHETICS_CAPABLE=true
+ENV BEAT_SETUID_AS={{ .user }}
 ENV SUITES_DIR={{ $beatHome }}/suites
 ENV NODE_VERSION=14.17.5
 ENV PATH="$NODE_PATH/node/bin:$PATH"

go.mod

@@ -192,6 +192,7 @@ require (
 	k8s.io/api v0.21.1
 	k8s.io/apimachinery v0.21.1
 	k8s.io/client-go v0.21.1
+	kernel.org/pub/linux/libs/security/libcap/cap v1.2.57
 )
 
 require (
@@ -283,6 +284,7 @@ require (
 	k8s.io/klog/v2 v2.8.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 // indirect
 	k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect
+	kernel.org/pub/linux/libs/security/libcap/psx v1.2.57 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.1.0 // indirect
 	sigs.k8s.io/yaml v1.2.0 // indirect
 )

go.sum

@@ -1344,6 +1344,10 @@ k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0=
 k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
 k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
+kernel.org/pub/linux/libs/security/libcap/cap v1.2.57 h1:2nmqI+aw7EQZuelYktkQHBE4jESD2tOR+lOJEnv/Apo=
+kernel.org/pub/linux/libs/security/libcap/cap v1.2.57/go.mod h1:uI99C3r4SXvJeuqoEtx/eWt7UbmfqqZ80H8q+9t/A7I=
+kernel.org/pub/linux/libs/security/libcap/psx v1.2.57 h1:NOFATXSf5z/cMR3HIwQ3Xrd3nwnWl5xThmNr5U/F0pI=
+kernel.org/pub/linux/libs/security/libcap/psx v1.2.57/go.mod h1:+l6Ee2F59XiJ2I6WR5ObpC1utCQJZ/VLsEbQCD8RG24=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

heartbeat/beater/heartbeat.go

@@ -20,6 +20,7 @@ package beater
 import (
 	"errors"
 	"fmt"
+	"syscall"
 	"time"
 
 	"github.com/elastic/beats/v7/heartbeat/config"
@@ -81,6 +82,9 @@ func New(b *beat.Beat, rawConfig *common.Config) (beat.Beater, error) {
 func (bt *Heartbeat) Run(b *beat.Beat) error {
 	logp.Info("heartbeat is running! Hit CTRL-C to stop it.")
 
+	groups, _ := syscall.Getgroups()
+	logp.Info("Effective user/group ids: %d/%d, with groups: %v", syscall.Geteuid(), syscall.Getegid(), groups)
+
 	stopStaticMonitors, err := bt.RunStaticMonitors(b)
 	if err != nil {
 		return err

heartbeat/scripts/mage/package.go

@@ -48,7 +48,7 @@ func CustomizePackaging() {
 		pkgType := args.Types[0]
 		switch pkgType {
 		case devtools.Docker:
-			args.Spec.ExtraVar("linux_capabilities", "cap_net_raw=eip")
+			args.Spec.ExtraVar("linux_capabilities", "cap_net_raw+eip")
 			args.Spec.Files[monitorsDTarget] = monitorsD
 		case devtools.TarGz, devtools.Zip:
 			args.Spec.Files[monitorsDTarget] = monitorsD

heartbeat/security.go

@@ -0,0 +1,237 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// TODO: Support other architectures (e.g. arm)
+//go:build linux
+// +build linux
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/user"
+	"runtime"
+	"strconv"
+	"syscall"
+
+	"kernel.org/pub/linux/libs/security/libcap/cap"
+
+	"github.com/elastic/beats/v7/libbeat/common/seccomp"
+)
+
+func init() {
+	// Here we set a bunch of linux specific security stuff.
+	// In the context of a container, where users frequently run as root, we follow BEAT_SETUID_AS to setuid/gid
+	// and add capabilities to make this actually run as a regular user. This also helps Node.js in synthetics, which
+	// does not want to run as root. It's also just generally more secure.
+	if localUserName := os.Getenv("BEAT_SETUID_AS"); localUserName != "" && syscall.Geteuid() == 0 {
+		err := changeUser(localUserName)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	// Attempt to set capabilities before we setup seccomp rules
+	// Note that we discard any errors because they are not actionable.
+	// The beat should use `getcap` at a later point to examine available capabilities
+	// rather than relying on errors from `setcap`
+	setCapabilities()
+
+	switch runtime.GOARCH {
+	case "amd64", "386":
+		err := setSeccompRules()
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+
+func changeUser(localUserName string) error {
+	localUser, err := user.Lookup(localUserName)
+	if err != nil {
+		return fmt.Errorf("could not lookup '%s': %w", localUser, err)
+	}
+	localUserUid, err := strconv.Atoi(localUser.Uid)
+	if err != nil {
+		return fmt.Errorf("could not parse UID '%s' as int: %w", localUser.Uid, err)
+	}
+	localUserGid, err := strconv.Atoi(localUser.Gid)
+	if err != nil {
+		return fmt.Errorf("could not parse GID '%s' as int: %w", localUser.Uid, err)
+	}
+	// We include the root group because the docker image contains many directories (data,logs)
+	// that are owned by root:root with 0775 perms. The heartbeat user is in both groups
+	// in the container, but we need to repeat that here.
+	err = syscall.Setgroups([]int{localUserGid, 0})
+	if err != nil {
+		return fmt.Errorf("could not set groups: %w", err)
+	}
+
+	// Set the main group as localUserUid so new files created are owned by the user's group
+	err = syscall.Setgid(localUserGid)
+	if err != nil {
+		return fmt.Errorf("could not set gid to %d: %w", localUserGid, err)
+	}
+
+	// Note this is not the regular SetUID! Look at the 'cap' package docs for it, it preserves
+	// capabilities post-SetUID, which we use to lock things down immediately
+	err = cap.SetUID(localUserUid)
+	if err != nil {
+		return fmt.Errorf("could not setuid to %d: %w", localUserUid, err)
+	}
+
+	// This may not be necessary, but is good hygeine, we do some shelling out to node/npm etc.
+	// and $HOME should reflect the user's preferences
+	return os.Setenv("HOME", localUser.HomeDir)
+}
+
+func setCapabilities() error {
+	// Start with an empty capability set
+	newcaps := cap.NewSet()
+	// Both permitted and effective are required! Permitted makes the permmission
+	// possible to get, effective makes it 'active'
+	err := newcaps.SetFlag(cap.Permitted, true, cap.NET_RAW)
+	if err != nil {
+		return fmt.Errorf("error setting permitted setcap: %w", err)
+	}
+	err = newcaps.SetFlag(cap.Effective, true, cap.NET_RAW)
+	if err != nil {
+		return fmt.Errorf("error setting effective setcap: %w", err)
+	}
+
+	// We do not want these capabilities to be inherited by subprocesses
+	err = newcaps.SetFlag(cap.Inheritable, false, cap.NET_RAW)
+	if err != nil {
+		return fmt.Errorf("error setting inheritable setcap: %w", err)
+	}
+
+	// Apply the new capabilities to the current process (incl. all threads)
+	err = newcaps.SetProc()
+	if err != nil {
+		return fmt.Errorf("error setting new process capabilities via setcap: %w", err)
+	}
+
+	return nil
+}
+
+func setSeccompRules() error {
+	// We require a number of syscalls to run. This list was generated with
+	// mage build && env ELASTIC_SYNTHETICS_CAPABLE=true strace -f --output=syscalls  ./heartbeat --path.config sample-synthetics-config/ -e
+	// then grepping for 'EPERM' in the 'syscalls' file.
+	syscalls := []string{
+		"access",
+		"arch_prctl",
+		"bind",
+		"brk",
+		"capget",
+		"chdir",
+		"chmod",
+		"chown",
+		"clone",
+		"close",
+		"connect",
+		"creat",
+		"dup2",
+		"epoll_ctl",
+		"epoll_pwait",
+		"eventfd2",
+		"execve",
+		"exit",
+		"faccessat",
+		"fadvise64",
+		"fallocate",
+		"fcntl",
+		"flock",
+		"fstat",
+		"fsync",
+		"futex",
+		"capget",
+		"getcwd",
+		"getdents64",
+		"getegid",
+		"geteuid",
+		"getgid",
+		"getpeername",
+		"getpgrp",
+		"getpid",
+		"getppid",
+		"getpriority",
+		"getrandom",
+		"getresuid",
+		"getresgid",
+		"getrusage",
+		"getsockname",
+		"gettid",
+		"getuid",
+		"ioctl",
+		"inotify_init",
+		"lchown",
+		"link",
+		"lseek",
+		"madvise",
+		"memfd_create",
+		"mkdir",
+		"mkdirat",
+		"mlock",
+		"mmap",
+		"mprotect",
+		"munmap",
+		"nanosleep",
+		"name_to_handle_at",
+		"newfstatat",
+		"openat",
+		"pipe",
+		"pipe2",
+		"poll",
+		"prctl",
+		"pread64",
+		"prlimit64",
+		"pwrite64",
+		"read",
+		"readlink",
+		"readlinkat",
+		"recvfrom",
+		"rename",
+		"rmdir",
+		"rt_sigaction",
+		"rt_sigprocmask",
+		"rt_sigreturn",
+		"sched_getaffinity",
+		"sched_getparam",
+		"sched_getscheduler",
+		"select",
+		"sendto",
+		"set_robust_list",
+		"set_tid_address",
+		"setpriority",
+		"setsid",
+		"sigaltstack",
+		"socket",
+		"socketpair",
+		"stat",
+		"statx",
+		"symlink",
+		"umask",
+		"uname",
+		"unlink",
+		"utimensat",
+		"write",
+	}
+
+	return seccomp.ModifyDefaultPolicy(seccomp.AddSyscall, syscalls...)
+}

packetbeat/scripts/mage/package.go

@@ -60,7 +60,7 @@ func CustomizePackaging() {
 				args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.yml", configYml)
 				args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.reference.yml", referenceConfigYml)
 			case devtools.Docker:
-				args.Spec.ExtraVar("linux_capabilities", "cap_net_raw,cap_net_admin=eip")
+				args.Spec.ExtraVar("linux_capabilities", "cap_net_raw,cap_net_admin+eip")
 			default:
 				panic(errors.Errorf("unhandled package type: %v", pkgType))
 			}