libbeat: override /sys/fs/cgroup paths in Docker (#22879) (#23050)

* libbeat: override /sys/fs/cgroup paths in Docker When running in Docker, make sure metrics are read from `/sys/fs/cgroup/<subsystem>`, ignoring any paths in `/proc/self/cgroup`. Fixes #22844 (cherry picked from commit 7e1520f)
elastic · Dec 16, 2020 · 190be26 · 190be26
1 parent 2b4c6b6
commit 190be26
Show file tree

Hide file tree

Showing 6 changed files with 21 additions and 6 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -160,6 +160,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Periodic metrics in logs will now report `libbeat.output.events.active` and `beat.memstats.rss`
   as gauges (rather than counters). {pull}22877[22877]
 - Use PROGRAMDATA environment variable instead of C:\ProgramData for windows install service {pull}22874[22874]
+- Fix reporting of cgroup metrics when running under Docker {pull}22879[22879]
 
 
 *Auditbeat*

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -8267,11 +8267,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]/
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/gosigar
-Version: v0.12.0
+Version: v0.13.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/gosigar@v0.12.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/gosigar@v0.13.0/LICENSE:
 
                               Apache License
                         Version 2.0, January 2004

diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl
@@ -89,6 +89,11 @@ USER {{ .user }}
 EXPOSE {{ $port }}
 {{- end }}
 
+# When running under Docker, we must ensure libbeat monitoring pulls cgroup
+# metrics from /sys/fs/cgroup/<subsystem>/, ignoring any paths found in
+# /proc/self/cgroup.
+ENV LIBBEAT_MONITORING_CGROUPS_HIERARCHY_OVERRIDE=/
+
 WORKDIR {{ $beatHome }}
 ENTRYPOINT ["/usr/bin/tini", "--", "/usr/local/bin/docker-entrypoint"]
 CMD ["-environment", "container"]
diff --git a/go.mod b/go.mod
@@ -72,7 +72,7 @@ require (
 	github.com/elastic/go-txfile v0.0.7
 	github.com/elastic/go-ucfg v0.8.3
 	github.com/elastic/go-windows v1.0.1 // indirect
-	github.com/elastic/gosigar v0.12.0
+	github.com/elastic/gosigar v0.13.0
 	github.com/fatih/color v1.5.0
 	github.com/fsnotify/fsevents v0.1.1
 	github.com/fsnotify/fsnotify v1.4.9

diff --git a/go.sum b/go.sum
@@ -278,8 +278,8 @@ github.com/elastic/go-ucfg v0.8.3/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+F
 github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU=
 github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0=
 github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss=
-github.com/elastic/gosigar v0.12.0 h1:AsdhYCJlTudhfOYQyFNgx+fIVTfrDO0V1ST0vHgiapU=
-github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
+github.com/elastic/gosigar v0.13.0 h1:EIeuQcLPKia759s6mlVztlxUyKiKYHo6y6kOODOLO7A=
+github.com/elastic/gosigar v0.13.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
 github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec h1:rAHd7DeHIHjSzvnkl197GKh9TCWGKg/z2BBbbGOEiWI=
 github.com/elastic/sarama v1.19.1-0.20200629123429-0e7b69039eec/go.mod h1:X690XXMxlbtN8c7xcpsENKNlbj8VClCZ2hwSOhSyNmE=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=

diff --git a/libbeat/cmd/instance/metrics/metrics.go b/libbeat/cmd/instance/metrics/metrics.go
@@ -21,6 +21,7 @@ package metrics
 
 import (
 	"fmt"
+	"os"
 	"runtime"
 
 	"github.com/elastic/beats/v7/libbeat/common"
@@ -36,6 +37,11 @@ var (
 	systemMetrics    *monitoring.Registry
 )
 
+// libbeatMonitoringCgroupsHierarchyOverride is an undocumented environment variable which
+// overrides the cgroups path under /sys/fs/cgroup, which should be set to "/" when running
+// Beats under Docker.
+const libbeatMonitoringCgroupsHierarchyOverride = "LIBBEAT_MONITORING_CGROUPS_HIERARCHY_OVERRIDE"
+
 func init() {
 	systemMetrics = monitoring.Default.NewRegistry("system")
 }
@@ -278,7 +284,10 @@ func reportBeatCgroups(_ monitoring.Mode, V monitoring.Visitor) {
 		return
 	}
 
-	cgroups, err := cgroup.NewReader("", true)
+	cgroups, err := cgroup.NewReaderOptions(cgroup.ReaderOptions{
+		IgnoreRootCgroups:        true,
+		CgroupsHierarchyOverride: os.Getenv(libbeatMonitoringCgroupsHierarchyOverride),
+	})
 	if err != nil {
 		if err == cgroup.ErrCgroupsMissing {
 			logp.Warn("cgroup data collection disabled: %v", err)