From b9d0932179a0c5b3a8863f3d6cdafe8584cedc8e Mon Sep 17 00:00:00 2001 From: Miguel Oliveira <41488876+migeyel@users.noreply.github.com> Date: Sun, 14 Jul 2024 11:33:12 -0300 Subject: [PATCH] Add node_filesystem_mount_info metric (#2970) * Add node_filesystem_mount_info metric Fixes: #1384 --------- Signed-off-by: Miguel Oliveira --- collector/filesystem_common.go | 15 +++++++- collector/filesystem_linux.go | 35 +++++++++++++------ collector/filesystem_linux_test.go | 8 +++++ collector/fixtures/proc/1/mountinfo | 31 ++++++++++++++++ collector/fixtures/proc/1/mounts | 32 ----------------- collector/fixtures_bindmount/proc/1/mountinfo | 6 ++++ collector/fixtures_bindmount/proc/mounts | 6 ---- collector/fixtures_hidepid/proc/mounts | 1 - .../fixtures_hidepid/proc/self/mountinfo | 1 + 9 files changed, 84 insertions(+), 51 deletions(-) create mode 100644 collector/fixtures/proc/1/mountinfo delete mode 100644 collector/fixtures/proc/1/mounts create mode 100644 collector/fixtures_bindmount/proc/1/mountinfo delete mode 100644 collector/fixtures_bindmount/proc/mounts delete mode 100644 collector/fixtures_hidepid/proc/mounts create mode 100644 collector/fixtures_hidepid/proc/self/mountinfo diff --git a/collector/filesystem_common.go b/collector/filesystem_common.go index f5d5135241..d224e2c20f 100644 --- a/collector/filesystem_common.go +++ b/collector/filesystem_common.go @@ -69,11 +69,12 @@ type filesystemCollector struct { sizeDesc, freeDesc, availDesc *prometheus.Desc filesDesc, filesFreeDesc *prometheus.Desc roDesc, deviceErrorDesc *prometheus.Desc + mountInfoDesc *prometheus.Desc logger log.Logger } type filesystemLabels struct { - device, mountPoint, fsType, options, deviceError string + device, mountPoint, fsType, options, deviceError, major, minor string } type filesystemStats struct { @@ -155,6 +156,13 @@ func NewFilesystemCollector(logger log.Logger) (Collector, error) { filesystemLabelNames, nil, ) + mountInfoDesc := prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "mount_info"), + "Filesystem mount information.", + []string{"device", "major", "minor", "mountpoint"}, + nil, + ) + return &filesystemCollector{ excludedMountPointsPattern: mountPointPattern, excludedFSTypesPattern: filesystemsTypesPattern, @@ -165,6 +173,7 @@ func NewFilesystemCollector(logger log.Logger) (Collector, error) { filesFreeDesc: filesFreeDesc, roDesc: roDesc, deviceErrorDesc: deviceErrorDesc, + mountInfoDesc: mountInfoDesc, logger: logger, }, nil } @@ -215,6 +224,10 @@ func (c *filesystemCollector) Update(ch chan<- prometheus.Metric) error { c.filesFreeDesc, prometheus.GaugeValue, s.filesFree, s.labels.device, s.labels.mountPoint, s.labels.fsType, s.labels.deviceError, ) + ch <- prometheus.MustNewConstMetric( + c.mountInfoDesc, prometheus.GaugeValue, + 1.0, s.labels.device, s.labels.major, s.labels.minor, s.labels.mountPoint, + ) } return nil } diff --git a/collector/filesystem_linux.go b/collector/filesystem_linux.go index 1d0c8493b9..728e8cca43 100644 --- a/collector/filesystem_linux.go +++ b/collector/filesystem_linux.go @@ -178,11 +178,11 @@ func stuckMountWatcher(mountPoint string, success chan struct{}, logger log.Logg } func mountPointDetails(logger log.Logger) ([]filesystemLabels, error) { - file, err := os.Open(procFilePath("1/mounts")) + file, err := os.Open(procFilePath("1/mountinfo")) if errors.Is(err, os.ErrNotExist) { - // Fallback to `/proc/mounts` if `/proc/1/mounts` is missing due hidepid. - level.Debug(logger).Log("msg", "Reading root mounts failed, falling back to system mounts", "err", err) - file, err = os.Open(procFilePath("mounts")) + // Fallback to `/proc/self/mountinfo` if `/proc/1/mountinfo` is missing due hidepid. + level.Debug(logger).Log("msg", "Reading root mounts failed, falling back to self mounts", "err", err) + file, err = os.Open(procFilePath("self/mountinfo")) } if err != nil { return nil, err @@ -199,20 +199,33 @@ func parseFilesystemLabels(r io.Reader) ([]filesystemLabels, error) { for scanner.Scan() { parts := strings.Fields(scanner.Text()) - if len(parts) < 4 { + if len(parts) < 10 { return nil, fmt.Errorf("malformed mount point information: %q", scanner.Text()) } + major, minor := 0, 0 + _, err := fmt.Sscanf(parts[2], "%d:%d", &major, &minor) + if err != nil { + return nil, fmt.Errorf("malformed mount point information: %q", scanner.Text()) + } + + m := 5 + for parts[m+1] != "-" { + m++ + } + // Ensure we handle the translation of \040 and \011 // as per fstab(5). - parts[1] = strings.Replace(parts[1], "\\040", " ", -1) - parts[1] = strings.Replace(parts[1], "\\011", "\t", -1) + parts[4] = strings.Replace(parts[4], "\\040", " ", -1) + parts[4] = strings.Replace(parts[4], "\\011", "\t", -1) filesystems = append(filesystems, filesystemLabels{ - device: parts[0], - mountPoint: rootfsStripPrefix(parts[1]), - fsType: parts[2], - options: parts[3], + device: parts[m+3], + mountPoint: rootfsStripPrefix(parts[4]), + fsType: parts[m+2], + options: parts[5], + major: fmt.Sprint(major), + minor: fmt.Sprint(minor), deviceError: "", }) } diff --git a/collector/filesystem_linux_test.go b/collector/filesystem_linux_test.go index 325ffc87b2..e29385527c 100644 --- a/collector/filesystem_linux_test.go +++ b/collector/filesystem_linux_test.go @@ -87,10 +87,18 @@ func TestMountPointDetails(t *testing.T) { t.Log(err) } + foundSet := map[string]bool{} for _, fs := range filesystems { if _, ok := expected[fs.mountPoint]; !ok { t.Errorf("Got unexpected %s", fs.mountPoint) } + foundSet[fs.mountPoint] = true + } + + for mountPoint := range expected { + if _, ok := foundSet[mountPoint]; !ok { + t.Errorf("Expected %s, got nothing", mountPoint) + } } } diff --git a/collector/fixtures/proc/1/mountinfo b/collector/fixtures/proc/1/mountinfo new file mode 100644 index 0000000000..11c50a6fb0 --- /dev/null +++ b/collector/fixtures/proc/1/mountinfo @@ -0,0 +1,31 @@ +24 29 0:22 / /sys rw,nosuid,nodev,noexec,relatime shared:7 - sysfs sysfs rw +25 29 0:23 / /proc rw,nosuid,nodev,noexec,relatime shared:13 - proc proc rw +26 29 0:5 / /dev rw,nosuid,relatime shared:2 - devtmpfs udev rw,size=7978892k,nr_inodes=1994723,mode=755 +27 26 0:24 / /dev/pts rw,nosuid,noexec,relatime shared:3 - devpts devpts rw,gid=5,mode=620,ptmxmode=000 +28 29 0:25 / /run rw,nosuid,relatime shared:5 - tmpfs tmpfs rw,size=1617716k,mode=755 +29 1 259:2 / / rw,relatime shared:1 - ext4 /dev/dm-2 errors=remount-ro,data=ordered +30 24 0:6 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime shared:8 - securityfs securityfs rw +31 26 0:26 / /dev/shm rw,nosuid,nodev shared:4 - tmpfs tmpfs rw,inode64 +32 28 0:27 / /run/lock rw,nosuid,nodev,noexec,relatime shared:6 - tmpfs tmpfs rw,size=5120k +33 24 0:28 / /sys/fs/cgroup ro,nosuid,nodev,noexec shared:9 - tmpfs tmpfs ro,mode=755 +34 31 0:24 / /sys/fs/cgroup/systemd rw,nosuid,nodev,noexec,relatime shared:10 - cgroup cgroup rw,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd +35 32 0:25 / /sys/fs/pstore rw,nosuid,nodev,noexec,relatime shared:11 - pstore pstore rw +36 33 0:26 / /sys/fs/cgroup/cpuset rw,nosuid,nodev,noexec,relatime shared:12 - cgroup cgroup rw,cpuset +37 34 0:27 / /sys/fs/cgroup/cpu,cpuacct rw,nosuid,nodev,noexec,relatime shared:14 - cgroup cgroup rw,cpu,cpuacct +38 35 0:28 / /sys/fs/cgroup/devices rw,nosuid,nodev,noexec,relatime shared:16 - cgroup cgroup rw,devices +39 36 0:29 / /sys/fs/cgroup/freezer rw,nosuid,nodev,noexec,relatime shared:17 - cgroup cgroup rw,freezer +40 37 0:30 / /sys/fs/cgroup/net_cls,net_prio rw,nosuid,nodev,noexec,relatime shared:18 - cgroup cgroup rw,net_cls,net_prio +41 38 0:31 / /sys/fs/cgroup/blkio rw,nosuid,nodev,noexec,relatime shared:19 - cgroup cgroup rw,blkio +42 39 0:32 / /sys/fs/cgroup/perf_event rw,nosuid,nodev,noexec,relatime shared:20 - cgroup cgroup rw,perf_event +43 40 0:33 / /proc/sys/fs/binfmt_misc rw,relatime shared:21 - systemd-1 autofs rw,fd=22,pgrp=1,timeout=300,minproto=5,maxproto=5,direct +44 41 0:34 / /dev/mqueue rw,relatime shared:22 - mqueue mqueue rw +45 42 0:35 / /sys/kernel/debug rw,relatime shared:23 - debugfs debugfs rw +46 43 0:36 / /dev/hugepages rw,relatime shared:24 - hugetlbfs hugetlbfs rw +47 44 0:37 / /sys/fs/fuse/connections rw,relatime shared:25 - fusectl fusectl rw +48 45 260:3 / /boot rw,relatime shared:92 - ext2 /dev/sda3 rw +49 46 0:39 / /run/rpc_pipefs rw,relatime shared:27 - rpc_pipefs rpc_pipefs rw +265 37 0:41 / /proc/sys/fs/binfmt_misc rw,nosuid,nodev,noexec,relatime shared:94 - binfmt_misc binfmt_misc rw +3002 28 0:79 / /run/user/1000 rw,nosuid,nodev,relatime shared:1225 - tmpfs tmpfs rw,size=1603436k,nr_inodes=400859,mode=700,uid=1000,gid=1000 +3147 3002 0:81 / /run/user/1000/gvfs rw,nosuid,nodev,relatime shared:1290 - fuse.gvfsd-fuse gvfsd-fuse rw,user_id=1000,group_id=1000 +3148 3003 260:0 / /var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[vsanDatastore]\040bafb9e5a-8856-7e6c-699c-801844e77a4a/kubernetes-dynamic-pvc-3eba5bba-48a3-11e8-89ab-005056b92113.vmdk rw,relatime shared:31 - ext4 /dev/sda rw,data=ordered +3149 3004 260:0 / /var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[vsanDatastore]\011bafb9e5a-8856-7e6c-699c-801844e77a4a/kubernetes-dynamic-pvc-3eba5bba-48a3-11e8-89ab-005056b92113.vmdk rw,relatime shared:32 - ext4 /dev/sda rw,data=ordered diff --git a/collector/fixtures/proc/1/mounts b/collector/fixtures/proc/1/mounts deleted file mode 100644 index 7452d495a6..0000000000 --- a/collector/fixtures/proc/1/mounts +++ /dev/null @@ -1,32 +0,0 @@ -rootfs / rootfs rw 0 0 -sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0 -proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0 -udev /dev devtmpfs rw,relatime,size=10240k,nr_inodes=1008585,mode=755 0 0 -devpts /dev/pts devpts rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0 -tmpfs /run tmpfs rw,nosuid,relatime,size=1617716k,mode=755 0 0 -/dev/dm-2 / ext4 rw,relatime,errors=remount-ro,data=ordered 0 0 -securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0 -tmpfs /dev/shm tmpfs rw,nosuid,nodev 0 0 -tmpfs /run/lock tmpfs rw,nosuid,nodev,noexec,relatime,size=5120k 0 0 -tmpfs /sys/fs/cgroup tmpfs ro,nosuid,nodev,noexec,mode=755 0 0 -cgroup /sys/fs/cgroup/systemd cgroup rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd 0 0 -pstore /sys/fs/pstore pstore rw,nosuid,nodev,noexec,relatime 0 0 -cgroup /sys/fs/cgroup/cpuset cgroup rw,nosuid,nodev,noexec,relatime,cpuset 0 0 -cgroup /sys/fs/cgroup/cpu,cpuacct cgroup rw,nosuid,nodev,noexec,relatime,cpu,cpuacct 0 0 -cgroup /sys/fs/cgroup/devices cgroup rw,nosuid,nodev,noexec,relatime,devices 0 0 -cgroup /sys/fs/cgroup/freezer cgroup rw,nosuid,nodev,noexec,relatime,freezer 0 0 -cgroup /sys/fs/cgroup/net_cls,net_prio cgroup rw,nosuid,nodev,noexec,relatime,net_cls,net_prio 0 0 -cgroup /sys/fs/cgroup/blkio cgroup rw,nosuid,nodev,noexec,relatime,blkio 0 0 -cgroup /sys/fs/cgroup/perf_event cgroup rw,nosuid,nodev,noexec,relatime,perf_event 0 0 -systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=22,pgrp=1,timeout=300,minproto=5,maxproto=5,direct 0 0 -mqueue /dev/mqueue mqueue rw,relatime 0 0 -debugfs /sys/kernel/debug debugfs rw,relatime 0 0 -hugetlbfs /dev/hugepages hugetlbfs rw,relatime 0 0 -fusectl /sys/fs/fuse/connections fusectl rw,relatime 0 0 -/dev/sda3 /boot ext2 rw,relatime 0 0 -rpc_pipefs /run/rpc_pipefs rpc_pipefs rw,relatime 0 0 -binfmt_misc /proc/sys/fs/binfmt_misc binfmt_misc rw,relatime 0 0 -tmpfs /run/user/1000 tmpfs rw,nosuid,nodev,relatime,size=808860k,mode=700,uid=1000,gid=1000 0 0 -gvfsd-fuse /run/user/1000/gvfs fuse.gvfsd-fuse rw,nosuid,nodev,relatime,user_id=1000,group_id=1000 0 0 -/dev/sda /var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[vsanDatastore]\040bafb9e5a-8856-7e6c-699c-801844e77a4a/kubernetes-dynamic-pvc-3eba5bba-48a3-11e8-89ab-005056b92113.vmdk ext4 rw,relatime,data=ordered 0 0 -/dev/sda /var/lib/kubelet/plugins/kubernetes.io/vsphere-volume/mounts/[vsanDatastore]\011bafb9e5a-8856-7e6c-699c-801844e77a4a/kubernetes-dynamic-pvc-3eba5bba-48a3-11e8-89ab-005056b92113.vmdk ext4 rw,relatime,data=ordered 0 0 diff --git a/collector/fixtures_bindmount/proc/1/mountinfo b/collector/fixtures_bindmount/proc/1/mountinfo new file mode 100644 index 0000000000..e4f1bcc549 --- /dev/null +++ b/collector/fixtures_bindmount/proc/1/mountinfo @@ -0,0 +1,6 @@ +29 1 259:0 / /host rw,seclabel,relatime,data=ordered shared:1 - ext4 /dev/nvme1n0 rw +30 1 260:0 / /host/media/volume1 rw,seclabel,relatime,data=ordered shared:1 - ext4 /dev/nvme1n1 rw +31 1 261:0 / /host/media/volume2 rw,seclabel,relatime,data=ordered shared:1 - ext4 /dev/nvme1n2 rw +31 26 0:26 / /dev/shm rw,nosuid,nodev shared:4 - tmpfs tmpfs rw,inode64 +32 28 0:27 / /run/lock rw,nosuid,nodev,noexec,relatime shared:6 - tmpfs tmpfs rw,size=5120k,inode64 +33 24 0:28 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:9 - cgroup2 cgroup2 rw diff --git a/collector/fixtures_bindmount/proc/mounts b/collector/fixtures_bindmount/proc/mounts deleted file mode 100644 index 32f9567e98..0000000000 --- a/collector/fixtures_bindmount/proc/mounts +++ /dev/null @@ -1,6 +0,0 @@ -/dev/nvme1n0 /host ext4 rw,seclabel,relatime,data=ordered 0 0 -/dev/nvme1n1 /host/media/volume1 ext4 rw,seclabel,relatime,data=ordered 0 0 -/dev/nvme1n2 /host/media/volume2 ext4 rw,seclabel,relatime,data=ordered 0 0 -tmpfs /dev/shm tmpfs rw,nosuid,nodev 0 0 -tmpfs /run/lock tmpfs rw,nosuid,nodev,noexec,relatime,size=5120k 0 0 -tmpfs /sys/fs/cgroup tmpfs ro,nosuid,nodev,noexec,mode=755 0 0 diff --git a/collector/fixtures_hidepid/proc/mounts b/collector/fixtures_hidepid/proc/mounts deleted file mode 100644 index fb6a96359c..0000000000 --- a/collector/fixtures_hidepid/proc/mounts +++ /dev/null @@ -1 +0,0 @@ -rootfs / rootfs rw 0 0 diff --git a/collector/fixtures_hidepid/proc/self/mountinfo b/collector/fixtures_hidepid/proc/self/mountinfo new file mode 100644 index 0000000000..dde22f67af --- /dev/null +++ b/collector/fixtures_hidepid/proc/self/mountinfo @@ -0,0 +1 @@ +29 1 259:2 / / rw,relatime shared:1 - ext4 /dev/nvme0n1p2 rw,errors=remount-ro