diff --git a/receiver/hostmetricsreceiver/README.md b/receiver/hostmetricsreceiver/README.md index 27719fcd424..08dfec2349f 100644 --- a/receiver/hostmetricsreceiver/README.md +++ b/receiver/hostmetricsreceiver/README.md @@ -77,9 +77,15 @@ disk: ```yaml filesystem: - : + : devices: [ , ... ] match_type: + : + fs_types: [ , ... ] + match_type: + : + mount_points: [ , ... ] + match_type: ``` #### Network diff --git a/receiver/hostmetricsreceiver/internal/scraper/filesystemscraper/config.go b/receiver/hostmetricsreceiver/internal/scraper/filesystemscraper/config.go index c304e914c79..b28e08913fa 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/filesystemscraper/config.go +++ b/receiver/hostmetricsreceiver/internal/scraper/filesystemscraper/config.go @@ -15,6 +15,8 @@ package filesystemscraper import ( + "fmt" + "go.opentelemetry.io/collector/internal/processor/filterset" "go.opentelemetry.io/collector/receiver/hostmetricsreceiver/internal" ) @@ -23,15 +25,116 @@ import ( type Config struct { internal.ConfigSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct - // Include specifies a filter on the devices that should be included from the generated metrics. - // Exclude specifies a filter on the devices that should be excluded from the generated metrics. - // If neither `include` or `exclude` are set, metrics will be generated for all devices. - Include MatchConfig `mapstructure:"include"` - Exclude MatchConfig `mapstructure:"exclude"` + // IncludeDevices specifies a filter on the devices that should be included in the generated metrics. + IncludeDevices DeviceMatchConfig `mapstructure:"include_devices"` + // ExcludeDevices specifies a filter on the devices that should be excluded from the generated metrics. + ExcludeDevices DeviceMatchConfig `mapstructure:"exclude_devices"` + + // IncludeFSTypes specifies a filter on the filesystem types that should be included in the generated metrics. + IncludeFSTypes FSTypeMatchConfig `mapstructure:"include_fs_types"` + // ExcludeFSTypes specifies a filter on the filesystem types points that should be excluded from the generated metrics. + ExcludeFSTypes FSTypeMatchConfig `mapstructure:"exclude_fs_types"` + + // IncludeMountPoints specifies a filter on the mount points that should be included in the generated metrics. + IncludeMountPoints MountPointMatchConfig `mapstructure:"include_mount_points"` + // ExcludeMountPoints specifies a filter on the mount points that should be excluded from the generated metrics. + ExcludeMountPoints MountPointMatchConfig `mapstructure:"exclude_mount_points"` } -type MatchConfig struct { +type DeviceMatchConfig struct { filterset.Config `mapstructure:",squash"` Devices []string `mapstructure:"devices"` } + +type FSTypeMatchConfig struct { + filterset.Config `mapstructure:",squash"` + + FSTypes []string `mapstructure:"fs_types"` +} + +type MountPointMatchConfig struct { + filterset.Config `mapstructure:",squash"` + + MountPoints []string `mapstructure:"mount_points"` +} + +type fsFilter struct { + includeDeviceFilter filterset.FilterSet + excludeDeviceFilter filterset.FilterSet + includeFSTypeFilter filterset.FilterSet + excludeFSTypeFilter filterset.FilterSet + includeMountPointFilter filterset.FilterSet + excludeMountPointFilter filterset.FilterSet + filtersExist bool +} + +func (cfg *Config) createFilter() (*fsFilter, error) { + var err error + filter := fsFilter{} + + filter.includeDeviceFilter, err = newIncludeFilterHelper(cfg.IncludeDevices.Devices, &cfg.IncludeDevices.Config, deviceLabelName) + if err != nil { + return nil, err + } + + filter.excludeDeviceFilter, err = newExcludeFilterHelper(cfg.ExcludeDevices.Devices, &cfg.ExcludeDevices.Config, deviceLabelName) + if err != nil { + return nil, err + } + + filter.includeFSTypeFilter, err = newIncludeFilterHelper(cfg.IncludeFSTypes.FSTypes, &cfg.IncludeFSTypes.Config, typeLabelName) + if err != nil { + return nil, err + } + + filter.excludeFSTypeFilter, err = newExcludeFilterHelper(cfg.ExcludeFSTypes.FSTypes, &cfg.ExcludeFSTypes.Config, typeLabelName) + if err != nil { + return nil, err + } + + filter.includeMountPointFilter, err = newIncludeFilterHelper(cfg.IncludeMountPoints.MountPoints, &cfg.IncludeMountPoints.Config, mountPointLabelName) + if err != nil { + return nil, err + } + + filter.excludeMountPointFilter, err = newExcludeFilterHelper(cfg.ExcludeMountPoints.MountPoints, &cfg.ExcludeMountPoints.Config, mountPointLabelName) + if err != nil { + return nil, err + } + + filter.setFiltersExist() + return &filter, nil +} + +func (f *fsFilter) setFiltersExist() { + f.filtersExist = f.includeMountPointFilter != nil || f.excludeMountPointFilter != nil || + f.includeFSTypeFilter != nil || f.excludeFSTypeFilter != nil || + f.includeDeviceFilter != nil || f.excludeDeviceFilter != nil +} + +const ( + excludeKey = "exclude" + includeKey = "include" +) + +func newIncludeFilterHelper(items []string, filterSet *filterset.Config, typ string) (filterset.FilterSet, error) { + return newFilterHelper(items, filterSet, includeKey, typ) +} + +func newExcludeFilterHelper(items []string, filterSet *filterset.Config, typ string) (filterset.FilterSet, error) { + return newFilterHelper(items, filterSet, excludeKey, typ) +} + +func newFilterHelper(items []string, filterSet *filterset.Config, typ string, filterType string) (filterset.FilterSet, error) { + var err error + var filter filterset.FilterSet + + if len(items) > 0 { + filter, err = filterset.CreateFilterSet(items, filterSet) + if err != nil { + return nil, fmt.Errorf("error creating %s %s filters: %w", filterType, typ, err) + } + } + return filter, nil +} diff --git a/receiver/hostmetricsreceiver/internal/scraper/filesystemscraper/factory_test.go b/receiver/hostmetricsreceiver/internal/scraper/filesystemscraper/factory_test.go index f80b3e1ba66..7d2712f02bb 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/filesystemscraper/factory_test.go +++ b/receiver/hostmetricsreceiver/internal/scraper/filesystemscraper/factory_test.go @@ -40,7 +40,7 @@ func TestCreateMetricsScraper(t *testing.T) { func TestCreateMetricsScraper_Error(t *testing.T) { factory := &Factory{} - cfg := &Config{Include: MatchConfig{Devices: []string{""}}} + cfg := &Config{IncludeDevices: DeviceMatchConfig{Devices: []string{""}}} _, err := factory.CreateMetricsScraper(context.Background(), zap.NewNop(), cfg) diff --git a/receiver/hostmetricsreceiver/internal/scraper/filesystemscraper/filesystem_scraper.go b/receiver/hostmetricsreceiver/internal/scraper/filesystemscraper/filesystem_scraper.go index cdc0a8ce742..acd6ef5cdd3 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/filesystemscraper/filesystem_scraper.go +++ b/receiver/hostmetricsreceiver/internal/scraper/filesystemscraper/filesystem_scraper.go @@ -16,7 +16,6 @@ package filesystemscraper import ( "context" - "fmt" "strings" "time" @@ -24,15 +23,13 @@ import ( "go.opentelemetry.io/collector/component/componenterror" "go.opentelemetry.io/collector/consumer/pdata" - "go.opentelemetry.io/collector/internal/processor/filterset" "go.opentelemetry.io/collector/receiver/hostmetricsreceiver/internal" ) // scraper for FileSystem Metrics type scraper struct { - config *Config - includeFS filterset.FilterSet - excludeFS filterset.FilterSet + config *Config + fsFilter fsFilter // for mocking gopsutil disk.Partitions & disk.Usage partitions func(bool) ([]disk.PartitionStat, error) @@ -46,24 +43,12 @@ type deviceUsage struct { // newFileSystemScraper creates a FileSystem Scraper func newFileSystemScraper(_ context.Context, cfg *Config) (*scraper, error) { - scraper := &scraper{config: cfg, partitions: disk.Partitions, usage: disk.Usage} - - var err error - - if len(cfg.Include.Devices) > 0 { - scraper.includeFS, err = filterset.CreateFilterSet(cfg.Include.Devices, &cfg.Include.Config) - if err != nil { - return nil, fmt.Errorf("error creating device include filters: %w", err) - } - } - - if len(cfg.Exclude.Devices) > 0 { - scraper.excludeFS, err = filterset.CreateFilterSet(cfg.Exclude.Devices, &cfg.Exclude.Config) - if err != nil { - return nil, fmt.Errorf("error creating device exclude filters: %w", err) - } + fsFilter, err := cfg.createFilter() + if err != nil { + return nil, err } + scraper := &scraper{config: cfg, partitions: disk.Partitions, usage: disk.Usage, fsFilter: *fsFilter} return scraper, nil } @@ -92,7 +77,9 @@ func (s *scraper) ScrapeMetrics(_ context.Context) (pdata.MetricSlice, error) { var errors []error usages := make([]*deviceUsage, 0, len(partitions)) for _, partition := range partitions { - // TODO: Add filters to include/exclude mount points + if !s.fsFilter.includePartition(partition) { + continue + } usage, err := s.usage(partition.Mountpoint) if err != nil { errors = append(errors, err) @@ -102,9 +89,6 @@ func (s *scraper) ScrapeMetrics(_ context.Context) (pdata.MetricSlice, error) { usages = append(usages, &deviceUsage{partition, usage}) } - // filter devices by name - usages = s.filterByDevice(usages) - if len(usages) > 0 { metrics.Resize(1 + systemSpecificMetricsLen) @@ -155,21 +139,27 @@ func exists(options []string, opt string) bool { return false } -func (s *scraper) filterByDevice(usages []*deviceUsage) []*deviceUsage { - if s.includeFS == nil && s.excludeFS == nil { - return usages +func (f *fsFilter) includePartition(partition disk.PartitionStat) bool { + // If filters do not exist, return early. + if !f.filtersExist || (f.includeDevice(partition.Device) && + f.includeFSType(partition.Fstype) && + f.includeMountPoint(partition.Mountpoint)) { + return true } + return false +} - filteredUsages := make([]*deviceUsage, 0, len(usages)) - for _, usage := range usages { - if s.includeDevice(usage.partition.Device) { - filteredUsages = append(filteredUsages, usage) - } - } - return filteredUsages +func (f *fsFilter) includeDevice(deviceName string) bool { + return (f.includeDeviceFilter == nil || f.includeDeviceFilter.Matches(deviceName)) && + (f.excludeDeviceFilter == nil || !f.excludeDeviceFilter.Matches(deviceName)) +} + +func (f *fsFilter) includeFSType(fsType string) bool { + return (f.includeFSTypeFilter == nil || f.includeFSTypeFilter.Matches(fsType)) && + (f.excludeFSTypeFilter == nil || !f.excludeFSTypeFilter.Matches(fsType)) } -func (s *scraper) includeDevice(deviceName string) bool { - return (s.includeFS == nil || s.includeFS.Matches(deviceName)) && - (s.excludeFS == nil || !s.excludeFS.Matches(deviceName)) +func (f *fsFilter) includeMountPoint(mountPoint string) bool { + return (f.includeMountPointFilter == nil || f.includeMountPointFilter.Matches(mountPoint)) && + (f.excludeMountPointFilter == nil || !f.excludeMountPointFilter.Matches(mountPoint)) } diff --git a/receiver/hostmetricsreceiver/internal/scraper/filesystemscraper/filesystem_scraper_test.go b/receiver/hostmetricsreceiver/internal/scraper/filesystemscraper/filesystem_scraper_test.go index dbb32931e45..d4ec2b12897 100644 --- a/receiver/hostmetricsreceiver/internal/scraper/filesystemscraper/filesystem_scraper_test.go +++ b/receiver/hostmetricsreceiver/internal/scraper/filesystemscraper/filesystem_scraper_test.go @@ -31,14 +31,15 @@ import ( func TestScrapeMetrics(t *testing.T) { type testCase struct { - name string - config Config - partitionsFunc func(bool) ([]disk.PartitionStat, error) - usageFunc func(string) (*disk.UsageStat, error) - expectMetrics bool - expectedDeviceDataPoints int - newErrRegex string - expectedErr string + name string + config Config + partitionsFunc func(bool) ([]disk.PartitionStat, error) + usageFunc func(string) (*disk.UsageStat, error) + expectMetrics bool + expectedDeviceDataPoints int + expectedDeviceLabelValues []map[string]string + newErrRegex string + expectedErr string } testCases := []testCase{ @@ -47,8 +48,8 @@ func TestScrapeMetrics(t *testing.T) { expectMetrics: true, }, { - name: "Include single process filter", - config: Config{Include: MatchConfig{filterset.Config{MatchType: "strict"}, []string{"a"}}}, + name: "Include single device filter", + config: Config{IncludeDevices: DeviceMatchConfig{filterset.Config{MatchType: "strict"}, []string{"a"}}}, partitionsFunc: func(bool) ([]disk.PartitionStat, error) { return []disk.PartitionStat{{Device: "a"}, {Device: "b"}}, nil }, @@ -59,20 +60,106 @@ func TestScrapeMetrics(t *testing.T) { expectedDeviceDataPoints: 1, }, { - name: "Include Filter that matches nothing", - config: Config{Include: MatchConfig{filterset.Config{MatchType: "strict"}, []string{"@*^#&*$^#)"}}}, + name: "Include Device Filter that matches nothing", + config: Config{IncludeDevices: DeviceMatchConfig{filterset.Config{MatchType: "strict"}, []string{"@*^#&*$^#)"}}}, expectMetrics: false, }, { - name: "Invalid Include Filter", - config: Config{Include: MatchConfig{Devices: []string{"test"}}}, + name: "Include filter with devices, filesystem type and mount points", + config: Config{ + IncludeDevices: DeviceMatchConfig{ + Config: filterset.Config{ + MatchType: filterset.Strict, + }, + Devices: []string{"device_a", "device_b"}, + }, + ExcludeFSTypes: FSTypeMatchConfig{ + Config: filterset.Config{ + MatchType: filterset.Strict, + }, + FSTypes: []string{"fs_type_b"}, + }, + ExcludeMountPoints: MountPointMatchConfig{ + Config: filterset.Config{ + MatchType: filterset.Strict, + }, + MountPoints: []string{"mount_point_b", "mount_point_c"}, + }, + }, + usageFunc: func(s string) (*disk.UsageStat, error) { + return &disk.UsageStat{ + Fstype: "fs_type_a", + }, nil + }, + partitionsFunc: func(b bool) ([]disk.PartitionStat, error) { + return []disk.PartitionStat{ + { + Device: "device_a", + Mountpoint: "mount_point_a", + Fstype: "fs_type_a", + }, + { + Device: "device_a", + Mountpoint: "mount_point_b", + Fstype: "fs_type_b", + }, + { + Device: "device_b", + Mountpoint: "mount_point_c", + Fstype: "fs_type_b", + }, + { + Device: "device_b", + Mountpoint: "mount_point_d", + Fstype: "fs_type_c", + }, + }, nil + }, + expectMetrics: true, + expectedDeviceDataPoints: 2, + expectedDeviceLabelValues: []map[string]string{ + { + "device": "device_a", + "mount.point": "mount_point_a", + "type": "fs_type_a", + }, + { + "device": "device_b", + "mount.point": "mount_point_d", + "type": "fs_type_c", + }, + }, + }, + { + name: "Invalid Include Device Filter", + config: Config{IncludeDevices: DeviceMatchConfig{Devices: []string{"test"}}}, newErrRegex: "^error creating device include filters:", }, { - name: "Invalid Exclude Filter", - config: Config{Exclude: MatchConfig{Devices: []string{"test"}}}, + name: "Invalid Exclude Device Filter", + config: Config{ExcludeDevices: DeviceMatchConfig{Devices: []string{"test"}}}, newErrRegex: "^error creating device exclude filters:", }, + { + name: "Invalid Include Filesystems Filter", + config: Config{IncludeFSTypes: FSTypeMatchConfig{FSTypes: []string{"test"}}}, + newErrRegex: "^error creating type include filters:", + }, + { + name: "Invalid Exclude Filesystems Filter", + config: Config{ExcludeFSTypes: FSTypeMatchConfig{FSTypes: []string{"test"}}}, + newErrRegex: "^error creating type exclude filters:", + }, + { + name: "Invalid Include Moountpoints Filter", + config: Config{IncludeMountPoints: MountPointMatchConfig{MountPoints: []string{"test"}}}, + newErrRegex: "^error creating mount.point include filters:", + }, + { + name: "Invalid Exclude Moountpoints Filter", + config: Config{ExcludeMountPoints: MountPointMatchConfig{MountPoints: []string{"test"}}}, + newErrRegex: "^error creating mount.point exclude filters:", + }, { name: "Partitions Error", partitionsFunc: func(bool) ([]disk.PartitionStat, error) { return nil, errors.New("err1") }, @@ -120,11 +207,23 @@ func TestScrapeMetrics(t *testing.T) { assert.GreaterOrEqual(t, metrics.Len(), 1) - assertFileSystemUsageMetricValid(t, metrics.At(0), fileSystemUsageDescriptor, test.expectedDeviceDataPoints*fileSystemStatesLen) + assertFileSystemUsageMetricValid( + t, + metrics.At(0), + fileSystemUsageDescriptor, + test.expectedDeviceDataPoints*fileSystemStatesLen, + test.expectedDeviceLabelValues, + ) if isUnix() { assertFileSystemUsageMetricHasUnixSpecificStateLabels(t, metrics.At(0)) - assertFileSystemUsageMetricValid(t, metrics.At(1), fileSystemINodesUsageDescriptor, test.expectedDeviceDataPoints*2) + assertFileSystemUsageMetricValid( + t, + metrics.At(1), + fileSystemINodesUsageDescriptor, + test.expectedDeviceDataPoints*2, + test.expectedDeviceLabelValues, + ) } internal.AssertSameTimeStampForAllMetrics(t, metrics) @@ -132,7 +231,12 @@ func TestScrapeMetrics(t *testing.T) { } } -func assertFileSystemUsageMetricValid(t *testing.T, metric pdata.Metric, descriptor pdata.Metric, expectedDeviceDataPoints int) { +func assertFileSystemUsageMetricValid( + t *testing.T, + metric pdata.Metric, + descriptor pdata.Metric, + expectedDeviceDataPoints int, + expectedDeviceLabelValues []map[string]string) { internal.AssertDescriptorEqual(t, descriptor, metric) for i := 0; i < metric.IntSum().DataPoints().Len(); i++ { for _, label := range []string{deviceLabelName, typeLabelName, mountModeLabelName, mountPointLabelName} { @@ -142,6 +246,20 @@ func assertFileSystemUsageMetricValid(t *testing.T, metric pdata.Metric, descrip if expectedDeviceDataPoints > 0 { assert.Equal(t, expectedDeviceDataPoints, metric.IntSum().DataPoints().Len()) + + // Assert label values if specified. + if expectedDeviceLabelValues != nil { + dpsPerDevice := expectedDeviceDataPoints / len(expectedDeviceLabelValues) + deviceIdx := 0 + for i := 0; i < metric.IntSum().DataPoints().Len(); i += dpsPerDevice { + for j := i; j < i+dpsPerDevice; j++ { + for labelKey, labelValue := range expectedDeviceLabelValues[deviceIdx] { + internal.AssertIntSumMetricLabelHasValue(t, metric, j, labelKey, labelValue) + } + } + deviceIdx++ + } + } } else { assert.GreaterOrEqual(t, metric.IntSum().DataPoints().Len(), fileSystemStatesLen) }