From 24802b65735c221177b7af87979403b9ef948a83 Mon Sep 17 00:00:00 2001 From: Christopher Roberts <90332286+chrroberts-pure@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:41:31 -0600 Subject: [PATCH] [receiver/purefa] Pure Storage FlashArray - Purity v6.6.11+ Native OpenMetrics Support (#36251) This PR implements the changes required to scrape metrics from arrays with newer Purity versions. In Pure Storage FlashArray Purity version 6.6.11, they released a feature that allows OpenMetrics to be scraped directly from the array without the use of an OpenMetrics Exporter. These changes include - Enabling support to disable TLS Insecure Skip Verify if required (default is to verify) - Enabling support for the `namespace` HTTP param that is supported by the Purity OpenMetrics exporter. #### Link to tracking issue None #### Testing Tests have been updated for the scraper and default configs for the new Namespace and TLSInsecureSkipVerify configurations. I have tested this with a live FlashArray running 6.6.11 #### Documentation The Readme was updated to show a new example of a scrape using Purity version 6.6.11+. --- .chloggen/36251-purefa-native-om-support.yaml | 27 ++++++++++++++++ receiver/purefareceiver/README.md | 32 +++++++++++-------- receiver/purefareceiver/config.go | 8 ++++- receiver/purefareceiver/config_test.go | 11 ++++--- receiver/purefareceiver/factory.go | 11 ++++--- receiver/purefareceiver/go.mod | 2 +- receiver/purefareceiver/internal/scraper.go | 18 ++++++++++- .../purefareceiver/internal/scraper_test.go | 12 ++++++- receiver/purefareceiver/receiver.go | 10 +++--- receiver/purefareceiver/testdata/config.yaml | 1 + 10 files changed, 100 insertions(+), 32 deletions(-) create mode 100644 .chloggen/36251-purefa-native-om-support.yaml diff --git a/.chloggen/36251-purefa-native-om-support.yaml b/.chloggen/36251-purefa-native-om-support.yaml new file mode 100644 index 000000000000..82f3cd5f2e11 --- /dev/null +++ b/.chloggen/36251-purefa-native-om-support.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: 'enhancement' + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: purefareceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Implements support for scraping Pure Storage FlashArray with Purity version 6.6.11+ + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [36251] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/receiver/purefareceiver/README.md b/receiver/purefareceiver/README.md index bf5f2d9b2fcd..48af5f922bf4 100644 --- a/receiver/purefareceiver/README.md +++ b/receiver/purefareceiver/README.md @@ -12,12 +12,16 @@ [contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib -The Pure Storage FlashArray receiver, receives metrics from Pure Storage internal services hosts. +The Pure Storage FlashArray receiver, receives metrics from the Pure Storage FlashArray. ## Configuration The following settings are required: - - `endpoint` (default: `http://172.0.0.0:9490/metrics/array`): The URL of the scraper selected endpoint + - `endpoint` (default: `http://127.0.0.0:9490/metrics/array`): The URL of the scraper selected endpoint. + - `fa_array_name` (no default): The array's pretty name to be used as a metrics label. + - `namespace` (default:purefa): The selected Pure Storage OpenMetrics Namespace to query. + +In the below examples array01 is using the [Pure Storage FlashArray OpenMetrics exporter](https://github.com/PureStorage-OpenConnect/pure-fa-openmetrics-exporter), while array02 is using the native on-box metrics provided in Purity//FA v6.6.11+. Example: @@ -55,15 +59,17 @@ receivers: env: dev settings: reload_intervals: - array: 10s - hosts: 13s - directories: 15s - pods: 30s - volumes: 25s + array: 20s + hosts: 60s + directories: 60s + pods: 60s + volumes: 60s purefa/array02: fa_array_name: foobar02 - endpoint: http://127.0.0.1:9490/metrics + endpoint: https://127.0.0.1/metrics + tls: + insecure_skip_verify: true array: - address: array02 auth: @@ -87,11 +93,11 @@ receivers: env: production settings: reload_intervals: - array: 15s - hosts: 15s - directories: 15s - pods: 30s - volumes: 25s + array: 20s + hosts: 60s + directories: 60s + pods: 60s + volumes: 60s service: extensions: [bearertokenauth/array01,bearertokenauth/array02] diff --git a/receiver/purefareceiver/config.go b/receiver/purefareceiver/config.go index fe2d0b9e3562..1bde48af1fb8 100644 --- a/receiver/purefareceiver/config.go +++ b/receiver/purefareceiver/config.go @@ -41,8 +41,11 @@ type Config struct { // Env represents the respective environment value valid to scrape Env string `mapstructure:"env"` - // ArrayName represents the display name that is appended to the received metrics, as the `host` label if not provided by OpenMetrics output, and to the `fa_array_name` label always. + // ArrayName represents the display name that is appended to the received metrics, as the `host` label if not provided by OpenMetrics output, and to the `fa_array_name` label always ArrayName string `mapstructure:"fa_array_name"` + + // Namespace selects the OpenMetrics namespace requested + Namespace string `mapstructure:"namespace"` } type Settings struct { @@ -63,6 +66,9 @@ func (c *Config) Validate() error { if c.ArrayName == "" { errs = multierr.Append(errs, errors.New("the array's pretty name as 'fa_array_name' must be provided")) } + if c.Namespace == "" { + errs = multierr.Append(errs, errors.New("a specified namespace must be provided")) + } if c.Settings.ReloadIntervals.Array == 0 { errs = multierr.Append(errs, errors.New("reload interval for 'array' must be provided")) } diff --git a/receiver/purefareceiver/config_test.go b/receiver/purefareceiver/config_test.go index 4147505e4b84..8c7799b648d9 100644 --- a/receiver/purefareceiver/config_test.go +++ b/receiver/purefareceiver/config_test.go @@ -34,13 +34,14 @@ func TestLoadConfig(t *testing.T) { expected: &Config{ ClientConfig: clientConfig, ArrayName: "foobar.example.com", + Namespace: "purefa", Settings: &Settings{ ReloadIntervals: &ReloadIntervals{ - Array: 15 * time.Second, - Hosts: 15 * time.Second, - Directories: 15 * time.Second, - Pods: 15 * time.Second, - Volumes: 15 * time.Second, + Array: 60 * time.Second, + Hosts: 60 * time.Second, + Directories: 60 * time.Second, + Pods: 60 * time.Second, + Volumes: 60 * time.Second, }, }, }, diff --git a/receiver/purefareceiver/factory.go b/receiver/purefareceiver/factory.go index 3be54df2dd55..6ebae400dda3 100644 --- a/receiver/purefareceiver/factory.go +++ b/receiver/purefareceiver/factory.go @@ -29,14 +29,15 @@ func NewFactory() receiver.Factory { func createDefaultConfig() component.Config { return &Config{ ArrayName: "foobar.example.com", + Namespace: "purefa", ClientConfig: confighttp.NewDefaultClientConfig(), Settings: &Settings{ ReloadIntervals: &ReloadIntervals{ - Array: 15 * time.Second, - Hosts: 15 * time.Second, - Directories: 15 * time.Second, - Pods: 15 * time.Second, - Volumes: 15 * time.Second, + Array: 60 * time.Second, + Hosts: 60 * time.Second, + Directories: 60 * time.Second, + Pods: 60 * time.Second, + Volumes: 60 * time.Second, }, }, } diff --git a/receiver/purefareceiver/go.mod b/receiver/purefareceiver/go.mod index 6a7583b96d5e..6e5b3fa1b278 100644 --- a/receiver/purefareceiver/go.mod +++ b/receiver/purefareceiver/go.mod @@ -12,6 +12,7 @@ require ( go.opentelemetry.io/collector/component/componenttest v0.115.1-0.20241206185113-3f3e208e71b8 go.opentelemetry.io/collector/config/configauth v0.115.1-0.20241206185113-3f3e208e71b8 go.opentelemetry.io/collector/config/confighttp v0.115.1-0.20241206185113-3f3e208e71b8 + go.opentelemetry.io/collector/config/configtls v1.21.1-0.20241206185113-3f3e208e71b8 go.opentelemetry.io/collector/confmap v1.21.1-0.20241206185113-3f3e208e71b8 go.opentelemetry.io/collector/consumer v1.21.1-0.20241206185113-3f3e208e71b8 go.opentelemetry.io/collector/consumer/consumertest v0.115.1-0.20241206185113-3f3e208e71b8 @@ -144,7 +145,6 @@ require ( go.opentelemetry.io/collector/config/configcompression v1.21.1-0.20241206185113-3f3e208e71b8 // indirect go.opentelemetry.io/collector/config/configopaque v1.21.1-0.20241206185113-3f3e208e71b8 // indirect go.opentelemetry.io/collector/config/configtelemetry v0.115.1-0.20241206185113-3f3e208e71b8 // indirect - go.opentelemetry.io/collector/config/configtls v1.21.1-0.20241206185113-3f3e208e71b8 // indirect go.opentelemetry.io/collector/config/internal v0.115.1-0.20241206185113-3f3e208e71b8 // indirect go.opentelemetry.io/collector/consumer/consumererror v0.115.1-0.20241206185113-3f3e208e71b8 // indirect go.opentelemetry.io/collector/consumer/consumerprofiles v0.115.1-0.20241206185113-3f3e208e71b8 // indirect diff --git a/receiver/purefareceiver/internal/scraper.go b/receiver/purefareceiver/internal/scraper.go index 3788f6c5ea9d..ac205ce4c49b 100644 --- a/receiver/purefareceiver/internal/scraper.go +++ b/receiver/purefareceiver/internal/scraper.go @@ -14,6 +14,7 @@ import ( "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery" "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/configtls" "go.opentelemetry.io/collector/receiver" ) @@ -34,6 +35,8 @@ const ( type scraper struct { scraperType ScraperType endpoint string + namespace string + tlsSettings configtls.ClientConfig configs []ScraperConfig scrapeInterval time.Duration labels model.LabelSet @@ -42,6 +45,8 @@ type scraper struct { func NewScraper(_ context.Context, scraperType ScraperType, endpoint string, + namespace string, + tlsSettings configtls.ClientConfig, configs []ScraperConfig, scrapeInterval time.Duration, labels model.LabelSet, @@ -49,6 +54,8 @@ func NewScraper(_ context.Context, return &scraper{ scraperType: scraperType, endpoint: endpoint, + namespace: namespace, + tlsSettings: tlsSettings, configs: configs, scrapeInterval: scrapeInterval, labels: labels, @@ -71,9 +78,17 @@ func (h *scraper) ToPrometheusReceiverConfig(host component.Host, _ receiver.Fac httpConfig := configutil.HTTPClientConfig{} httpConfig.BearerToken = configutil.Secret(bearerToken) + httpConfig.TLSConfig = configutil.TLSConfig{ + CAFile: h.tlsSettings.CAFile, + CertFile: h.tlsSettings.CertFile, + KeyFile: h.tlsSettings.KeyFile, + InsecureSkipVerify: h.tlsSettings.InsecureSkipVerify, + ServerName: h.tlsSettings.ServerName, + } scrapeConfig := &config.ScrapeConfig{ HTTPClientConfig: httpConfig, + ScrapeProtocols: config.DefaultScrapeProtocols, ScrapeInterval: model.Duration(h.scrapeInterval), ScrapeTimeout: model.Duration(h.scrapeInterval), JobName: fmt.Sprintf("%s/%s/%s", "purefa", h.scraperType, arr.Address), @@ -81,7 +96,8 @@ func (h *scraper) ToPrometheusReceiverConfig(host component.Host, _ receiver.Fac Scheme: u.Scheme, MetricsPath: fmt.Sprintf("/metrics/%s", h.scraperType), Params: url.Values{ - "endpoint": {arr.Address}, + "endpoint": {arr.Address}, + "namespace": {h.namespace}, }, ServiceDiscoveryConfigs: discovery.Configs{ diff --git a/receiver/purefareceiver/internal/scraper_test.go b/receiver/purefareceiver/internal/scraper_test.go index 749f7543d386..7a0cedce074b 100644 --- a/receiver/purefareceiver/internal/scraper_test.go +++ b/receiver/purefareceiver/internal/scraper_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config/configauth" + "go.opentelemetry.io/collector/config/configtls" "go.opentelemetry.io/collector/extension/extensiontest" "github.com/open-telemetry/opentelemetry-collector-contrib/extension/bearertokenauthextension" @@ -37,6 +38,11 @@ func TestToPrometheusConfig(t *testing.T) { } endpoint := "http://example.com" + namespace := "purefa" + tlsSettings := configtls.ClientConfig{ + InsecureSkipVerify: true, + ServerName: "TestThisServerName", + } interval := 15 * time.Second cfgs := []ScraperConfig{ { @@ -47,7 +53,7 @@ func TestToPrometheusConfig(t *testing.T) { }, } - scraper := NewScraper(context.Background(), "hosts", endpoint, cfgs, interval, model.LabelSet{}) + scraper := NewScraper(context.Background(), "hosts", endpoint, namespace, tlsSettings, cfgs, interval, model.LabelSet{}) // test scCfgs, err := scraper.ToPrometheusReceiverConfig(host, prFactory) @@ -55,7 +61,11 @@ func TestToPrometheusConfig(t *testing.T) { // verify assert.NoError(t, err) assert.Len(t, scCfgs, 1) + assert.NotNil(t, scCfgs[0].ScrapeProtocols) + assert.EqualValues(t, "purefa", scCfgs[0].Params.Get("namespace")) assert.EqualValues(t, "the-token", scCfgs[0].HTTPClientConfig.BearerToken) + assert.True(t, scCfgs[0].HTTPClientConfig.TLSConfig.InsecureSkipVerify) + assert.Equal(t, "TestThisServerName", scCfgs[0].HTTPClientConfig.TLSConfig.ServerName) assert.Equal(t, "array01", scCfgs[0].Params.Get("endpoint")) assert.Equal(t, "/metrics/hosts", scCfgs[0].MetricsPath) assert.Equal(t, "purefa/hosts/array01", scCfgs[0].JobName) diff --git a/receiver/purefareceiver/receiver.go b/receiver/purefareceiver/receiver.go index ad2b9bec5c9e..af17a41b5f16 100644 --- a/receiver/purefareceiver/receiver.go +++ b/receiver/purefareceiver/receiver.go @@ -53,35 +53,35 @@ func (r *purefaReceiver) Start(ctx context.Context, compHost component.Host) err "fa_array_name": ArrayName, } - arrScraper := internal.NewScraper(ctx, internal.ScraperTypeArray, r.cfg.Endpoint, r.cfg.Array, r.cfg.Settings.ReloadIntervals.Array, commomLabel) + arrScraper := internal.NewScraper(ctx, internal.ScraperTypeArray, r.cfg.Endpoint, r.cfg.Namespace, r.cfg.TLSSetting, r.cfg.Array, r.cfg.Settings.ReloadIntervals.Array, commomLabel) if scCfgs, err := arrScraper.ToPrometheusReceiverConfig(compHost, fact); err == nil { scrapeCfgs = append(scrapeCfgs, scCfgs...) } else { return err } - hostScraper := internal.NewScraper(ctx, internal.ScraperTypeHosts, r.cfg.Endpoint, r.cfg.Hosts, r.cfg.Settings.ReloadIntervals.Hosts, labelSet) + hostScraper := internal.NewScraper(ctx, internal.ScraperTypeHosts, r.cfg.Endpoint, r.cfg.Namespace, r.cfg.TLSSetting, r.cfg.Hosts, r.cfg.Settings.ReloadIntervals.Hosts, labelSet) if scCfgs, err := hostScraper.ToPrometheusReceiverConfig(compHost, fact); err == nil { scrapeCfgs = append(scrapeCfgs, scCfgs...) } else { return err } - directoriesScraper := internal.NewScraper(ctx, internal.ScraperTypeDirectories, r.cfg.Endpoint, r.cfg.Directories, r.cfg.Settings.ReloadIntervals.Directories, commomLabel) + directoriesScraper := internal.NewScraper(ctx, internal.ScraperTypeDirectories, r.cfg.Endpoint, r.cfg.Namespace, r.cfg.TLSSetting, r.cfg.Directories, r.cfg.Settings.ReloadIntervals.Directories, commomLabel) if scCfgs, err := directoriesScraper.ToPrometheusReceiverConfig(compHost, fact); err == nil { scrapeCfgs = append(scrapeCfgs, scCfgs...) } else { return err } - podsScraper := internal.NewScraper(ctx, internal.ScraperTypePods, r.cfg.Endpoint, r.cfg.Pods, r.cfg.Settings.ReloadIntervals.Pods, commomLabel) + podsScraper := internal.NewScraper(ctx, internal.ScraperTypePods, r.cfg.Endpoint, r.cfg.Namespace, r.cfg.TLSSetting, r.cfg.Pods, r.cfg.Settings.ReloadIntervals.Pods, commomLabel) if scCfgs, err := podsScraper.ToPrometheusReceiverConfig(compHost, fact); err == nil { scrapeCfgs = append(scrapeCfgs, scCfgs...) } else { return err } - volumesScraper := internal.NewScraper(ctx, internal.ScraperTypeVolumes, r.cfg.Endpoint, r.cfg.Volumes, r.cfg.Settings.ReloadIntervals.Volumes, labelSet) + volumesScraper := internal.NewScraper(ctx, internal.ScraperTypeVolumes, r.cfg.Endpoint, r.cfg.Namespace, r.cfg.TLSSetting, r.cfg.Volumes, r.cfg.Settings.ReloadIntervals.Volumes, labelSet) if scCfgs, err := volumesScraper.ToPrometheusReceiverConfig(compHost, fact); err == nil { scrapeCfgs = append(scrapeCfgs, scCfgs...) } else { diff --git a/receiver/purefareceiver/testdata/config.yaml b/receiver/purefareceiver/testdata/config.yaml index 6bb13a8f1e8f..ed99af405911 100644 --- a/receiver/purefareceiver/testdata/config.yaml +++ b/receiver/purefareceiver/testdata/config.yaml @@ -5,6 +5,7 @@ receivers: purefa/with_custom_intervals: fa_array_name: foobar.example.com + namespace: purefa endpoint: http://172.31.60.208:9490/metrics array: - address: array01