From c0c6a0fa50c510c8d1597219fe21acfeaa7e087f Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 14 Jul 2020 17:05:27 +0300 Subject: [PATCH 01/24] Infer types in remote_write Signed-off-by: chrismark --- .../module/prometheus/collector/data.go | 5 +- .../module/prometheus/remote_write/data.go | 13 +- .../prometheus/remote_write/remote_write.go | 53 +++- ...prometheus.yml.disabled => prometheus.yml} | 18 +- .../{system.yml => system.yml.disabled} | 0 .../prometheus/collector/counter_test.go | 4 +- .../module/prometheus/collector/data.go | 40 +-- .../module/prometheus/collector/histogram.go | 4 +- .../prometheus/collector/histogram_test.go | 4 +- .../prometheus/remote_write/_meta/data.json | 51 ++++ .../remote_write/_meta/docs.asciidoc | 63 ++++ .../prometheus/remote_write/_meta/fields.yml | 6 + .../module/prometheus/remote_write/config.go | 20 ++ .../module/prometheus/remote_write/data.go | 273 ++++++++++++++++++ .../prometheus/remote_write/remote_write.go | 35 +++ 15 files changed, 548 insertions(+), 41 deletions(-) rename metricbeat/modules.d/{prometheus.yml.disabled => prometheus.yml} (89%) rename metricbeat/modules.d/{system.yml => system.yml.disabled} (100%) create mode 100644 x-pack/metricbeat/module/prometheus/remote_write/_meta/data.json create mode 100644 x-pack/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/prometheus/remote_write/config.go create mode 100644 x-pack/metricbeat/module/prometheus/remote_write/data.go create mode 100644 x-pack/metricbeat/module/prometheus/remote_write/remote_write.go diff --git a/metricbeat/module/prometheus/collector/data.go b/metricbeat/module/prometheus/collector/data.go index 717ec2dc732..6fc2823316f 100644 --- a/metricbeat/module/prometheus/collector/data.go +++ b/metricbeat/module/prometheus/collector/data.go @@ -18,12 +18,11 @@ package collector import ( - "math" - "strconv" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/helper/labelhash" "github.com/elastic/beats/v7/metricbeat/mb" + "math" + "strconv" dto "github.com/prometheus/client_model/go" ) diff --git a/metricbeat/module/prometheus/remote_write/data.go b/metricbeat/module/prometheus/remote_write/data.go index 8dfa6072d6a..d15749ab932 100644 --- a/metricbeat/module/prometheus/remote_write/data.go +++ b/metricbeat/module/prometheus/remote_write/data.go @@ -26,7 +26,18 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb" ) -func samplesToEvents(metrics model.Samples) map[string]mb.Event { + +// DefaultRemoteWriteEventsGeneratorFactory returns the default prometheus events generator +func DefaultRemoteWriteEventsGeneratorFactory(ms mb.BaseMetricSet) (RemoteWriteEventsGenerator, error) { + return &remoteWriteEventGenerator{}, nil +} + +type remoteWriteEventGenerator struct{} + +func (p *remoteWriteEventGenerator) Start() {} +func (p *remoteWriteEventGenerator) Stop() {} + +func (p *remoteWriteEventGenerator) GenerateEvents(metrics model.Samples) map[string]mb.Event { eventList := map[string]mb.Event{} for _, metric := range metrics { diff --git a/metricbeat/module/prometheus/remote_write/remote_write.go b/metricbeat/module/prometheus/remote_write/remote_write.go index b0f22455eb9..e4c4d5567f4 100644 --- a/metricbeat/module/prometheus/remote_write/remote_write.go +++ b/metricbeat/module/prometheus/remote_write/remote_write.go @@ -33,15 +33,33 @@ import ( ) func init() { - mb.Registry.MustAddMetricSet("prometheus", "remote_write", New, + mb.Registry.MustAddMetricSet("prometheus", "remote_write", + MetricSetBuilder("prometheus", DefaultRemoteWriteEventsGeneratorFactory), mb.WithHostParser(parse.EmptyHostParser), ) } +// RemoteWriteEventsGenerator converts a Prometheus metric Samples mb.Event map +type RemoteWriteEventsGenerator interface { + // Start must be called before using the generator + Start() + + // converts a Prometheus metric family into a list of PromEvents + GenerateEvents(metrics model.Samples) map[string]mb.Event + + // Stop must be called when the generator won't be used anymore + Stop() +} + +// RemoteWriteEventsGeneratorFactory creates a PromEventsGenerator when instanciating a metricset +type RemoteWriteEventsGeneratorFactory func(ms mb.BaseMetricSet) (RemoteWriteEventsGenerator, error) + type MetricSet struct { mb.BaseMetricSet server serverhelper.Server events chan mb.Event + promEventsGen RemoteWriteEventsGenerator + eventGenStarted bool } func New(base mb.BaseMetricSet) (mb.MetricSet, error) { @@ -62,6 +80,37 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return m, nil } +// MetricSetBuilder returns a builder function for a new Prometheus metricset using +// the given namespace and event generator +func MetricSetBuilder(namespace string, genFactory RemoteWriteEventsGeneratorFactory) func(base mb.BaseMetricSet) (mb.MetricSet, error) { + return func(base mb.BaseMetricSet) (mb.MetricSet, error) { + config := defaultConfig() + err := base.Module().UnpackConfig(&config) + if err != nil { + return nil, err + } + + promEventsGen, err := genFactory(base) + if err != nil { + return nil, err + } + + m := &MetricSet{ + BaseMetricSet: base, + promEventsGen: promEventsGen, + eventGenStarted: false, + } + svc, err := httpserver.NewHttpServerWithHandler(base, m.handleFunc) + if err != nil { + return nil, err + } + m.server = svc + + return m, nil + } +} + + func (m *MetricSet) Run(reporter mb.PushReporterV2) { // Start event watcher m.server.Start() @@ -100,7 +149,7 @@ func (m *MetricSet) handleFunc(writer http.ResponseWriter, req *http.Request) { } samples := protoToSamples(&protoReq) - events := samplesToEvents(samples) + events := m.promEventsGen.GenerateEvents(samples) for _, e := range events { select { diff --git a/metricbeat/modules.d/prometheus.yml.disabled b/metricbeat/modules.d/prometheus.yml similarity index 89% rename from metricbeat/modules.d/prometheus.yml.disabled rename to metricbeat/modules.d/prometheus.yml index 82f45573931..d0f46d0c85f 100644 --- a/metricbeat/modules.d/prometheus.yml.disabled +++ b/metricbeat/modules.d/prometheus.yml @@ -2,11 +2,11 @@ # Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-prometheus.html # Metrics collected from a Prometheus endpoint -- module: prometheus - period: 10s - metricsets: ["collector"] - hosts: ["localhost:9090"] - metrics_path: /metrics +#- module: prometheus +# period: 10s +# metricsets: ["collector"] +# hosts: ["localhost:9090"] +# metrics_path: /metrics #metrics_filters: # include: [] # exclude: [] @@ -20,10 +20,10 @@ # Metrics sent by a Prometheus server using remote_write option -#- module: prometheus -# metricsets: ["remote_write"] -# host: "localhost" -# port: "9201" +- module: prometheus + metricsets: ["remote_write"] + host: "localhost" + port: "9201" # Secure settings for the server using TLS/SSL: #ssl.certificate: "/etc/pki/server/cert.pem" diff --git a/metricbeat/modules.d/system.yml b/metricbeat/modules.d/system.yml.disabled similarity index 100% rename from metricbeat/modules.d/system.yml rename to metricbeat/modules.d/system.yml.disabled diff --git a/x-pack/metricbeat/module/prometheus/collector/counter_test.go b/x-pack/metricbeat/module/prometheus/collector/counter_test.go index dc4e9cd6423..57bae0052cf 100644 --- a/x-pack/metricbeat/module/prometheus/collector/counter_test.go +++ b/x-pack/metricbeat/module/prometheus/collector/counter_test.go @@ -51,13 +51,13 @@ func Test_CounterCache(t *testing.T) { for i, val := range tt.valuesUint64 { want := tt.expectedUin64[i] if got, _ := tt.counterCache.RateUint64(tt.counterName, val); got != want { - t.Errorf("counterCache.RateUint64() = %v, want %v", got, want) + t.Errorf("CounterCache.RateUint64() = %v, want %v", got, want) } } for i, val := range tt.valuesFloat64 { want := tt.expectedFloat64[i] if got, _ := tt.counterCache.RateFloat64(tt.counterName, val); got != want { - t.Errorf("counterCache.RateFloat64() = %v, want %v", got, want) + t.Errorf("CounterCache.RateFloat64() = %v, want %v", got, want) } } }) diff --git a/x-pack/metricbeat/module/prometheus/collector/data.go b/x-pack/metricbeat/module/prometheus/collector/data.go index 23ef386291b..f0a367d5115 100644 --- a/x-pack/metricbeat/module/prometheus/collector/data.go +++ b/x-pack/metricbeat/module/prometheus/collector/data.go @@ -28,9 +28,9 @@ func promEventsGeneratorFactory(base mb.BaseMetricSet) (collector.PromEventsGene // to make sure that all counters are available between fetches counters := NewCounterCache(base.Module().Config().Period * 5) - g := typedGenerator{ - counterCache: counters, - rateCounters: config.RateCounters, + g := TypedGenerator{ + CounterCache: counters, + RateCounters: config.RateCounters, } return &g, nil @@ -39,29 +39,29 @@ func promEventsGeneratorFactory(base mb.BaseMetricSet) (collector.PromEventsGene return collector.DefaultPromEventsGeneratorFactory(base) } -type typedGenerator struct { - counterCache CounterCache - rateCounters bool +type TypedGenerator struct { + CounterCache CounterCache + RateCounters bool } -func (g *typedGenerator) Start() { +func (g *TypedGenerator) Start() { cfgwarn.Beta("Prometheus 'use_types' setting is beta") - if g.rateCounters { + if g.RateCounters { cfgwarn.Experimental("Prometheus 'rate_counters' setting is experimental") } - g.counterCache.Start() + g.CounterCache.Start() } -func (g *typedGenerator) Stop() { - logp.Debug("prometheus.collector.cache", "stopping counterCache") - g.counterCache.Stop() +func (g *TypedGenerator) Stop() { + logp.Debug("prometheus.collector.cache", "stopping CounterCache") + g.CounterCache.Stop() } // GeneratePromEvents stores all Prometheus metrics using // specific Elasticsearch data types. -func (g *typedGenerator) GeneratePromEvents(mf *dto.MetricFamily) []collector.PromEvent { +func (g *TypedGenerator) GeneratePromEvents(mf *dto.MetricFamily) []collector.PromEvent { var events []collector.PromEvent name := *mf.Name @@ -138,7 +138,7 @@ func (g *typedGenerator) GeneratePromEvents(mf *dto.MetricFamily) []collector.Pr events = append(events, collector.PromEvent{ Data: common.MapStr{ name: common.MapStr{ - "histogram": promHistogramToES(g.counterCache, name, labels, histogram), + "histogram": PromHistogramToES(g.CounterCache, name, labels, histogram), }, }, Labels: labels, @@ -167,26 +167,26 @@ func (g *typedGenerator) GeneratePromEvents(mf *dto.MetricFamily) []collector.Pr } // rateCounterUint64 fills a counter value and optionally adds the rate if rate_counters is enabled -func (g *typedGenerator) rateCounterUint64(name string, labels common.MapStr, value uint64) common.MapStr { +func (g *TypedGenerator) rateCounterUint64(name string, labels common.MapStr, value uint64) common.MapStr { d := common.MapStr{ "counter": value, } - if g.rateCounters { - d["rate"], _ = g.counterCache.RateUint64(name+labels.String(), value) + if g.RateCounters { + d["rate"], _ = g.CounterCache.RateUint64(name+labels.String(), value) } return d } // rateCounterFloat64 fills a counter value and optionally adds the rate if rate_counters is enabled -func (g *typedGenerator) rateCounterFloat64(name string, labels common.MapStr, value float64) common.MapStr { +func (g *TypedGenerator) rateCounterFloat64(name string, labels common.MapStr, value float64) common.MapStr { d := common.MapStr{ "counter": value, } - if g.rateCounters { - d["rate"], _ = g.counterCache.RateFloat64(name+labels.String(), value) + if g.RateCounters { + d["rate"], _ = g.CounterCache.RateFloat64(name+labels.String(), value) } return d diff --git a/x-pack/metricbeat/module/prometheus/collector/histogram.go b/x-pack/metricbeat/module/prometheus/collector/histogram.go index 63ed3bf69ce..1d23264a2fb 100644 --- a/x-pack/metricbeat/module/prometheus/collector/histogram.go +++ b/x-pack/metricbeat/module/prometheus/collector/histogram.go @@ -13,7 +13,7 @@ import ( dto "github.com/prometheus/client_model/go" ) -// promHistogramToES takes a Prometheus histogram and converts it to an ES histogram: +// PromHistogramToES takes a Prometheus histogram and converts it to an ES histogram: // // ES histograms look like this: // @@ -27,7 +27,7 @@ import ( // - undoing counters accumulation for each bucket (counts) // // https://www.elastic.co/guide/en/elasticsearch/reference/master/histogram.html -func promHistogramToES(cc CounterCache, name string, labels common.MapStr, histogram *dto.Histogram) common.MapStr { +func PromHistogramToES(cc CounterCache, name string, labels common.MapStr, histogram *dto.Histogram) common.MapStr { var values []float64 var counts []uint64 diff --git a/x-pack/metricbeat/module/prometheus/collector/histogram_test.go b/x-pack/metricbeat/module/prometheus/collector/histogram_test.go index b0906068e76..460d0f3fffe 100644 --- a/x-pack/metricbeat/module/prometheus/collector/histogram_test.go +++ b/x-pack/metricbeat/module/prometheus/collector/histogram_test.go @@ -17,7 +17,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" ) -// TestPromHistogramToES tests that calling promHistogramToES multiple +// TestPromHistogramToES tests that calling PromHistogramToES multiple // times with the same cache produces each time the expected results. func TestPromHistogramToES(t *testing.T) { type sample struct { @@ -398,7 +398,7 @@ func TestPromHistogramToES(t *testing.T) { for i, s := range c.samples { t.Logf("#%d: %+v", i, s.histogram) - result := promHistogramToES(cache, metricName, labels, &s.histogram) + result := PromHistogramToES(cache, metricName, labels, &s.histogram) assert.EqualValues(t, s.expected, result) } }) diff --git a/x-pack/metricbeat/module/prometheus/remote_write/_meta/data.json b/x-pack/metricbeat/module/prometheus/remote_write/_meta/data.json new file mode 100644 index 00000000000..a29e8381a31 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/remote_write/_meta/data.json @@ -0,0 +1,51 @@ +{ + "@timestamp": "2020-02-28T13:55:37.221Z", + "@metadata": { + "beat": "metricbeat", + "type": "_doc", + "version": "8.0.0" + }, + "service": { + "type": "prometheus" + }, + "agent": { + "version": "8.0.0", + "type": "metricbeat", + "ephemeral_id": "ead09243-0aa0-4fd2-8732-1e09a6d36338", + "hostname": "host1", + "id": "bd12ee45-881f-48e4-af20-13b139548607" + }, + "ecs": { + "version": "1.4.0" + }, + "host": {}, + "event": { + "dataset": "prometheus.remote_write", + "module": "prometheus" + }, + "metricset": { + "name": "remote_write" + }, + "prometheus": { + "metrics": { + "container_tasks_state": 0 + }, + "labels": { + "name": "nodeexporter", + "id": "/docker/1d6ec1931c9b527d4fe8e28d9c798f6ec612f48af51949f3219b5ca77e120b10", + "container_label_com_docker_compose_oneoff": "False", + "instance": "cadvisor:8080", + "container_label_com_docker_compose_service": "nodeexporter", + "state": "iowaiting", + "monitor": "docker-host-alpha", + "container_label_com_docker_compose_project": "dockprom", + "job": "cadvisor", + "image": "prom/node-exporter:v0.18.1", + "container_label_maintainer": "The Prometheus Authors ", + "container_label_com_docker_compose_config_hash": "2cc2fedf6da5ff0996a209d9801fb74962a8f4c21e44be03ed82659817d9e7f9", + "container_label_com_docker_compose_version": "1.24.1", + "container_label_com_docker_compose_container_number": "1", + "container_label_org_label_schema_group": "monitoring" + } + } +} diff --git a/x-pack/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc b/x-pack/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc new file mode 100644 index 00000000000..99f5e120d1a --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc @@ -0,0 +1,63 @@ +This is the remote_write metricset of the module prometheus. This metricset can receive metrics from a Prometheus server that +has configured https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write[remote_write] setting accordingly, for instance: + +["source","yaml",subs="attributes"] +------------------------------------------------------------------------------ +remote_write: + - url: "http://localhost:9201/write" +------------------------------------------------------------------------------ + + +TIP: In order to assure the health of the whole queue, the following configuration + https://prometheus.io/docs/practices/remote_write/#parameters[parameters] should be considered: + +- `max_shards`: Sets the maximum number of parallelism with which Prometheus will try to send samples to Metricbeat. +It is recommended that this setting should be equal to the number of cores of the machine where Metricbeat runs. +Metricbeat can handle connections in parallel and hence setting `max_shards` to the number of parallelism that +Metricbeat can actually achieve is the optimal queue configuration. +- `max_samples_per_send`: Sets the number of samples to batch together for each send. Recommended values are +between 100 (default) and 1000. Having a bigger batch can lead to improved throughput and in more efficient +storage since Metricbeat groups metrics with the same labels into same event documents. +However this will increase the memory usage of Metricbeat. +- `capacity`: It is recommended to set capacity to 3-5 times `max_samples_per_send`. +Capacity sets the number of samples that are queued in memory per shard, and hence capacity should be high enough so as to +be able to cover `max_samples_per_send`. + + +Metrics sent to the http endpoint will be put by default under the `prometheus.metrics` prefix with their labels under `prometheus.labels`. +A basic configuration would look like: + +["source","yaml",subs="attributes"] +------------------------------------------------------------------------------ +- module: prometheus + metricsets: ["remote_write"] + host: "localhost" + port: "9201" +------------------------------------------------------------------------------ + + + +Also consider using secure settings for the server, configuring the module with TLS/SSL as shown: + +["source","yaml",subs="attributes"] +------------------------------------------------------------------------------ +- module: prometheus + metricsets: ["remote_write"] + host: "localhost" + ssl.certificate: "/etc/pki/server/cert.pem" + ssl.key: "/etc/pki/server/cert.key" + port: "9201" +------------------------------------------------------------------------------ + +and on Prometheus side: + +["source","yaml",subs="attributes"] +------------------------------------------------------------------------------ +remote_write: + - url: "https://localhost:9201/write" + tls_config: + cert_file: "/etc/prometheus/my_key.pem" + key_file: "/etc/prometheus/my_key.key" + # Disable validation of the server certificate. + #insecure_skip_verify: true +------------------------------------------------------------------------------ diff --git a/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml b/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml new file mode 100644 index 00000000000..e3b2f050a40 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml @@ -0,0 +1,6 @@ +- name: remote_write + type: group + description: > + remote write metrics from Prometheus server + release: beta + fields: diff --git a/x-pack/metricbeat/module/prometheus/remote_write/config.go b/x-pack/metricbeat/module/prometheus/remote_write/config.go new file mode 100644 index 00000000000..2334d6e5ff9 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/remote_write/config.go @@ -0,0 +1,20 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package remote_write + +import "errors" + +type config struct { + UseTypes bool `config:"use_types"` + RateCounters bool `config:"rate_counters"` +} + +func (c *config) Validate() error { + if c.RateCounters && !c.UseTypes { + return errors.New("'rate_counters' can only be enabled when `use_types` is also enabled") + } + + return nil +} diff --git a/x-pack/metricbeat/module/prometheus/remote_write/data.go b/x-pack/metricbeat/module/prometheus/remote_write/data.go new file mode 100644 index 00000000000..e6ad04c200d --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/remote_write/data.go @@ -0,0 +1,273 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package remote_write + +import ( + "math" + "strconv" + "strings" + "time" + + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/model" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/module/prometheus/remote_write" + xcollector "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus/collector" +) + +type histogram struct { + timestamp time.Time + buckets []*dto.Bucket + labels common.MapStr + metricName string +} + +func remoteWriteEventsGeneratorFactory(base mb.BaseMetricSet) (remote_write.RemoteWriteEventsGenerator, error) { + config := config{} + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + if config.UseTypes { + // use a counter cache with a timeout of 5x the period, as a safe value + // to make sure that all counters are available between fetches + counters := xcollector.NewCounterCache(base.Module().Config().Period * 5) + + g := RemoteWriteTypedGenerator{ + CounterCache: counters, + RateCounters: config.RateCounters, + } + + return &g, nil + } + + return remote_write.DefaultRemoteWriteEventsGeneratorFactory(base) +} + +type RemoteWriteTypedGenerator struct { + CounterCache xcollector.CounterCache + RateCounters bool +} + +func (g *RemoteWriteTypedGenerator) Start() { + cfgwarn.Beta("Prometheus 'use_types' setting is beta") + + if g.RateCounters { + cfgwarn.Experimental("Prometheus 'rate_counters' setting is experimental") + } + + g.CounterCache.Start() +} + +func (g *RemoteWriteTypedGenerator) Stop() { + logp.Debug("prometheus.collector.cache", "stopping CounterCache") + g.CounterCache.Stop() +} + +func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[string]mb.Event { + var data common.MapStr + var histograms map[string]histogram + eventList := map[string]mb.Event{} + + for _, metric := range metrics { + labels := common.MapStr{} + + if metric == nil { + continue + } + name := string(metric.Metric["__name__"]) + delete(metric.Metric, "__name__") + + for k, v := range metric.Metric { + labels[string(k)] = v + } + + promType := findType(name, labels) + val := float64(metric.Value) + if !math.IsNaN(val) && !math.IsInf(val, 0) { + // join metrics with same labels in a single event + labelsHash := labels.String() + if _, ok := eventList[labelsHash]; !ok { + eventList[labelsHash] = mb.Event{ + ModuleFields: common.MapStr{ + "metrics": common.MapStr{}, + }, + } + + // Add labels + if len(labels) > 0 { + eventList[labelsHash].ModuleFields["labels"] = labels + } + } + + e := eventList[labelsHash] + e.Timestamp = metric.Timestamp.Time() + switch promType { + case "counter_float": + data = common.MapStr{ + name: g.rateCounterFloat64(name, labels, val), + } + case "counter_int": + //events = append(events, collector.PromEvent{ + // Data: common.MapStr{ + // name: g.rateCounterUint64(name, labels, uint64(val)), + // }, + // Labels: labels, + // Timestamp: metric.Timestamp.Time(), + //}) + data = common.MapStr{ + name: g.rateCounterUint64(name, labels, uint64(val)), + } + case "other": + //events = append(events, collector.PromEvent{ + // Data: common.MapStr{ + // name: common.MapStr{ + // "value": val, + // }, + // }, + // Labels: labels, + // Timestamp: metric.Timestamp.Time(), + //}) + data = common.MapStr{ + name: common.MapStr{ + "value": val, + }, + } + case "histogram": + labelsClone := labels.Clone() + labelsClone.Delete("le") + + histKey := name + labelsClone.String() + + le, _ := labels.GetValue("le") + upperBound := le.(string) + + bucket, err := strconv.ParseFloat(upperBound, 64) + if err != nil { + continue + } + v := uint64(val) + b := &dto.Bucket{ + CumulativeCount: &v, + UpperBound: &bucket, + } + hist, ok := histograms[histKey] + if !ok { + hist = histogram{} + } + hist.buckets = append(hist.buckets, b) + hist.timestamp = metric.Timestamp.Time() + hist.labels = labels + hist.metricName = name + continue + } + e.ModuleFields["metrics"].(common.MapStr).Update(data) + } + } + + g.processPromHistograms(eventList, histograms) + return eventList +} + +// rateCounterUint64 fills a counter value and optionally adds the rate if rate_counters is enabled +func (g *RemoteWriteTypedGenerator) rateCounterUint64(name string, labels common.MapStr, value uint64) common.MapStr { + d := common.MapStr{ + "counter": value, + } + + if g.RateCounters { + d["rate"], _ = g.CounterCache.RateUint64(name+labels.String(), value) + } + + return d +} + +// rateCounterFloat64 fills a counter value and optionally adds the rate if rate_counters is enabled +func (g *RemoteWriteTypedGenerator) rateCounterFloat64(name string, labels common.MapStr, value float64) common.MapStr { + d := common.MapStr{ + "counter": value, + } + + if g.RateCounters { + d["rate"], _ = g.CounterCache.RateFloat64(name+labels.String(), value) + } + + return d +} + +func (g *RemoteWriteTypedGenerator) processPromHistograms(eventList map[string]mb.Event, histograms map[string]histogram) { + for name, histogram := range histograms { + labelsHash := histogram.labels.String() + if _, ok := eventList[labelsHash]; !ok { + eventList[labelsHash] = mb.Event{ + ModuleFields: common.MapStr{ + "metrics": common.MapStr{}, + }, + } + + // Add labels + if len(histogram.labels) > 0 { + eventList[labelsHash].ModuleFields["labels"] = histogram.labels + } + } + + e := eventList[labelsHash] + e.Timestamp = histogram.timestamp + + + hist := dto.Histogram{ + Bucket: histogram.buckets, + } + + //events = append(events, collector.PromEvent{ + // Data: common.MapStr{ + // name: common.MapStr{ + // "histogram": xcollector.PromHistogramToES(g.CounterCache, histogram.metricName, histogram.labels, &hist), + // }, + // }, + // Labels: histogram.labels, + // Timestamp: histogram.timestamp, + //}) + + data := common.MapStr{ + name: common.MapStr{ + "histogram": xcollector.PromHistogramToES(g.CounterCache, histogram.metricName, histogram.labels, &hist), + }, + } + e.ModuleFields["metrics"].(common.MapStr).Update(data) + } +} + +func findType(metricName string, labels common.MapStr) string { + leLabel := false + if _, ok := labels["le"]; ok { + leLabel = true + } + if strings.Contains(metricName, "_total") || strings.Contains(metricName, "_sum") { + return "counter_float" + } else if strings.Contains(metricName, "_sum") { + return "counter_int" + } else if strings.Contains(metricName, "_bucket") && leLabel { + return "histogram" + } + return "other" +} diff --git a/x-pack/metricbeat/module/prometheus/remote_write/remote_write.go b/x-pack/metricbeat/module/prometheus/remote_write/remote_write.go new file mode 100644 index 00000000000..298dae7dc86 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/remote_write/remote_write.go @@ -0,0 +1,35 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package remote_write + +import ( + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/beats/v7/metricbeat/module/prometheus/remote_write" +) + +func init() { + mb.Registry.MustAddMetricSet("prometheus", "remote_write", + remote_write.MetricSetBuilder("prometheus", remoteWriteEventsGeneratorFactory), + mb.WithHostParser(parse.EmptyHostParser), + + // must replace ensures that we are replacing the oss implementation with this one + // so we can make use of ES histograms (basic only) when use_types is enabled + mb.MustReplace(), + ) +} From 456f909d2552e7c22724e4eee096a537f1d297db Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 16 Jul 2020 10:07:07 +0300 Subject: [PATCH 02/24] fmt update Signed-off-by: chrismark --- metricbeat/docs/fields.asciidoc | 7 ++- .../module/prometheus/collector/data.go | 5 +- .../module/prometheus/remote_write/data.go | 1 - .../prometheus/remote_write/remote_write.go | 20 ++++-- ...prometheus.yml => prometheus.yml.disabled} | 18 +++--- .../{system.yml.disabled => system.yml} | 0 .../prometheus/remote_write/_meta/data.json | 51 --------------- .../remote_write/_meta/docs.asciidoc | 63 ------------------- .../prometheus/remote_write/_meta/fields.yml | 7 +-- .../module/prometheus/remote_write/data.go | 2 +- 10 files changed, 36 insertions(+), 138 deletions(-) rename metricbeat/modules.d/{prometheus.yml => prometheus.yml.disabled} (89%) rename metricbeat/modules.d/{system.yml.disabled => system.yml} (100%) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 2c9c7c00124..0e666085fe0 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -35084,7 +35084,12 @@ type: object *`prometheus.*.histogram`*:: + -- -Prometheus histogram metric - release: ga +Prometheus histogram metric - release: ga - name: remote_write + type: group + description: > + remote write metrics from Prometheus server + release: beta + fields: type: object diff --git a/metricbeat/module/prometheus/collector/data.go b/metricbeat/module/prometheus/collector/data.go index 6fc2823316f..717ec2dc732 100644 --- a/metricbeat/module/prometheus/collector/data.go +++ b/metricbeat/module/prometheus/collector/data.go @@ -18,11 +18,12 @@ package collector import ( + "math" + "strconv" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/helper/labelhash" "github.com/elastic/beats/v7/metricbeat/mb" - "math" - "strconv" dto "github.com/prometheus/client_model/go" ) diff --git a/metricbeat/module/prometheus/remote_write/data.go b/metricbeat/module/prometheus/remote_write/data.go index d15749ab932..8381fafd19e 100644 --- a/metricbeat/module/prometheus/remote_write/data.go +++ b/metricbeat/module/prometheus/remote_write/data.go @@ -26,7 +26,6 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb" ) - // DefaultRemoteWriteEventsGeneratorFactory returns the default prometheus events generator func DefaultRemoteWriteEventsGeneratorFactory(ms mb.BaseMetricSet) (RemoteWriteEventsGenerator, error) { return &remoteWriteEventGenerator{}, nil diff --git a/metricbeat/module/prometheus/remote_write/remote_write.go b/metricbeat/module/prometheus/remote_write/remote_write.go index e4c4d5567f4..7e6a60d3962 100644 --- a/metricbeat/module/prometheus/remote_write/remote_write.go +++ b/metricbeat/module/prometheus/remote_write/remote_write.go @@ -56,9 +56,9 @@ type RemoteWriteEventsGeneratorFactory func(ms mb.BaseMetricSet) (RemoteWriteEve type MetricSet struct { mb.BaseMetricSet - server serverhelper.Server - events chan mb.Event - promEventsGen RemoteWriteEventsGenerator + server serverhelper.Server + events chan mb.Event + promEventsGen RemoteWriteEventsGenerator eventGenStarted bool } @@ -110,7 +110,6 @@ func MetricSetBuilder(namespace string, genFactory RemoteWriteEventsGeneratorFac } } - func (m *MetricSet) Run(reporter mb.PushReporterV2) { // Start event watcher m.server.Start() @@ -126,7 +125,20 @@ func (m *MetricSet) Run(reporter mb.PushReporterV2) { } } +// Close stops the metricset +func (m *MetricSet) Close() error { + if m.eventGenStarted { + m.promEventsGen.Stop() + } + return nil +} + func (m *MetricSet) handleFunc(writer http.ResponseWriter, req *http.Request) { + if !m.eventGenStarted { + m.promEventsGen.Start() + m.eventGenStarted = true + } + compressed, err := ioutil.ReadAll(req.Body) if err != nil { m.Logger().Errorf("Read error %v", err) diff --git a/metricbeat/modules.d/prometheus.yml b/metricbeat/modules.d/prometheus.yml.disabled similarity index 89% rename from metricbeat/modules.d/prometheus.yml rename to metricbeat/modules.d/prometheus.yml.disabled index d0f46d0c85f..82f45573931 100644 --- a/metricbeat/modules.d/prometheus.yml +++ b/metricbeat/modules.d/prometheus.yml.disabled @@ -2,11 +2,11 @@ # Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-prometheus.html # Metrics collected from a Prometheus endpoint -#- module: prometheus -# period: 10s -# metricsets: ["collector"] -# hosts: ["localhost:9090"] -# metrics_path: /metrics +- module: prometheus + period: 10s + metricsets: ["collector"] + hosts: ["localhost:9090"] + metrics_path: /metrics #metrics_filters: # include: [] # exclude: [] @@ -20,10 +20,10 @@ # Metrics sent by a Prometheus server using remote_write option -- module: prometheus - metricsets: ["remote_write"] - host: "localhost" - port: "9201" +#- module: prometheus +# metricsets: ["remote_write"] +# host: "localhost" +# port: "9201" # Secure settings for the server using TLS/SSL: #ssl.certificate: "/etc/pki/server/cert.pem" diff --git a/metricbeat/modules.d/system.yml.disabled b/metricbeat/modules.d/system.yml similarity index 100% rename from metricbeat/modules.d/system.yml.disabled rename to metricbeat/modules.d/system.yml diff --git a/x-pack/metricbeat/module/prometheus/remote_write/_meta/data.json b/x-pack/metricbeat/module/prometheus/remote_write/_meta/data.json index a29e8381a31..e69de29bb2d 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/_meta/data.json +++ b/x-pack/metricbeat/module/prometheus/remote_write/_meta/data.json @@ -1,51 +0,0 @@ -{ - "@timestamp": "2020-02-28T13:55:37.221Z", - "@metadata": { - "beat": "metricbeat", - "type": "_doc", - "version": "8.0.0" - }, - "service": { - "type": "prometheus" - }, - "agent": { - "version": "8.0.0", - "type": "metricbeat", - "ephemeral_id": "ead09243-0aa0-4fd2-8732-1e09a6d36338", - "hostname": "host1", - "id": "bd12ee45-881f-48e4-af20-13b139548607" - }, - "ecs": { - "version": "1.4.0" - }, - "host": {}, - "event": { - "dataset": "prometheus.remote_write", - "module": "prometheus" - }, - "metricset": { - "name": "remote_write" - }, - "prometheus": { - "metrics": { - "container_tasks_state": 0 - }, - "labels": { - "name": "nodeexporter", - "id": "/docker/1d6ec1931c9b527d4fe8e28d9c798f6ec612f48af51949f3219b5ca77e120b10", - "container_label_com_docker_compose_oneoff": "False", - "instance": "cadvisor:8080", - "container_label_com_docker_compose_service": "nodeexporter", - "state": "iowaiting", - "monitor": "docker-host-alpha", - "container_label_com_docker_compose_project": "dockprom", - "job": "cadvisor", - "image": "prom/node-exporter:v0.18.1", - "container_label_maintainer": "The Prometheus Authors ", - "container_label_com_docker_compose_config_hash": "2cc2fedf6da5ff0996a209d9801fb74962a8f4c21e44be03ed82659817d9e7f9", - "container_label_com_docker_compose_version": "1.24.1", - "container_label_com_docker_compose_container_number": "1", - "container_label_org_label_schema_group": "monitoring" - } - } -} diff --git a/x-pack/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc b/x-pack/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc index 99f5e120d1a..e69de29bb2d 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc @@ -1,63 +0,0 @@ -This is the remote_write metricset of the module prometheus. This metricset can receive metrics from a Prometheus server that -has configured https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write[remote_write] setting accordingly, for instance: - -["source","yaml",subs="attributes"] ------------------------------------------------------------------------------- -remote_write: - - url: "http://localhost:9201/write" ------------------------------------------------------------------------------- - - -TIP: In order to assure the health of the whole queue, the following configuration - https://prometheus.io/docs/practices/remote_write/#parameters[parameters] should be considered: - -- `max_shards`: Sets the maximum number of parallelism with which Prometheus will try to send samples to Metricbeat. -It is recommended that this setting should be equal to the number of cores of the machine where Metricbeat runs. -Metricbeat can handle connections in parallel and hence setting `max_shards` to the number of parallelism that -Metricbeat can actually achieve is the optimal queue configuration. -- `max_samples_per_send`: Sets the number of samples to batch together for each send. Recommended values are -between 100 (default) and 1000. Having a bigger batch can lead to improved throughput and in more efficient -storage since Metricbeat groups metrics with the same labels into same event documents. -However this will increase the memory usage of Metricbeat. -- `capacity`: It is recommended to set capacity to 3-5 times `max_samples_per_send`. -Capacity sets the number of samples that are queued in memory per shard, and hence capacity should be high enough so as to -be able to cover `max_samples_per_send`. - - -Metrics sent to the http endpoint will be put by default under the `prometheus.metrics` prefix with their labels under `prometheus.labels`. -A basic configuration would look like: - -["source","yaml",subs="attributes"] ------------------------------------------------------------------------------- -- module: prometheus - metricsets: ["remote_write"] - host: "localhost" - port: "9201" ------------------------------------------------------------------------------- - - - -Also consider using secure settings for the server, configuring the module with TLS/SSL as shown: - -["source","yaml",subs="attributes"] ------------------------------------------------------------------------------- -- module: prometheus - metricsets: ["remote_write"] - host: "localhost" - ssl.certificate: "/etc/pki/server/cert.pem" - ssl.key: "/etc/pki/server/cert.key" - port: "9201" ------------------------------------------------------------------------------- - -and on Prometheus side: - -["source","yaml",subs="attributes"] ------------------------------------------------------------------------------- -remote_write: - - url: "https://localhost:9201/write" - tls_config: - cert_file: "/etc/prometheus/my_key.pem" - key_file: "/etc/prometheus/my_key.key" - # Disable validation of the server certificate. - #insecure_skip_verify: true ------------------------------------------------------------------------------- diff --git a/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml b/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml index e3b2f050a40..15e53c4a8eb 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml +++ b/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml @@ -1,6 +1 @@ -- name: remote_write - type: group - description: > - remote write metrics from Prometheus server - release: beta - fields: +return nil diff --git a/x-pack/metricbeat/module/prometheus/remote_write/data.go b/x-pack/metricbeat/module/prometheus/remote_write/data.go index e6ad04c200d..646b17833a5 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/data.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/data.go @@ -79,7 +79,7 @@ func (g *RemoteWriteTypedGenerator) Start() { } func (g *RemoteWriteTypedGenerator) Stop() { - logp.Debug("prometheus.collector.cache", "stopping CounterCache") + logp.Debug("prometheus.remote_write.cache", "stopping CounterCache") g.CounterCache.Stop() } From ccd18571a29a99f40936e9fb887d50ca134fc150 Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 16 Jul 2020 11:05:37 +0300 Subject: [PATCH 03/24] fmt update x-pack Signed-off-by: chrismark --- x-pack/metricbeat/include/list.go | 1 + x-pack/metricbeat/module/prometheus/fields.go | 2 +- .../module/prometheus/remote_write/data.go | 20 +++---------------- .../prometheus/remote_write/remote_write.go | 19 +++--------------- 4 files changed, 8 insertions(+), 34 deletions(-) diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 71876550fc8..045b9a4592a 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -53,6 +53,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/oracle/tablespace" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus/collector" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus/remote_write" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/redisenterprise" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/sql" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/sql/query" diff --git a/x-pack/metricbeat/module/prometheus/fields.go b/x-pack/metricbeat/module/prometheus/fields.go index 63330de9110..bf8cc4c176e 100644 --- a/x-pack/metricbeat/module/prometheus/fields.go +++ b/x-pack/metricbeat/module/prometheus/fields.go @@ -19,5 +19,5 @@ func init() { // AssetPrometheus returns asset data. // This is the base64 encoded gzipped contents of module/prometheus. func AssetPrometheus() string { - return "eJzEkktqAzEMhvdzih8vQ5MDeNEzFLosJSi24rjxC0tTmtuXZIYw9EGzKMQ7+/+QPgmvceSTRes1sx54lPVHI3ccAI2a2MI8XSPoqbFHZu3RiRkAz+J6bBprsXgcAOBZSQXiOp3Zfa8ZhEUNLr7VWHQzAJ0Tk7BFoAEQVo0liMWLEUnmAeag2szrAOwjJy/20mGNQpmXzpvV5p3SyJcYF02Luntjp/PTdNlOia/jLvH3ZJuptVjCjJmVmZkfxjyfxVSBxsDzZn6XdHUsyv1+mrPAn6Kd9I7LPHf3N7seomgNnfKNwl/5/3G+Vl36Ts7LX/4ZAAD//xMCFRw=" + return "eJzEkstqMzEMhfd+ioOX4U8ewIv/GQpdlhKUGcVx4xuWXJq3L8kMYXqjWRSinX0O0qeD1jjyyaG2klgP3GX9Vmk4GkCDRnawD1cJeqo8IrG2MIg1wMgytFA1lOzw3wDAo5IKZGh09u5bSSAsenAeawlZNwZoHJmEHTwZQFg1ZC8OT1Yk2n+wB9Vqnw2wDxxHcZcJa2RKvGTerDavFDtfZFwwHcruhQedv6bHdlLG0neRvyrbRLWG7GebXdnZ882a51ps5al7npP5GXIoPSu3+2HOAL+CNtI7hnmePt7MegiixTdKNwJ/9v8N87Xrkndi/njlUzXW3jJyiOY9AAD//85gGik=" } diff --git a/x-pack/metricbeat/module/prometheus/remote_write/data.go b/x-pack/metricbeat/module/prometheus/remote_write/data.go index 646b17833a5..50ae036c000 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/data.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/data.go @@ -1,19 +1,6 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. package remote_write @@ -233,7 +220,6 @@ func (g *RemoteWriteTypedGenerator) processPromHistograms(eventList map[string]m e := eventList[labelsHash] e.Timestamp = histogram.timestamp - hist := dto.Histogram{ Bucket: histogram.buckets, } diff --git a/x-pack/metricbeat/module/prometheus/remote_write/remote_write.go b/x-pack/metricbeat/module/prometheus/remote_write/remote_write.go index 298dae7dc86..3244500163f 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/remote_write.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/remote_write.go @@ -1,19 +1,6 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. package remote_write From 3a8cf2ea606f8b1fd5bc3f030ce3630e9537f852 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 17 Jul 2020 11:28:59 +0300 Subject: [PATCH 04/24] fixes Signed-off-by: chrismark --- .../prometheus/remote_write/remote_write.go | 12 ++++++++-- .../prometheus/remote_write/_meta/data.json | 24 +++++++++++++++++++ .../module/prometheus/remote_write/data.go | 8 +++---- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/metricbeat/module/prometheus/remote_write/remote_write.go b/metricbeat/module/prometheus/remote_write/remote_write.go index 7e6a60d3962..09544ef0964 100644 --- a/metricbeat/module/prometheus/remote_write/remote_write.go +++ b/metricbeat/module/prometheus/remote_write/remote_write.go @@ -20,6 +20,7 @@ package remote_write import ( "io/ioutil" "net/http" + "fmt" "github.com/gogo/protobuf/proto" "github.com/golang/snappy" @@ -97,6 +98,7 @@ func MetricSetBuilder(namespace string, genFactory RemoteWriteEventsGeneratorFac m := &MetricSet{ BaseMetricSet: base, + events: make(chan mb.Event), promEventsGen: promEventsGen, eventGenStarted: false, } @@ -113,13 +115,15 @@ func MetricSetBuilder(namespace string, genFactory RemoteWriteEventsGeneratorFac func (m *MetricSet) Run(reporter mb.PushReporterV2) { // Start event watcher m.server.Start() - + //fmt.Println("Run server") for { select { case <-reporter.Done(): m.server.Stop() return case e := <-m.events: + //fmt.Println("reporter") + //fmt.Println(e) reporter.Event(e) } } @@ -134,6 +138,7 @@ func (m *MetricSet) Close() error { } func (m *MetricSet) handleFunc(writer http.ResponseWriter, req *http.Request) { + fmt.Println("handleFunc 1") if !m.eventGenStarted { m.promEventsGen.Start() m.eventGenStarted = true @@ -159,11 +164,14 @@ func (m *MetricSet) handleFunc(writer http.ResponseWriter, req *http.Request) { http.Error(writer, err.Error(), http.StatusBadRequest) return } - + //fmt.Println("handleFunc 2") samples := protoToSamples(&protoReq) + //fmt.Println("handleFunc 3") events := m.promEventsGen.GenerateEvents(samples) + //fmt.Println("handleFunc 4") for _, e := range events { + //fmt.Println(e) select { case <-req.Context().Done(): return diff --git a/x-pack/metricbeat/module/prometheus/remote_write/_meta/data.json b/x-pack/metricbeat/module/prometheus/remote_write/_meta/data.json index e69de29bb2d..54a1b92e428 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/_meta/data.json +++ b/x-pack/metricbeat/module/prometheus/remote_write/_meta/data.json @@ -0,0 +1,24 @@ +{ + "@timestamp": "2020-07-17T08:23:53.958Z", + "service": { + "type": "prometheus" + }, + "event": { + "dataset": "prometheus.remote_write", + "module": "prometheus" + }, + "metricset": { + "name": "remote_write" + }, + "prometheus": { + "labels": { + "instance": "nodeexporter:9100", + "job": "nodeexporter", + "device": "eth0" + }, + "node_network_transmit_packets_total": { + "counter": 609, + "rate": 3 + } + } +} diff --git a/x-pack/metricbeat/module/prometheus/remote_write/data.go b/x-pack/metricbeat/module/prometheus/remote_write/data.go index 50ae036c000..a735f080cd6 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/data.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/data.go @@ -95,9 +95,7 @@ func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str labelsHash := labels.String() if _, ok := eventList[labelsHash]; !ok { eventList[labelsHash] = mb.Event{ - ModuleFields: common.MapStr{ - "metrics": common.MapStr{}, - }, + ModuleFields: common.MapStr{}, } // Add labels @@ -146,7 +144,7 @@ func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str histKey := name + labelsClone.String() le, _ := labels.GetValue("le") - upperBound := le.(string) + upperBound := string(le.(model.LabelValue)) bucket, err := strconv.ParseFloat(upperBound, 64) if err != nil { @@ -167,7 +165,7 @@ func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str hist.metricName = name continue } - e.ModuleFields["metrics"].(common.MapStr).Update(data) + e.ModuleFields.Update(data) } } From 063c2699a56a1b12f2fe0d8cd137c43cddb92431 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 17 Jul 2020 13:31:23 +0300 Subject: [PATCH 05/24] Add unit tests for counter, gauge, summary metric types Signed-off-by: chrismark --- .../module/prometheus/remote_write/data.go | 36 +- .../remote_write/remote_write_test.go | 661 ++++++++++++++++++ 2 files changed, 668 insertions(+), 29 deletions(-) create mode 100644 x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go diff --git a/x-pack/metricbeat/module/prometheus/remote_write/data.go b/x-pack/metricbeat/module/prometheus/remote_write/data.go index a735f080cd6..399956e9d27 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/data.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/data.go @@ -100,6 +100,11 @@ func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str // Add labels if len(labels) > 0 { + if promType == "histogram" { + labelsClone := labels.Clone() + labelsClone.Delete("le") + labels = labelsClone.Clone() + } eventList[labelsHash].ModuleFields["labels"] = labels } } @@ -112,26 +117,10 @@ func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str name: g.rateCounterFloat64(name, labels, val), } case "counter_int": - //events = append(events, collector.PromEvent{ - // Data: common.MapStr{ - // name: g.rateCounterUint64(name, labels, uint64(val)), - // }, - // Labels: labels, - // Timestamp: metric.Timestamp.Time(), - //}) data = common.MapStr{ name: g.rateCounterUint64(name, labels, uint64(val)), } case "other": - //events = append(events, collector.PromEvent{ - // Data: common.MapStr{ - // name: common.MapStr{ - // "value": val, - // }, - // }, - // Labels: labels, - // Timestamp: metric.Timestamp.Time(), - //}) data = common.MapStr{ name: common.MapStr{ "value": val, @@ -161,7 +150,7 @@ func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str } hist.buckets = append(hist.buckets, b) hist.timestamp = metric.Timestamp.Time() - hist.labels = labels + hist.labels = labelsClone hist.metricName = name continue } @@ -191,7 +180,6 @@ func (g *RemoteWriteTypedGenerator) rateCounterFloat64(name string, labels commo d := common.MapStr{ "counter": value, } - if g.RateCounters { d["rate"], _ = g.CounterCache.RateFloat64(name+labels.String(), value) } @@ -222,16 +210,6 @@ func (g *RemoteWriteTypedGenerator) processPromHistograms(eventList map[string]m Bucket: histogram.buckets, } - //events = append(events, collector.PromEvent{ - // Data: common.MapStr{ - // name: common.MapStr{ - // "histogram": xcollector.PromHistogramToES(g.CounterCache, histogram.metricName, histogram.labels, &hist), - // }, - // }, - // Labels: histogram.labels, - // Timestamp: histogram.timestamp, - //}) - data := common.MapStr{ name: common.MapStr{ "histogram": xcollector.PromHistogramToES(g.CounterCache, histogram.metricName, histogram.labels, &hist), @@ -248,7 +226,7 @@ func findType(metricName string, labels common.MapStr) string { } if strings.Contains(metricName, "_total") || strings.Contains(metricName, "_sum") { return "counter_float" - } else if strings.Contains(metricName, "_sum") { + } else if strings.Contains(metricName, "_count") { return "counter_int" } else if strings.Contains(metricName, "_bucket") && leLabel { return "histogram" diff --git a/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go b/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go new file mode 100644 index 00000000000..e0bd350a7c4 --- /dev/null +++ b/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go @@ -0,0 +1,661 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !integration + +package remote_write + +import ( + "github.com/prometheus/common/model" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/common" + xcollector "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus/collector" +) + +// TestGenerateEventsCounter tests counter simple cases +func TestGenerateEventsCounter(t *testing.T) { + + counters := xcollector.NewCounterCache(1 * time.Second) + + g := RemoteWriteTypedGenerator{ + CounterCache: counters, + RateCounters: true, + } + g.CounterCache.Start() + labels := common.MapStr{ + "listener_name": model.LabelValue("http"), + } + + // first fetch + metrics := model.Samples{ + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_closed_total", + "listener_name": "http", + }, + Value: model.SampleValue(42), + Timestamp: model.Time(424242), + }, + } + events := g.GenerateEvents(metrics) + + + expected := common.MapStr{ + "net_conntrack_listener_conn_closed_total": common.MapStr{ + "counter": float64(42), + "rate": float64(0), + }, + "labels": labels, + } + + assert.Equal(t, len(events), 1) + e := events[labels.String()] + assert.EqualValues(t, e.ModuleFields, expected) + + + // repeat in order to test the rate + metrics = model.Samples{ + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_closed_total", + "listener_name": "http", + }, + Value: model.SampleValue(45), + Timestamp: model.Time(424242), + }, + } + events = g.GenerateEvents(metrics) + + expected = common.MapStr{ + "net_conntrack_listener_conn_closed_total": common.MapStr{ + "counter": float64(45), + "rate": float64(3), + }, + "labels": labels, + } + + assert.Equal(t, len(events), 1) + e = events[labels.String()] + assert.EqualValues(t, e.ModuleFields, expected) + +} + + +// TestGenerateEventsCounterSameLabels tests multiple counters with same labels +func TestGenerateEventsCounterSameLabels(t *testing.T) { + + counters := xcollector.NewCounterCache(1 * time.Second) + + g := RemoteWriteTypedGenerator{ + CounterCache: counters, + RateCounters: true, + } + g.CounterCache.Start() + labels := common.MapStr{ + "listener_name": model.LabelValue("http"), + } + + // first fetch + metrics := model.Samples{ + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_closed_total", + "listener_name": "http", + }, + Value: model.SampleValue(42), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_panic_total", + "listener_name": "http", + }, + Value: model.SampleValue(43), + Timestamp: model.Time(424242), + }, + } + events := g.GenerateEvents(metrics) + + + expected := common.MapStr{ + "net_conntrack_listener_conn_closed_total": common.MapStr{ + "counter": float64(42), + "rate": float64(0), + }, + "net_conntrack_listener_conn_panic_total": common.MapStr{ + "counter": float64(43), + "rate": float64(0), + }, + "labels": labels, + } + + assert.Equal(t, len(events), 1) + e := events[labels.String()] + assert.EqualValues(t, e.ModuleFields, expected) + + + // repeat in order to test the rate + metrics = model.Samples{ + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_closed_total", + "listener_name": "http", + }, + Value: model.SampleValue(45), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_panic_total", + "listener_name": "http", + }, + Value: model.SampleValue(47), + Timestamp: model.Time(424242), + }, + } + events = g.GenerateEvents(metrics) + + expected = common.MapStr{ + "net_conntrack_listener_conn_closed_total": common.MapStr{ + "counter": float64(45), + "rate": float64(3), + }, + "net_conntrack_listener_conn_panic_total": common.MapStr{ + "counter": float64(47), + "rate": float64(4), + }, + "labels": labels, + } + + assert.Equal(t, len(events), 1) + e = events[labels.String()] + assert.EqualValues(t, e.ModuleFields, expected) + +} + + +// TestGenerateEventsCounterDifferentLabels tests multiple counters with different labels +func TestGenerateEventsCounterDifferentLabels(t *testing.T) { + + counters := xcollector.NewCounterCache(1 * time.Second) + + g := RemoteWriteTypedGenerator{ + CounterCache: counters, + RateCounters: true, + } + g.CounterCache.Start() + labels := common.MapStr{ + "listener_name": model.LabelValue("http"), + } + labels2 := common.MapStr{ + "listener_name": model.LabelValue("http"), + "device": model.LabelValue("eth0"), + } + + // first fetch + metrics := model.Samples{ + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_closed_total", + "listener_name": "http", + }, + Value: model.SampleValue(42), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_panic_total", + "listener_name": "http", + }, + Value: model.SampleValue(43), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_panic_total", + "listener_name": "http", + "device": "eth0", + }, + Value: model.SampleValue(44), + Timestamp: model.Time(424242), + }, + } + events := g.GenerateEvents(metrics) + + + expected1 := common.MapStr{ + "net_conntrack_listener_conn_closed_total": common.MapStr{ + "counter": float64(42), + "rate": float64(0), + }, + "net_conntrack_listener_conn_panic_total": common.MapStr{ + "counter": float64(43), + "rate": float64(0), + }, + "labels": labels, + } + expected2 := common.MapStr{ + "net_conntrack_listener_conn_panic_total": common.MapStr{ + "counter": float64(44), + "rate": float64(0), + }, + "labels": labels2, + } + + assert.Equal(t, len(events), 2) + e := events[labels.String()] + assert.EqualValues(t, e.ModuleFields, expected1) + e = events[labels2.String()] + assert.EqualValues(t, e.ModuleFields, expected2) + + + // repeat in order to test the rate + metrics = model.Samples{ + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_closed_total", + "listener_name": "http", + }, + Value: model.SampleValue(45), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_panic_total", + "listener_name": "http", + }, + Value: model.SampleValue(47), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_panic_total", + "listener_name": "http", + "device": "eth0", + }, + Value: model.SampleValue(50), + Timestamp: model.Time(424242), + }, + } + events = g.GenerateEvents(metrics) + + expected1 = common.MapStr{ + "net_conntrack_listener_conn_closed_total": common.MapStr{ + "counter": float64(45), + "rate": float64(3), + }, + "net_conntrack_listener_conn_panic_total": common.MapStr{ + "counter": float64(47), + "rate": float64(4), + }, + "labels": labels, + } + expected2 = common.MapStr{ + "net_conntrack_listener_conn_panic_total": common.MapStr{ + "counter": float64(50), + "rate": float64(6), + }, + "labels": labels2, + } + + assert.Equal(t, len(events), 2) + e = events[labels.String()] + assert.EqualValues(t, e.ModuleFields, expected1) + e = events[labels2.String()] + assert.EqualValues(t, e.ModuleFields, expected2) + +} + +// TestGenerateEventsGaugeDifferentLabels tests multiple gauges with different labels +func TestGenerateEventsGaugeDifferentLabels(t *testing.T) { + + counters := xcollector.NewCounterCache(1 * time.Second) + + g := RemoteWriteTypedGenerator{ + CounterCache: counters, + RateCounters: true, + } + g.CounterCache.Start() + labels := common.MapStr{ + "listener_name": model.LabelValue("http"), + } + labels2 := common.MapStr{ + "listener_name": model.LabelValue("http"), + "device": model.LabelValue("eth0"), + } + + // first fetch + metrics := model.Samples{ + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_closed_total", + "listener_name": "http", + }, + Value: model.SampleValue(42), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_panic_total", + "listener_name": "http", + }, + Value: model.SampleValue(43), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_panic_total", + "listener_name": "http", + "device": "eth0", + }, + Value: model.SampleValue(44), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_open", + "listener_name": "http", + "device": "eth0", + }, + Value: model.SampleValue(49), + Timestamp: model.Time(424242), + }, + } + events := g.GenerateEvents(metrics) + + + expected1 := common.MapStr{ + "net_conntrack_listener_conn_closed_total": common.MapStr{ + "counter": float64(42), + "rate": float64(0), + }, + "net_conntrack_listener_conn_panic_total": common.MapStr{ + "counter": float64(43), + "rate": float64(0), + }, + "labels": labels, + } + expected2 := common.MapStr{ + "net_conntrack_listener_conn_panic_total": common.MapStr{ + "counter": float64(44), + "rate": float64(0), + }, + "net_conntrack_listener_conn_open": common.MapStr{ + "value": float64(49), + }, + "labels": labels2, + } + + assert.Equal(t, len(events), 2) + e := events[labels.String()] + assert.EqualValues(t, e.ModuleFields, expected1) + e = events[labels2.String()] + assert.EqualValues(t, e.ModuleFields, expected2) + + + // repeat in order to test the rate + metrics = model.Samples{ + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_closed_total", + "listener_name": "http", + }, + Value: model.SampleValue(45), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_panic_total", + "listener_name": "http", + }, + Value: model.SampleValue(47), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_panic_total", + "listener_name": "http", + "device": "eth0", + }, + Value: model.SampleValue(50), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_open", + "listener_name": "http", + "device": "eth0", + }, + Value: model.SampleValue(59), + Timestamp: model.Time(424242), + }, + } + events = g.GenerateEvents(metrics) + + expected1 = common.MapStr{ + "net_conntrack_listener_conn_closed_total": common.MapStr{ + "counter": float64(45), + "rate": float64(3), + }, + "net_conntrack_listener_conn_panic_total": common.MapStr{ + "counter": float64(47), + "rate": float64(4), + }, + "labels": labels, + } + expected2 = common.MapStr{ + "net_conntrack_listener_conn_panic_total": common.MapStr{ + "counter": float64(50), + "rate": float64(6), + }, + "net_conntrack_listener_conn_open": common.MapStr{ + "value": float64(59), + }, + "labels": labels2, + } + + assert.Equal(t, len(events), 2) + e = events[labels.String()] + assert.EqualValues(t, e.ModuleFields, expected1) + e = events[labels2.String()] + assert.EqualValues(t, e.ModuleFields, expected2) + +} + +// TestGenerateEventsQuantilesDifferentLabels tests multiple gauges with different labels +func TestGenerateEventsQuantilesDifferentLabels(t *testing.T) { + + counters := xcollector.NewCounterCache(1 * time.Second) + + g := RemoteWriteTypedGenerator{ + CounterCache: counters, + RateCounters: true, + } + g.CounterCache.Start() + labels := common.MapStr{ + "runtime": model.LabelValue("linux"), + "quantile": model.LabelValue("0.25"), + } + labels2 := common.MapStr{ + "runtime": model.LabelValue("linux"), + "quantile": model.LabelValue("0.50"), + } + labels3 := common.MapStr{ + "runtime": model.LabelValue("linux"), + } + + // first fetch + metrics := model.Samples{ + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "go_gc_duration_seconds", + "runtime": "linux", + "quantile": "0.25", + }, + Value: model.SampleValue(42), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "go_gc_duration_seconds", + "runtime": "linux", + "quantile": "0.50", + }, + Value: model.SampleValue(43), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "go_gc_duration_seconds_sum", + "runtime": "linux", + }, + Value: model.SampleValue(44), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "go_gc_duration_seconds_count", + "runtime": "linux", + }, + Value: model.SampleValue(45), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "go_gc_duration_seconds_2", + "runtime": "linux", + "quantile": "0.25", + }, + Value: model.SampleValue(46), + Timestamp: model.Time(424242), + }, + } + events := g.GenerateEvents(metrics) + + + expected := common.MapStr{ + "go_gc_duration_seconds": common.MapStr{ + "value": float64(42), + }, + "go_gc_duration_seconds_2": common.MapStr{ + "value": float64(46), + }, + "labels": labels, + } + expected2 := common.MapStr{ + "go_gc_duration_seconds": common.MapStr{ + "value": float64(43), + }, + "labels": labels2, + } + expected3 := common.MapStr{ + "go_gc_duration_seconds_count": common.MapStr{ + "counter": uint64(45), + "rate": uint64(0), + }, + "go_gc_duration_seconds_sum": common.MapStr{ + "counter": float64(44), + "rate": float64(0), + }, + "labels": labels3, + } + + assert.Equal(t, len(events), 3) + e := events[labels.String()] + assert.EqualValues(t, e.ModuleFields, expected) + e = events[labels2.String()] + assert.EqualValues(t, e.ModuleFields, expected2) + e = events[labels3.String()] + assert.EqualValues(t, e.ModuleFields, expected3) + + + // repeat in order to test the rate + metrics = model.Samples{ + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "go_gc_duration_seconds", + "runtime": "linux", + "quantile": "0.25", + }, + Value: model.SampleValue(52), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "go_gc_duration_seconds", + "runtime": "linux", + "quantile": "0.50", + }, + Value: model.SampleValue(53), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "go_gc_duration_seconds_sum", + "runtime": "linux", + }, + Value: model.SampleValue(54), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "go_gc_duration_seconds_count", + "runtime": "linux", + }, + Value: model.SampleValue(55), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "go_gc_duration_seconds_2", + "runtime": "linux", + "quantile": "0.25", + }, + Value: model.SampleValue(56), + Timestamp: model.Time(424242), + }, + } + events = g.GenerateEvents(metrics) + + + expected = common.MapStr{ + "go_gc_duration_seconds": common.MapStr{ + "value": float64(52), + }, + "go_gc_duration_seconds_2": common.MapStr{ + "value": float64(56), + }, + "labels": labels, + } + expected2 = common.MapStr{ + "go_gc_duration_seconds": common.MapStr{ + "value": float64(53), + }, + "labels": labels2, + } + expected3 = common.MapStr{ + "go_gc_duration_seconds_count": common.MapStr{ + "counter": uint64(55), + "rate": uint64(10), + }, + "go_gc_duration_seconds_sum": common.MapStr{ + "counter": float64(54), + "rate": float64(10), + }, + "labels": labels3, + } + + assert.Equal(t, len(events), 3) + e = events[labels.String()] + assert.EqualValues(t, e.ModuleFields, expected) + e = events[labels2.String()] + assert.EqualValues(t, e.ModuleFields, expected2) + e = events[labels3.String()] + assert.EqualValues(t, e.ModuleFields, expected3) + +} From 273cc8781ba3423ec6e0280abc09848621d3db31 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 17 Jul 2020 16:59:04 +0300 Subject: [PATCH 06/24] Add histogram unit tests Signed-off-by: chrismark --- .../module/prometheus/remote_write/data.go | 28 +- .../remote_write/remote_write_test.go | 405 +++++++++++++++++- 2 files changed, 418 insertions(+), 15 deletions(-) diff --git a/x-pack/metricbeat/module/prometheus/remote_write/data.go b/x-pack/metricbeat/module/prometheus/remote_write/data.go index 399956e9d27..6e1e588a580 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/data.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/data.go @@ -72,7 +72,7 @@ func (g *RemoteWriteTypedGenerator) Stop() { func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[string]mb.Event { var data common.MapStr - var histograms map[string]histogram + histograms := map[string]histogram{} eventList := map[string]mb.Event{} for _, metric := range metrics { @@ -93,6 +93,11 @@ func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str if !math.IsNaN(val) && !math.IsInf(val, 0) { // join metrics with same labels in a single event labelsHash := labels.String() + labelsClone := labels.Clone() + labelsClone.Delete("le") + if promType == "histogram" { + labelsHash = labelsClone.String() + } if _, ok := eventList[labelsHash]; !ok { eventList[labelsHash] = mb.Event{ ModuleFields: common.MapStr{}, @@ -101,11 +106,10 @@ func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str // Add labels if len(labels) > 0 { if promType == "histogram" { - labelsClone := labels.Clone() - labelsClone.Delete("le") - labels = labelsClone.Clone() + eventList[labelsHash].ModuleFields["labels"] = labelsClone + } else { + eventList[labelsHash].ModuleFields["labels"] = labels } - eventList[labelsHash].ModuleFields["labels"] = labels } } @@ -127,9 +131,6 @@ func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str }, } case "histogram": - labelsClone := labels.Clone() - labelsClone.Delete("le") - histKey := name + labelsClone.String() le, _ := labels.GetValue("le") @@ -152,6 +153,7 @@ func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str hist.timestamp = metric.Timestamp.Time() hist.labels = labelsClone hist.metricName = name + histograms[histKey] = hist continue } e.ModuleFields.Update(data) @@ -188,13 +190,11 @@ func (g *RemoteWriteTypedGenerator) rateCounterFloat64(name string, labels commo } func (g *RemoteWriteTypedGenerator) processPromHistograms(eventList map[string]mb.Event, histograms map[string]histogram) { - for name, histogram := range histograms { + for _, histogram := range histograms { labelsHash := histogram.labels.String() if _, ok := eventList[labelsHash]; !ok { eventList[labelsHash] = mb.Event{ - ModuleFields: common.MapStr{ - "metrics": common.MapStr{}, - }, + ModuleFields: common.MapStr{}, } // Add labels @@ -209,13 +209,13 @@ func (g *RemoteWriteTypedGenerator) processPromHistograms(eventList map[string]m hist := dto.Histogram{ Bucket: histogram.buckets, } - + name := strings.TrimSuffix(histogram.metricName, "_bucket") data := common.MapStr{ name: common.MapStr{ "histogram": xcollector.PromHistogramToES(g.CounterCache, histogram.metricName, histogram.labels, &hist), }, } - e.ModuleFields["metrics"].(common.MapStr).Update(data) + e.ModuleFields.Update(data) } } diff --git a/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go b/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go index e0bd350a7c4..aa9814de7aa 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go @@ -467,7 +467,7 @@ func TestGenerateEventsGaugeDifferentLabels(t *testing.T) { } -// TestGenerateEventsQuantilesDifferentLabels tests multiple gauges with different labels +// TestGenerateEventsQuantilesDifferentLabels tests summaries with different labels func TestGenerateEventsQuantilesDifferentLabels(t *testing.T) { counters := xcollector.NewCounterCache(1 * time.Second) @@ -659,3 +659,406 @@ func TestGenerateEventsQuantilesDifferentLabels(t *testing.T) { assert.EqualValues(t, e.ModuleFields, expected3) } + + +// TestGenerateEventsHistogramsDifferentLabels tests histograms with different labels +func TestGenerateEventsHistogramsDifferentLabels(t *testing.T) { + + counters := xcollector.NewCounterCache(1 * time.Second) + + g := RemoteWriteTypedGenerator{ + CounterCache: counters, + RateCounters: true, + } + g.CounterCache.Start() + labels := common.MapStr{ + "runtime": model.LabelValue("linux"), + } + labels2 := common.MapStr{ + "runtime": model.LabelValue("darwin"), + } + + // first fetch + metrics := model.Samples{ + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_duration_seconds_bucket", + "runtime": "linux", + "le": "0.25", + }, + Value: model.SampleValue(42), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_duration_seconds_bucket", + "runtime": "linux", + "le": "0.50", + }, + Value: model.SampleValue(43), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_duration_seconds_bucket", + "runtime": "linux", + "le": "+Inf", + }, + Value: model.SampleValue(44), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_duration_seconds_sum", + "runtime": "linux", + }, + Value: model.SampleValue(45), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_duration_seconds_count", + "runtime": "linux", + }, + Value: model.SampleValue(46), + Timestamp: model.Time(424242), + }, + // second histogram same label + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_bucket", + "runtime": "linux", + "le": "0.25", + }, + Value: model.SampleValue(52), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_bucket", + "runtime": "linux", + "le": "0.50", + }, + Value: model.SampleValue(53), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_bucket", + "runtime": "linux", + "le": "+Inf", + }, + Value: model.SampleValue(54), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_sum", + "runtime": "linux", + }, + Value: model.SampleValue(55), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_count", + "runtime": "linux", + }, + Value: model.SampleValue(56), + Timestamp: model.Time(424242), + }, + // third histogram different label + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_bucket", + "runtime": "darwin", + "le": "0.25", + }, + Value: model.SampleValue(62), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_bucket", + "runtime": "darwin", + "le": "0.50", + }, + Value: model.SampleValue(63), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_bucket", + "runtime": "darwin", + "le": "+Inf", + }, + Value: model.SampleValue(64), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_sum", + "runtime": "darwin", + }, + Value: model.SampleValue(65), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_count", + "runtime": "darwin", + }, + Value: model.SampleValue(66), + Timestamp: model.Time(424242), + }, + } + events := g.GenerateEvents(metrics) + + + expected := common.MapStr{ + "http_request_duration_seconds": common.MapStr{ + "histogram": common.MapStr{ + "values": []float64{float64(0.125),float64(0.375), float64(0.75)}, + "counts": []uint64{uint64(0), uint64(0), uint64(0)}, + }, + }, + "http_request_duration_seconds_sum": common.MapStr{ + "counter": float64(45), + "rate": float64(0), + }, + "http_request_duration_seconds_count": common.MapStr{ + "counter": uint64(46), + "rate": uint64(0), + }, + "http_request_bytes": common.MapStr{ + "histogram": common.MapStr{ + "values": []float64{float64(0.125),float64(0.375), float64(0.75)}, + "counts": []uint64{uint64(0), uint64(0), uint64(0)}, + }, + }, + "http_request_bytes_sum": common.MapStr{ + "counter": float64(55), + "rate": float64(0), + }, + "http_request_bytes_count": common.MapStr{ + "counter": uint64(56), + "rate": uint64(0), + }, + "labels": labels, + } + expected2 := common.MapStr{ + "http_request_bytes": common.MapStr{ + "histogram": common.MapStr{ + "values": []float64{float64(0.125),float64(0.375), float64(0.75)}, + "counts": []uint64{uint64(0), uint64(0), uint64(0)}, + }, + }, + "http_request_bytes_sum": common.MapStr{ + "counter": float64(65), + "rate": float64(0), + }, + "http_request_bytes_count": common.MapStr{ + "counter": uint64(66), + "rate": uint64(0), + }, + "labels": labels2, + } + + assert.Equal(t, 2, len(events)) + e := events[labels.String()] + assert.EqualValues(t, e.ModuleFields, expected) + e = events[labels2.String()] + assert.EqualValues(t, e.ModuleFields, expected2) + + // repeat in order to test the rate + metrics = model.Samples{ + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_duration_seconds_bucket", + "runtime": "linux", + "le": "0.25", + }, + Value: model.SampleValue(142), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_duration_seconds_bucket", + "runtime": "linux", + "le": "0.50", + }, + Value: model.SampleValue(143), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_duration_seconds_bucket", + "runtime": "linux", + "le": "+Inf", + }, + Value: model.SampleValue(144), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_duration_seconds_sum", + "runtime": "linux", + }, + Value: model.SampleValue(145), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_duration_seconds_count", + "runtime": "linux", + }, + Value: model.SampleValue(146), + Timestamp: model.Time(424242), + }, + // second histogram same label + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_bucket", + "runtime": "linux", + "le": "0.25", + }, + Value: model.SampleValue(252), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_bucket", + "runtime": "linux", + "le": "0.50", + }, + Value: model.SampleValue(253), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_bucket", + "runtime": "linux", + "le": "+Inf", + }, + Value: model.SampleValue(254), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_sum", + "runtime": "linux", + }, + Value: model.SampleValue(255), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_count", + "runtime": "linux", + }, + Value: model.SampleValue(256), + Timestamp: model.Time(424242), + }, + // third histogram different label + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_bucket", + "runtime": "darwin", + "le": "0.25", + }, + Value: model.SampleValue(362), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_bucket", + "runtime": "darwin", + "le": "0.50", + }, + Value: model.SampleValue(363), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_bucket", + "runtime": "darwin", + "le": "+Inf", + }, + Value: model.SampleValue(364), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_sum", + "runtime": "darwin", + }, + Value: model.SampleValue(365), + Timestamp: model.Time(424242), + }, + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "http_request_bytes_count", + "runtime": "darwin", + }, + Value: model.SampleValue(366), + Timestamp: model.Time(424242), + }, + } + events = g.GenerateEvents(metrics) + + + expected = common.MapStr{ + "http_request_duration_seconds": common.MapStr{ + "histogram": common.MapStr{ + "values": []float64{float64(0.125),float64(0.375), float64(0.75)}, + "counts": []uint64{uint64(100), uint64(100), uint64(100)}, + }, + }, + "http_request_duration_seconds_sum": common.MapStr{ + "counter": float64(145), + "rate": float64(100), + }, + "http_request_duration_seconds_count": common.MapStr{ + "counter": uint64(146), + "rate": uint64(100), + }, + "http_request_bytes": common.MapStr{ + "histogram": common.MapStr{ + "values": []float64{float64(0.125),float64(0.375), float64(0.75)}, + "counts": []uint64{uint64(200), uint64(200), uint64(200)}, + }, + }, + "http_request_bytes_sum": common.MapStr{ + "counter": float64(255), + "rate": float64(200), + }, + "http_request_bytes_count": common.MapStr{ + "counter": uint64(256), + "rate": uint64(200), + }, + "labels": labels, + } + expected2 = common.MapStr{ + "http_request_bytes": common.MapStr{ + "histogram": common.MapStr{ + "values": []float64{float64(0.125),float64(0.375), float64(0.75)}, + "counts": []uint64{uint64(300), uint64(300), uint64(300)}, + }, + }, + "http_request_bytes_sum": common.MapStr{ + "counter": float64(365), + "rate": float64(300), + }, + "http_request_bytes_count": common.MapStr{ + "counter": uint64(366), + "rate": uint64(300), + }, + "labels": labels2, + } + + assert.Equal(t, 2, len(events)) + e = events[labels.String()] + assert.EqualValues(t, e.ModuleFields, expected) + e = events[labels2.String()] + assert.EqualValues(t, e.ModuleFields, expected2) +} From 7d2cd4da67643ca5c4653fb2e8cd14f22c39658d Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 17 Jul 2020 17:07:33 +0300 Subject: [PATCH 07/24] fmt update Signed-off-by: chrismark --- metricbeat/docs/fields.asciidoc | 7 +- .../prometheus/remote_write/remote_write.go | 12 +- .../remote_write/remote_write_test.go | 403 +++++++++--------- 3 files changed, 198 insertions(+), 224 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 0e666085fe0..67f5d7b086d 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -35084,12 +35084,7 @@ type: object *`prometheus.*.histogram`*:: + -- -Prometheus histogram metric - release: ga - name: remote_write - type: group - description: > - remote write metrics from Prometheus server - release: beta - fields: +Prometheus histogram metric - release: ga return nil type: object diff --git a/metricbeat/module/prometheus/remote_write/remote_write.go b/metricbeat/module/prometheus/remote_write/remote_write.go index 09544ef0964..41a888fad46 100644 --- a/metricbeat/module/prometheus/remote_write/remote_write.go +++ b/metricbeat/module/prometheus/remote_write/remote_write.go @@ -18,9 +18,9 @@ package remote_write import ( + "fmt" "io/ioutil" "net/http" - "fmt" "github.com/gogo/protobuf/proto" "github.com/golang/snappy" @@ -98,7 +98,7 @@ func MetricSetBuilder(namespace string, genFactory RemoteWriteEventsGeneratorFac m := &MetricSet{ BaseMetricSet: base, - events: make(chan mb.Event), + events: make(chan mb.Event), promEventsGen: promEventsGen, eventGenStarted: false, } @@ -115,15 +115,12 @@ func MetricSetBuilder(namespace string, genFactory RemoteWriteEventsGeneratorFac func (m *MetricSet) Run(reporter mb.PushReporterV2) { // Start event watcher m.server.Start() - //fmt.Println("Run server") for { select { case <-reporter.Done(): m.server.Stop() return case e := <-m.events: - //fmt.Println("reporter") - //fmt.Println(e) reporter.Event(e) } } @@ -164,14 +161,11 @@ func (m *MetricSet) handleFunc(writer http.ResponseWriter, req *http.Request) { http.Error(writer, err.Error(), http.StatusBadRequest) return } - //fmt.Println("handleFunc 2") + samples := protoToSamples(&protoReq) - //fmt.Println("handleFunc 3") events := m.promEventsGen.GenerateEvents(samples) - //fmt.Println("handleFunc 4") for _, e := range events { - //fmt.Println(e) select { case <-req.Context().Done(): return diff --git a/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go b/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go index aa9814de7aa..b615d9ccc68 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go @@ -7,10 +7,11 @@ package remote_write import ( - "github.com/prometheus/common/model" "testing" "time" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/common" @@ -35,37 +36,35 @@ func TestGenerateEventsCounter(t *testing.T) { metrics := model.Samples{ &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_closed_total", + "__name__": "net_conntrack_listener_conn_closed_total", "listener_name": "http", }, - Value: model.SampleValue(42), + Value: model.SampleValue(42), Timestamp: model.Time(424242), }, } events := g.GenerateEvents(metrics) - expected := common.MapStr{ - "net_conntrack_listener_conn_closed_total": common.MapStr{ - "counter": float64(42), - "rate": float64(0), - }, - "labels": labels, + "net_conntrack_listener_conn_closed_total": common.MapStr{ + "counter": float64(42), + "rate": float64(0), + }, + "labels": labels, } assert.Equal(t, len(events), 1) e := events[labels.String()] assert.EqualValues(t, e.ModuleFields, expected) - // repeat in order to test the rate metrics = model.Samples{ &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_closed_total", + "__name__": "net_conntrack_listener_conn_closed_total", "listener_name": "http", }, - Value: model.SampleValue(45), + Value: model.SampleValue(45), Timestamp: model.Time(424242), }, } @@ -74,7 +73,7 @@ func TestGenerateEventsCounter(t *testing.T) { expected = common.MapStr{ "net_conntrack_listener_conn_closed_total": common.MapStr{ "counter": float64(45), - "rate": float64(3), + "rate": float64(3), }, "labels": labels, } @@ -85,7 +84,6 @@ func TestGenerateEventsCounter(t *testing.T) { } - // TestGenerateEventsCounterSameLabels tests multiple counters with same labels func TestGenerateEventsCounterSameLabels(t *testing.T) { @@ -104,32 +102,31 @@ func TestGenerateEventsCounterSameLabels(t *testing.T) { metrics := model.Samples{ &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_closed_total", + "__name__": "net_conntrack_listener_conn_closed_total", "listener_name": "http", }, - Value: model.SampleValue(42), + Value: model.SampleValue(42), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_panic_total", + "__name__": "net_conntrack_listener_conn_panic_total", "listener_name": "http", }, - Value: model.SampleValue(43), + Value: model.SampleValue(43), Timestamp: model.Time(424242), }, } events := g.GenerateEvents(metrics) - expected := common.MapStr{ "net_conntrack_listener_conn_closed_total": common.MapStr{ "counter": float64(42), - "rate": float64(0), + "rate": float64(0), }, "net_conntrack_listener_conn_panic_total": common.MapStr{ "counter": float64(43), - "rate": float64(0), + "rate": float64(0), }, "labels": labels, } @@ -138,23 +135,22 @@ func TestGenerateEventsCounterSameLabels(t *testing.T) { e := events[labels.String()] assert.EqualValues(t, e.ModuleFields, expected) - // repeat in order to test the rate metrics = model.Samples{ &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_closed_total", + "__name__": "net_conntrack_listener_conn_closed_total", "listener_name": "http", }, - Value: model.SampleValue(45), + Value: model.SampleValue(45), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_panic_total", + "__name__": "net_conntrack_listener_conn_panic_total", "listener_name": "http", }, - Value: model.SampleValue(47), + Value: model.SampleValue(47), Timestamp: model.Time(424242), }, } @@ -163,11 +159,11 @@ func TestGenerateEventsCounterSameLabels(t *testing.T) { expected = common.MapStr{ "net_conntrack_listener_conn_closed_total": common.MapStr{ "counter": float64(45), - "rate": float64(3), + "rate": float64(3), }, "net_conntrack_listener_conn_panic_total": common.MapStr{ "counter": float64(47), - "rate": float64(4), + "rate": float64(4), }, "labels": labels, } @@ -178,7 +174,6 @@ func TestGenerateEventsCounterSameLabels(t *testing.T) { } - // TestGenerateEventsCounterDifferentLabels tests multiple counters with different labels func TestGenerateEventsCounterDifferentLabels(t *testing.T) { @@ -194,55 +189,54 @@ func TestGenerateEventsCounterDifferentLabels(t *testing.T) { } labels2 := common.MapStr{ "listener_name": model.LabelValue("http"), - "device": model.LabelValue("eth0"), + "device": model.LabelValue("eth0"), } // first fetch metrics := model.Samples{ &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_closed_total", + "__name__": "net_conntrack_listener_conn_closed_total", "listener_name": "http", }, - Value: model.SampleValue(42), + Value: model.SampleValue(42), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_panic_total", + "__name__": "net_conntrack_listener_conn_panic_total", "listener_name": "http", }, - Value: model.SampleValue(43), + Value: model.SampleValue(43), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_panic_total", + "__name__": "net_conntrack_listener_conn_panic_total", "listener_name": "http", - "device": "eth0", + "device": "eth0", }, - Value: model.SampleValue(44), + Value: model.SampleValue(44), Timestamp: model.Time(424242), }, } events := g.GenerateEvents(metrics) - expected1 := common.MapStr{ "net_conntrack_listener_conn_closed_total": common.MapStr{ "counter": float64(42), - "rate": float64(0), + "rate": float64(0), }, "net_conntrack_listener_conn_panic_total": common.MapStr{ "counter": float64(43), - "rate": float64(0), + "rate": float64(0), }, "labels": labels, } expected2 := common.MapStr{ "net_conntrack_listener_conn_panic_total": common.MapStr{ "counter": float64(44), - "rate": float64(0), + "rate": float64(0), }, "labels": labels2, } @@ -253,32 +247,31 @@ func TestGenerateEventsCounterDifferentLabels(t *testing.T) { e = events[labels2.String()] assert.EqualValues(t, e.ModuleFields, expected2) - // repeat in order to test the rate metrics = model.Samples{ &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_closed_total", + "__name__": "net_conntrack_listener_conn_closed_total", "listener_name": "http", }, - Value: model.SampleValue(45), + Value: model.SampleValue(45), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_panic_total", + "__name__": "net_conntrack_listener_conn_panic_total", "listener_name": "http", }, - Value: model.SampleValue(47), + Value: model.SampleValue(47), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_panic_total", + "__name__": "net_conntrack_listener_conn_panic_total", "listener_name": "http", - "device": "eth0", + "device": "eth0", }, - Value: model.SampleValue(50), + Value: model.SampleValue(50), Timestamp: model.Time(424242), }, } @@ -287,18 +280,18 @@ func TestGenerateEventsCounterDifferentLabels(t *testing.T) { expected1 = common.MapStr{ "net_conntrack_listener_conn_closed_total": common.MapStr{ "counter": float64(45), - "rate": float64(3), + "rate": float64(3), }, "net_conntrack_listener_conn_panic_total": common.MapStr{ "counter": float64(47), - "rate": float64(4), + "rate": float64(4), }, "labels": labels, } expected2 = common.MapStr{ "net_conntrack_listener_conn_panic_total": common.MapStr{ "counter": float64(50), - "rate": float64(6), + "rate": float64(6), }, "labels": labels2, } @@ -326,64 +319,63 @@ func TestGenerateEventsGaugeDifferentLabels(t *testing.T) { } labels2 := common.MapStr{ "listener_name": model.LabelValue("http"), - "device": model.LabelValue("eth0"), + "device": model.LabelValue("eth0"), } // first fetch metrics := model.Samples{ &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_closed_total", + "__name__": "net_conntrack_listener_conn_closed_total", "listener_name": "http", }, - Value: model.SampleValue(42), + Value: model.SampleValue(42), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_panic_total", + "__name__": "net_conntrack_listener_conn_panic_total", "listener_name": "http", }, - Value: model.SampleValue(43), + Value: model.SampleValue(43), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_panic_total", + "__name__": "net_conntrack_listener_conn_panic_total", "listener_name": "http", - "device": "eth0", + "device": "eth0", }, - Value: model.SampleValue(44), + Value: model.SampleValue(44), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_open", + "__name__": "net_conntrack_listener_conn_open", "listener_name": "http", - "device": "eth0", + "device": "eth0", }, - Value: model.SampleValue(49), + Value: model.SampleValue(49), Timestamp: model.Time(424242), }, } events := g.GenerateEvents(metrics) - expected1 := common.MapStr{ "net_conntrack_listener_conn_closed_total": common.MapStr{ "counter": float64(42), - "rate": float64(0), + "rate": float64(0), }, "net_conntrack_listener_conn_panic_total": common.MapStr{ "counter": float64(43), - "rate": float64(0), + "rate": float64(0), }, "labels": labels, } expected2 := common.MapStr{ "net_conntrack_listener_conn_panic_total": common.MapStr{ "counter": float64(44), - "rate": float64(0), + "rate": float64(0), }, "net_conntrack_listener_conn_open": common.MapStr{ "value": float64(49), @@ -397,41 +389,40 @@ func TestGenerateEventsGaugeDifferentLabels(t *testing.T) { e = events[labels2.String()] assert.EqualValues(t, e.ModuleFields, expected2) - // repeat in order to test the rate metrics = model.Samples{ &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_closed_total", + "__name__": "net_conntrack_listener_conn_closed_total", "listener_name": "http", }, - Value: model.SampleValue(45), + Value: model.SampleValue(45), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_panic_total", + "__name__": "net_conntrack_listener_conn_panic_total", "listener_name": "http", }, - Value: model.SampleValue(47), + Value: model.SampleValue(47), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_panic_total", + "__name__": "net_conntrack_listener_conn_panic_total", "listener_name": "http", - "device": "eth0", + "device": "eth0", }, - Value: model.SampleValue(50), + Value: model.SampleValue(50), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ - "__name__": "net_conntrack_listener_conn_open", + "__name__": "net_conntrack_listener_conn_open", "listener_name": "http", - "device": "eth0", + "device": "eth0", }, - Value: model.SampleValue(59), + Value: model.SampleValue(59), Timestamp: model.Time(424242), }, } @@ -440,18 +431,18 @@ func TestGenerateEventsGaugeDifferentLabels(t *testing.T) { expected1 = common.MapStr{ "net_conntrack_listener_conn_closed_total": common.MapStr{ "counter": float64(45), - "rate": float64(3), + "rate": float64(3), }, "net_conntrack_listener_conn_panic_total": common.MapStr{ "counter": float64(47), - "rate": float64(4), + "rate": float64(4), }, "labels": labels, } expected2 = common.MapStr{ "net_conntrack_listener_conn_panic_total": common.MapStr{ "counter": float64(50), - "rate": float64(6), + "rate": float64(6), }, "net_conntrack_listener_conn_open": common.MapStr{ "value": float64(59), @@ -478,11 +469,11 @@ func TestGenerateEventsQuantilesDifferentLabels(t *testing.T) { } g.CounterCache.Start() labels := common.MapStr{ - "runtime": model.LabelValue("linux"), + "runtime": model.LabelValue("linux"), "quantile": model.LabelValue("0.25"), } labels2 := common.MapStr{ - "runtime": model.LabelValue("linux"), + "runtime": model.LabelValue("linux"), "quantile": model.LabelValue("0.50"), } labels3 := common.MapStr{ @@ -494,50 +485,49 @@ func TestGenerateEventsQuantilesDifferentLabels(t *testing.T) { &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "go_gc_duration_seconds", - "runtime": "linux", + "runtime": "linux", "quantile": "0.25", }, - Value: model.SampleValue(42), + Value: model.SampleValue(42), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "go_gc_duration_seconds", - "runtime": "linux", + "runtime": "linux", "quantile": "0.50", }, - Value: model.SampleValue(43), + Value: model.SampleValue(43), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "go_gc_duration_seconds_sum", - "runtime": "linux", + "runtime": "linux", }, - Value: model.SampleValue(44), + Value: model.SampleValue(44), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "go_gc_duration_seconds_count", - "runtime": "linux", + "runtime": "linux", }, - Value: model.SampleValue(45), + Value: model.SampleValue(45), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "go_gc_duration_seconds_2", - "runtime": "linux", + "runtime": "linux", "quantile": "0.25", }, - Value: model.SampleValue(46), + Value: model.SampleValue(46), Timestamp: model.Time(424242), }, } events := g.GenerateEvents(metrics) - expected := common.MapStr{ "go_gc_duration_seconds": common.MapStr{ "value": float64(42), @@ -556,11 +546,11 @@ func TestGenerateEventsQuantilesDifferentLabels(t *testing.T) { expected3 := common.MapStr{ "go_gc_duration_seconds_count": common.MapStr{ "counter": uint64(45), - "rate": uint64(0), + "rate": uint64(0), }, "go_gc_duration_seconds_sum": common.MapStr{ "counter": float64(44), - "rate": float64(0), + "rate": float64(0), }, "labels": labels3, } @@ -573,56 +563,54 @@ func TestGenerateEventsQuantilesDifferentLabels(t *testing.T) { e = events[labels3.String()] assert.EqualValues(t, e.ModuleFields, expected3) - // repeat in order to test the rate metrics = model.Samples{ &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "go_gc_duration_seconds", - "runtime": "linux", + "runtime": "linux", "quantile": "0.25", }, - Value: model.SampleValue(52), + Value: model.SampleValue(52), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "go_gc_duration_seconds", - "runtime": "linux", + "runtime": "linux", "quantile": "0.50", }, - Value: model.SampleValue(53), + Value: model.SampleValue(53), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "go_gc_duration_seconds_sum", - "runtime": "linux", + "runtime": "linux", }, - Value: model.SampleValue(54), + Value: model.SampleValue(54), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "go_gc_duration_seconds_count", - "runtime": "linux", + "runtime": "linux", }, - Value: model.SampleValue(55), + Value: model.SampleValue(55), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "go_gc_duration_seconds_2", - "runtime": "linux", + "runtime": "linux", "quantile": "0.25", }, - Value: model.SampleValue(56), + Value: model.SampleValue(56), Timestamp: model.Time(424242), }, } events = g.GenerateEvents(metrics) - expected = common.MapStr{ "go_gc_duration_seconds": common.MapStr{ "value": float64(52), @@ -641,11 +629,11 @@ func TestGenerateEventsQuantilesDifferentLabels(t *testing.T) { expected3 = common.MapStr{ "go_gc_duration_seconds_count": common.MapStr{ "counter": uint64(55), - "rate": uint64(10), + "rate": uint64(10), }, "go_gc_duration_seconds_sum": common.MapStr{ "counter": float64(54), - "rate": float64(10), + "rate": float64(10), }, "labels": labels3, } @@ -660,7 +648,6 @@ func TestGenerateEventsQuantilesDifferentLabels(t *testing.T) { } - // TestGenerateEventsHistogramsDifferentLabels tests histograms with different labels func TestGenerateEventsHistogramsDifferentLabels(t *testing.T) { @@ -683,183 +670,182 @@ func TestGenerateEventsHistogramsDifferentLabels(t *testing.T) { &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_duration_seconds_bucket", - "runtime": "linux", - "le": "0.25", + "runtime": "linux", + "le": "0.25", }, - Value: model.SampleValue(42), + Value: model.SampleValue(42), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_duration_seconds_bucket", - "runtime": "linux", - "le": "0.50", + "runtime": "linux", + "le": "0.50", }, - Value: model.SampleValue(43), + Value: model.SampleValue(43), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_duration_seconds_bucket", - "runtime": "linux", - "le": "+Inf", + "runtime": "linux", + "le": "+Inf", }, - Value: model.SampleValue(44), + Value: model.SampleValue(44), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_duration_seconds_sum", - "runtime": "linux", + "runtime": "linux", }, - Value: model.SampleValue(45), + Value: model.SampleValue(45), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_duration_seconds_count", - "runtime": "linux", + "runtime": "linux", }, - Value: model.SampleValue(46), + Value: model.SampleValue(46), Timestamp: model.Time(424242), }, // second histogram same label &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_bucket", - "runtime": "linux", - "le": "0.25", + "runtime": "linux", + "le": "0.25", }, - Value: model.SampleValue(52), + Value: model.SampleValue(52), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_bucket", - "runtime": "linux", - "le": "0.50", + "runtime": "linux", + "le": "0.50", }, - Value: model.SampleValue(53), + Value: model.SampleValue(53), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_bucket", - "runtime": "linux", - "le": "+Inf", + "runtime": "linux", + "le": "+Inf", }, - Value: model.SampleValue(54), + Value: model.SampleValue(54), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_sum", - "runtime": "linux", + "runtime": "linux", }, - Value: model.SampleValue(55), + Value: model.SampleValue(55), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_count", - "runtime": "linux", + "runtime": "linux", }, - Value: model.SampleValue(56), + Value: model.SampleValue(56), Timestamp: model.Time(424242), }, // third histogram different label &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_bucket", - "runtime": "darwin", - "le": "0.25", + "runtime": "darwin", + "le": "0.25", }, - Value: model.SampleValue(62), + Value: model.SampleValue(62), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_bucket", - "runtime": "darwin", - "le": "0.50", + "runtime": "darwin", + "le": "0.50", }, - Value: model.SampleValue(63), + Value: model.SampleValue(63), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_bucket", - "runtime": "darwin", - "le": "+Inf", + "runtime": "darwin", + "le": "+Inf", }, - Value: model.SampleValue(64), + Value: model.SampleValue(64), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_sum", - "runtime": "darwin", + "runtime": "darwin", }, - Value: model.SampleValue(65), + Value: model.SampleValue(65), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_count", - "runtime": "darwin", + "runtime": "darwin", }, - Value: model.SampleValue(66), + Value: model.SampleValue(66), Timestamp: model.Time(424242), }, } events := g.GenerateEvents(metrics) - expected := common.MapStr{ "http_request_duration_seconds": common.MapStr{ "histogram": common.MapStr{ - "values": []float64{float64(0.125),float64(0.375), float64(0.75)}, + "values": []float64{float64(0.125), float64(0.375), float64(0.75)}, "counts": []uint64{uint64(0), uint64(0), uint64(0)}, }, }, "http_request_duration_seconds_sum": common.MapStr{ "counter": float64(45), - "rate": float64(0), + "rate": float64(0), }, "http_request_duration_seconds_count": common.MapStr{ "counter": uint64(46), - "rate": uint64(0), + "rate": uint64(0), }, "http_request_bytes": common.MapStr{ "histogram": common.MapStr{ - "values": []float64{float64(0.125),float64(0.375), float64(0.75)}, + "values": []float64{float64(0.125), float64(0.375), float64(0.75)}, "counts": []uint64{uint64(0), uint64(0), uint64(0)}, }, }, "http_request_bytes_sum": common.MapStr{ "counter": float64(55), - "rate": float64(0), + "rate": float64(0), }, "http_request_bytes_count": common.MapStr{ "counter": uint64(56), - "rate": uint64(0), + "rate": uint64(0), }, "labels": labels, } expected2 := common.MapStr{ "http_request_bytes": common.MapStr{ "histogram": common.MapStr{ - "values": []float64{float64(0.125),float64(0.375), float64(0.75)}, + "values": []float64{float64(0.125), float64(0.375), float64(0.75)}, "counts": []uint64{uint64(0), uint64(0), uint64(0)}, }, }, "http_request_bytes_sum": common.MapStr{ "counter": float64(65), - "rate": float64(0), + "rate": float64(0), }, "http_request_bytes_count": common.MapStr{ "counter": uint64(66), - "rate": uint64(0), + "rate": uint64(0), }, "labels": labels2, } @@ -875,183 +861,182 @@ func TestGenerateEventsHistogramsDifferentLabels(t *testing.T) { &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_duration_seconds_bucket", - "runtime": "linux", - "le": "0.25", + "runtime": "linux", + "le": "0.25", }, - Value: model.SampleValue(142), + Value: model.SampleValue(142), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_duration_seconds_bucket", - "runtime": "linux", - "le": "0.50", + "runtime": "linux", + "le": "0.50", }, - Value: model.SampleValue(143), + Value: model.SampleValue(143), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_duration_seconds_bucket", - "runtime": "linux", - "le": "+Inf", + "runtime": "linux", + "le": "+Inf", }, - Value: model.SampleValue(144), + Value: model.SampleValue(144), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_duration_seconds_sum", - "runtime": "linux", + "runtime": "linux", }, - Value: model.SampleValue(145), + Value: model.SampleValue(145), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_duration_seconds_count", - "runtime": "linux", + "runtime": "linux", }, - Value: model.SampleValue(146), + Value: model.SampleValue(146), Timestamp: model.Time(424242), }, // second histogram same label &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_bucket", - "runtime": "linux", - "le": "0.25", + "runtime": "linux", + "le": "0.25", }, - Value: model.SampleValue(252), + Value: model.SampleValue(252), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_bucket", - "runtime": "linux", - "le": "0.50", + "runtime": "linux", + "le": "0.50", }, - Value: model.SampleValue(253), + Value: model.SampleValue(253), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_bucket", - "runtime": "linux", - "le": "+Inf", + "runtime": "linux", + "le": "+Inf", }, - Value: model.SampleValue(254), + Value: model.SampleValue(254), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_sum", - "runtime": "linux", + "runtime": "linux", }, - Value: model.SampleValue(255), + Value: model.SampleValue(255), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_count", - "runtime": "linux", + "runtime": "linux", }, - Value: model.SampleValue(256), + Value: model.SampleValue(256), Timestamp: model.Time(424242), }, // third histogram different label &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_bucket", - "runtime": "darwin", - "le": "0.25", + "runtime": "darwin", + "le": "0.25", }, - Value: model.SampleValue(362), + Value: model.SampleValue(362), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_bucket", - "runtime": "darwin", - "le": "0.50", + "runtime": "darwin", + "le": "0.50", }, - Value: model.SampleValue(363), + Value: model.SampleValue(363), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_bucket", - "runtime": "darwin", - "le": "+Inf", + "runtime": "darwin", + "le": "+Inf", }, - Value: model.SampleValue(364), + Value: model.SampleValue(364), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_sum", - "runtime": "darwin", + "runtime": "darwin", }, - Value: model.SampleValue(365), + Value: model.SampleValue(365), Timestamp: model.Time(424242), }, &model.Sample{ Metric: map[model.LabelName]model.LabelValue{ "__name__": "http_request_bytes_count", - "runtime": "darwin", + "runtime": "darwin", }, - Value: model.SampleValue(366), + Value: model.SampleValue(366), Timestamp: model.Time(424242), }, } events = g.GenerateEvents(metrics) - expected = common.MapStr{ "http_request_duration_seconds": common.MapStr{ "histogram": common.MapStr{ - "values": []float64{float64(0.125),float64(0.375), float64(0.75)}, + "values": []float64{float64(0.125), float64(0.375), float64(0.75)}, "counts": []uint64{uint64(100), uint64(100), uint64(100)}, }, }, "http_request_duration_seconds_sum": common.MapStr{ "counter": float64(145), - "rate": float64(100), + "rate": float64(100), }, "http_request_duration_seconds_count": common.MapStr{ "counter": uint64(146), - "rate": uint64(100), + "rate": uint64(100), }, "http_request_bytes": common.MapStr{ "histogram": common.MapStr{ - "values": []float64{float64(0.125),float64(0.375), float64(0.75)}, + "values": []float64{float64(0.125), float64(0.375), float64(0.75)}, "counts": []uint64{uint64(200), uint64(200), uint64(200)}, }, }, "http_request_bytes_sum": common.MapStr{ "counter": float64(255), - "rate": float64(200), + "rate": float64(200), }, "http_request_bytes_count": common.MapStr{ "counter": uint64(256), - "rate": uint64(200), + "rate": uint64(200), }, "labels": labels, } expected2 = common.MapStr{ "http_request_bytes": common.MapStr{ "histogram": common.MapStr{ - "values": []float64{float64(0.125),float64(0.375), float64(0.75)}, + "values": []float64{float64(0.125), float64(0.375), float64(0.75)}, "counts": []uint64{uint64(300), uint64(300), uint64(300)}, }, }, "http_request_bytes_sum": common.MapStr{ "counter": float64(365), - "rate": float64(300), + "rate": float64(300), }, "http_request_bytes_count": common.MapStr{ "counter": uint64(366), - "rate": uint64(300), + "rate": uint64(300), }, "labels": labels2, } From 69803fd2a944f5f777c8f1ad013fadcc665693d0 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 17 Jul 2020 17:08:30 +0300 Subject: [PATCH 08/24] Remove print Signed-off-by: chrismark --- metricbeat/module/prometheus/remote_write/remote_write.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/metricbeat/module/prometheus/remote_write/remote_write.go b/metricbeat/module/prometheus/remote_write/remote_write.go index 41a888fad46..3fa7e8ebea7 100644 --- a/metricbeat/module/prometheus/remote_write/remote_write.go +++ b/metricbeat/module/prometheus/remote_write/remote_write.go @@ -18,7 +18,6 @@ package remote_write import ( - "fmt" "io/ioutil" "net/http" @@ -135,7 +134,6 @@ func (m *MetricSet) Close() error { } func (m *MetricSet) handleFunc(writer http.ResponseWriter, req *http.Request) { - fmt.Println("handleFunc 1") if !m.eventGenStarted { m.promEventsGen.Start() m.eventGenStarted = true From 28c951217ca45d64b70e9d1d6ab16a6d00d9cca2 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 17 Jul 2020 17:17:59 +0300 Subject: [PATCH 09/24] Add docstrings Signed-off-by: chrismark --- x-pack/metricbeat/module/prometheus/remote_write/data.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/x-pack/metricbeat/module/prometheus/remote_write/data.go b/x-pack/metricbeat/module/prometheus/remote_write/data.go index 6e1e588a580..3b3befc5e6a 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/data.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/data.go @@ -70,6 +70,12 @@ func (g *RemoteWriteTypedGenerator) Stop() { g.CounterCache.Stop() } + +// GenerateEvents receives a list of Sample and: +// 1. guess the type of the sample metric +// 2. handle it properly using "types" logic +// 3. if metrics of histogram type then it is converted to ES histogram +// 4. metrics with the same set of labels are grouped into same events func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[string]mb.Event { var data common.MapStr histograms := map[string]histogram{} @@ -160,6 +166,7 @@ func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str } } + // process histograms together g.processPromHistograms(eventList, histograms) return eventList } @@ -189,6 +196,7 @@ func (g *RemoteWriteTypedGenerator) rateCounterFloat64(name string, labels commo return d } +// processPromHistograms receives a group of Histograms and converts each one to ES histogram func (g *RemoteWriteTypedGenerator) processPromHistograms(eventList map[string]mb.Event, histograms map[string]histogram) { for _, histogram := range histograms { labelsHash := histogram.labels.String() @@ -219,6 +227,7 @@ func (g *RemoteWriteTypedGenerator) processPromHistograms(eventList map[string]m } } +// findType evaluates the type of the metric by check the metricname format in order to handle it properly func findType(metricName string, labels common.MapStr) string { leLabel := false if _, ok := labels["le"]; ok { From 76e62c73b49d7e8a6f11e919110a59f8a2c5eea4 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 17 Jul 2020 17:26:13 +0300 Subject: [PATCH 10/24] Remove blank line deletion Signed-off-by: chrismark --- metricbeat/module/prometheus/remote_write/remote_write.go | 1 + 1 file changed, 1 insertion(+) diff --git a/metricbeat/module/prometheus/remote_write/remote_write.go b/metricbeat/module/prometheus/remote_write/remote_write.go index 3fa7e8ebea7..0652e070e68 100644 --- a/metricbeat/module/prometheus/remote_write/remote_write.go +++ b/metricbeat/module/prometheus/remote_write/remote_write.go @@ -114,6 +114,7 @@ func MetricSetBuilder(namespace string, genFactory RemoteWriteEventsGeneratorFac func (m *MetricSet) Run(reporter mb.PushReporterV2) { // Start event watcher m.server.Start() + for { select { case <-reporter.Done(): From f9895f7642054796dabe739887d23e67452a6098 Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 20 Jul 2020 10:19:17 +0300 Subject: [PATCH 11/24] fmt x-pack code Signed-off-by: chrismark --- x-pack/metricbeat/module/prometheus/remote_write/data.go | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/metricbeat/module/prometheus/remote_write/data.go b/x-pack/metricbeat/module/prometheus/remote_write/data.go index 3b3befc5e6a..a5f1eac6993 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/data.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/data.go @@ -70,7 +70,6 @@ func (g *RemoteWriteTypedGenerator) Stop() { g.CounterCache.Stop() } - // GenerateEvents receives a list of Sample and: // 1. guess the type of the sample metric // 2. handle it properly using "types" logic From 6ce59872d5a0422d87b4c8e599d2922763d01bfd Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 20 Jul 2020 10:59:50 +0300 Subject: [PATCH 12/24] Fix leftover Signed-off-by: chrismark --- metricbeat/docs/fields.asciidoc | 2 +- x-pack/metricbeat/module/prometheus/fields.go | 2 +- .../metricbeat/module/prometheus/remote_write/_meta/fields.yml | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 67f5d7b086d..2c9c7c00124 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -35084,7 +35084,7 @@ type: object *`prometheus.*.histogram`*:: + -- -Prometheus histogram metric - release: ga return nil +Prometheus histogram metric - release: ga type: object diff --git a/x-pack/metricbeat/module/prometheus/fields.go b/x-pack/metricbeat/module/prometheus/fields.go index bf8cc4c176e..63330de9110 100644 --- a/x-pack/metricbeat/module/prometheus/fields.go +++ b/x-pack/metricbeat/module/prometheus/fields.go @@ -19,5 +19,5 @@ func init() { // AssetPrometheus returns asset data. // This is the base64 encoded gzipped contents of module/prometheus. func AssetPrometheus() string { - return "eJzEkstqMzEMhfd+ioOX4U8ewIv/GQpdlhKUGcVx4xuWXJq3L8kMYXqjWRSinX0O0qeD1jjyyaG2klgP3GX9Vmk4GkCDRnawD1cJeqo8IrG2MIg1wMgytFA1lOzw3wDAo5IKZGh09u5bSSAsenAeawlZNwZoHJmEHTwZQFg1ZC8OT1Yk2n+wB9Vqnw2wDxxHcZcJa2RKvGTerDavFDtfZFwwHcruhQedv6bHdlLG0neRvyrbRLWG7GebXdnZ882a51ps5al7npP5GXIoPSu3+2HOAL+CNtI7hnmePt7MegiixTdKNwJ/9v8N87Xrkndi/njlUzXW3jJyiOY9AAD//85gGik=" + return "eJzEkktqAzEMhvdzih8vQ5MDeNEzFLosJSi24rjxC0tTmtuXZIYw9EGzKMQ7+/+QPgmvceSTRes1sx54lPVHI3ccAI2a2MI8XSPoqbFHZu3RiRkAz+J6bBprsXgcAOBZSQXiOp3Zfa8ZhEUNLr7VWHQzAJ0Tk7BFoAEQVo0liMWLEUnmAeag2szrAOwjJy/20mGNQpmXzpvV5p3SyJcYF02Luntjp/PTdNlOia/jLvH3ZJuptVjCjJmVmZkfxjyfxVSBxsDzZn6XdHUsyv1+mrPAn6Kd9I7LPHf3N7seomgNnfKNwl/5/3G+Vl36Ts7LX/4ZAAD//xMCFRw=" } diff --git a/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml b/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml index 15e53c4a8eb..e69de29bb2d 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml +++ b/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml @@ -1 +0,0 @@ -return nil From 46789fc0e2fb797ed85160c1bb585924de2b4e3d Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 20 Jul 2020 12:50:44 +0300 Subject: [PATCH 13/24] first part review comments Signed-off-by: chrismark --- .../prometheus/remote_write/remote_write.go | 17 +++++++-- .../module/prometheus/collector/data.go | 14 +++---- .../module/prometheus/remote_write/data.go | 21 ++++------ .../prometheus/remote_write/remote_write.go | 2 +- .../remote_write/remote_write_test.go | 38 +++++++++---------- 5 files changed, 48 insertions(+), 44 deletions(-) diff --git a/metricbeat/module/prometheus/remote_write/remote_write.go b/metricbeat/module/prometheus/remote_write/remote_write.go index 0652e070e68..9c4d3657723 100644 --- a/metricbeat/module/prometheus/remote_write/remote_write.go +++ b/metricbeat/module/prometheus/remote_write/remote_write.go @@ -34,7 +34,7 @@ import ( func init() { mb.Registry.MustAddMetricSet("prometheus", "remote_write", - MetricSetBuilder("prometheus", DefaultRemoteWriteEventsGeneratorFactory), + MetricSetBuilder(DefaultRemoteWriteEventsGeneratorFactory), mb.WithHostParser(parse.EmptyHostParser), ) } @@ -68,10 +68,19 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { if err != nil { return nil, err } + + promEventsGen, err := DefaultRemoteWriteEventsGeneratorFactory(base) + if err != nil { + return nil, err + } + m := &MetricSet{ - BaseMetricSet: base, - events: make(chan mb.Event), + BaseMetricSet: base, + events: make(chan mb.Event), + promEventsGen: promEventsGen, + eventGenStarted: false, } + svc, err := httpserver.NewHttpServerWithHandler(base, m.handleFunc) if err != nil { return nil, err @@ -82,7 +91,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // MetricSetBuilder returns a builder function for a new Prometheus metricset using // the given namespace and event generator -func MetricSetBuilder(namespace string, genFactory RemoteWriteEventsGeneratorFactory) func(base mb.BaseMetricSet) (mb.MetricSet, error) { +func MetricSetBuilder(genFactory RemoteWriteEventsGeneratorFactory) func(base mb.BaseMetricSet) (mb.MetricSet, error) { return func(base mb.BaseMetricSet) (mb.MetricSet, error) { config := defaultConfig() err := base.Module().UnpackConfig(&config) diff --git a/x-pack/metricbeat/module/prometheus/collector/data.go b/x-pack/metricbeat/module/prometheus/collector/data.go index f0a367d5115..855c2616b0e 100644 --- a/x-pack/metricbeat/module/prometheus/collector/data.go +++ b/x-pack/metricbeat/module/prometheus/collector/data.go @@ -28,7 +28,7 @@ func promEventsGeneratorFactory(base mb.BaseMetricSet) (collector.PromEventsGene // to make sure that all counters are available between fetches counters := NewCounterCache(base.Module().Config().Period * 5) - g := TypedGenerator{ + g := typedGenerator{ CounterCache: counters, RateCounters: config.RateCounters, } @@ -39,12 +39,12 @@ func promEventsGeneratorFactory(base mb.BaseMetricSet) (collector.PromEventsGene return collector.DefaultPromEventsGeneratorFactory(base) } -type TypedGenerator struct { +type typedGenerator struct { CounterCache CounterCache RateCounters bool } -func (g *TypedGenerator) Start() { +func (g *typedGenerator) Start() { cfgwarn.Beta("Prometheus 'use_types' setting is beta") if g.RateCounters { @@ -54,14 +54,14 @@ func (g *TypedGenerator) Start() { g.CounterCache.Start() } -func (g *TypedGenerator) Stop() { +func (g *typedGenerator) Stop() { logp.Debug("prometheus.collector.cache", "stopping CounterCache") g.CounterCache.Stop() } // GeneratePromEvents stores all Prometheus metrics using // specific Elasticsearch data types. -func (g *TypedGenerator) GeneratePromEvents(mf *dto.MetricFamily) []collector.PromEvent { +func (g *typedGenerator) GeneratePromEvents(mf *dto.MetricFamily) []collector.PromEvent { var events []collector.PromEvent name := *mf.Name @@ -167,7 +167,7 @@ func (g *TypedGenerator) GeneratePromEvents(mf *dto.MetricFamily) []collector.Pr } // rateCounterUint64 fills a counter value and optionally adds the rate if rate_counters is enabled -func (g *TypedGenerator) rateCounterUint64(name string, labels common.MapStr, value uint64) common.MapStr { +func (g *typedGenerator) rateCounterUint64(name string, labels common.MapStr, value uint64) common.MapStr { d := common.MapStr{ "counter": value, } @@ -180,7 +180,7 @@ func (g *TypedGenerator) rateCounterUint64(name string, labels common.MapStr, va } // rateCounterFloat64 fills a counter value and optionally adds the rate if rate_counters is enabled -func (g *TypedGenerator) rateCounterFloat64(name string, labels common.MapStr, value float64) common.MapStr { +func (g *typedGenerator) rateCounterFloat64(name string, labels common.MapStr, value float64) common.MapStr { d := common.MapStr{ "counter": value, } diff --git a/x-pack/metricbeat/module/prometheus/remote_write/data.go b/x-pack/metricbeat/module/prometheus/remote_write/data.go index a5f1eac6993..15ef54d74be 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/data.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/data.go @@ -18,7 +18,7 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/prometheus/remote_write" - xcollector "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus/collector" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus/collector" ) type histogram struct { @@ -37,7 +37,7 @@ func remoteWriteEventsGeneratorFactory(base mb.BaseMetricSet) (remote_write.Remo if config.UseTypes { // use a counter cache with a timeout of 5x the period, as a safe value // to make sure that all counters are available between fetches - counters := xcollector.NewCounterCache(base.Module().Config().Period * 5) + counters := collector.NewCounterCache(base.Module().Config().Period * 5) g := RemoteWriteTypedGenerator{ CounterCache: counters, @@ -51,7 +51,7 @@ func remoteWriteEventsGeneratorFactory(base mb.BaseMetricSet) (remote_write.Remo } type RemoteWriteTypedGenerator struct { - CounterCache xcollector.CounterCache + CounterCache collector.CounterCache RateCounters bool } @@ -121,14 +121,10 @@ func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str e := eventList[labelsHash] e.Timestamp = metric.Timestamp.Time() switch promType { - case "counter_float": + case "counter": data = common.MapStr{ name: g.rateCounterFloat64(name, labels, val), } - case "counter_int": - data = common.MapStr{ - name: g.rateCounterUint64(name, labels, uint64(val)), - } case "other": data = common.MapStr{ name: common.MapStr{ @@ -219,7 +215,7 @@ func (g *RemoteWriteTypedGenerator) processPromHistograms(eventList map[string]m name := strings.TrimSuffix(histogram.metricName, "_bucket") data := common.MapStr{ name: common.MapStr{ - "histogram": xcollector.PromHistogramToES(g.CounterCache, histogram.metricName, histogram.labels, &hist), + "histogram": collector.PromHistogramToES(g.CounterCache, histogram.metricName, histogram.labels, &hist), }, } e.ModuleFields.Update(data) @@ -232,10 +228,9 @@ func findType(metricName string, labels common.MapStr) string { if _, ok := labels["le"]; ok { leLabel = true } - if strings.Contains(metricName, "_total") || strings.Contains(metricName, "_sum") { - return "counter_float" - } else if strings.Contains(metricName, "_count") { - return "counter_int" + if strings.Contains(metricName, "_total") || strings.Contains(metricName, "_sum") || + strings.Contains(metricName, "_count") { + return "counter" } else if strings.Contains(metricName, "_bucket") && leLabel { return "histogram" } diff --git a/x-pack/metricbeat/module/prometheus/remote_write/remote_write.go b/x-pack/metricbeat/module/prometheus/remote_write/remote_write.go index 3244500163f..74eadff6d7b 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/remote_write.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/remote_write.go @@ -12,7 +12,7 @@ import ( func init() { mb.Registry.MustAddMetricSet("prometheus", "remote_write", - remote_write.MetricSetBuilder("prometheus", remoteWriteEventsGeneratorFactory), + remote_write.MetricSetBuilder(remoteWriteEventsGeneratorFactory), mb.WithHostParser(parse.EmptyHostParser), // must replace ensures that we are replacing the oss implementation with this one diff --git a/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go b/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go index b615d9ccc68..92f16bc8c39 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go @@ -545,8 +545,8 @@ func TestGenerateEventsQuantilesDifferentLabels(t *testing.T) { } expected3 := common.MapStr{ "go_gc_duration_seconds_count": common.MapStr{ - "counter": uint64(45), - "rate": uint64(0), + "counter": float64(45), + "rate": float64(0), }, "go_gc_duration_seconds_sum": common.MapStr{ "counter": float64(44), @@ -628,8 +628,8 @@ func TestGenerateEventsQuantilesDifferentLabels(t *testing.T) { } expected3 = common.MapStr{ "go_gc_duration_seconds_count": common.MapStr{ - "counter": uint64(55), - "rate": uint64(10), + "counter": float64(55), + "rate": float64(10), }, "go_gc_duration_seconds_sum": common.MapStr{ "counter": float64(54), @@ -813,8 +813,8 @@ func TestGenerateEventsHistogramsDifferentLabels(t *testing.T) { "rate": float64(0), }, "http_request_duration_seconds_count": common.MapStr{ - "counter": uint64(46), - "rate": uint64(0), + "counter": float64(46), + "rate": float64(0), }, "http_request_bytes": common.MapStr{ "histogram": common.MapStr{ @@ -827,8 +827,8 @@ func TestGenerateEventsHistogramsDifferentLabels(t *testing.T) { "rate": float64(0), }, "http_request_bytes_count": common.MapStr{ - "counter": uint64(56), - "rate": uint64(0), + "counter": float64(56), + "rate": float64(0), }, "labels": labels, } @@ -844,8 +844,8 @@ func TestGenerateEventsHistogramsDifferentLabels(t *testing.T) { "rate": float64(0), }, "http_request_bytes_count": common.MapStr{ - "counter": uint64(66), - "rate": uint64(0), + "counter": float64(66), + "rate": float64(0), }, "labels": labels2, } @@ -996,7 +996,7 @@ func TestGenerateEventsHistogramsDifferentLabels(t *testing.T) { "http_request_duration_seconds": common.MapStr{ "histogram": common.MapStr{ "values": []float64{float64(0.125), float64(0.375), float64(0.75)}, - "counts": []uint64{uint64(100), uint64(100), uint64(100)}, + "counts": []uint64{uint64(100), uint64(0), uint64(0)}, }, }, "http_request_duration_seconds_sum": common.MapStr{ @@ -1004,13 +1004,13 @@ func TestGenerateEventsHistogramsDifferentLabels(t *testing.T) { "rate": float64(100), }, "http_request_duration_seconds_count": common.MapStr{ - "counter": uint64(146), - "rate": uint64(100), + "counter": float64(146), + "rate": float64(100), }, "http_request_bytes": common.MapStr{ "histogram": common.MapStr{ "values": []float64{float64(0.125), float64(0.375), float64(0.75)}, - "counts": []uint64{uint64(200), uint64(200), uint64(200)}, + "counts": []uint64{uint64(200), uint64(0), uint64(0)}, }, }, "http_request_bytes_sum": common.MapStr{ @@ -1018,8 +1018,8 @@ func TestGenerateEventsHistogramsDifferentLabels(t *testing.T) { "rate": float64(200), }, "http_request_bytes_count": common.MapStr{ - "counter": uint64(256), - "rate": uint64(200), + "counter": float64(256), + "rate": float64(200), }, "labels": labels, } @@ -1027,7 +1027,7 @@ func TestGenerateEventsHistogramsDifferentLabels(t *testing.T) { "http_request_bytes": common.MapStr{ "histogram": common.MapStr{ "values": []float64{float64(0.125), float64(0.375), float64(0.75)}, - "counts": []uint64{uint64(300), uint64(300), uint64(300)}, + "counts": []uint64{uint64(300), uint64(0), uint64(0)}, }, }, "http_request_bytes_sum": common.MapStr{ @@ -1035,8 +1035,8 @@ func TestGenerateEventsHistogramsDifferentLabels(t *testing.T) { "rate": float64(300), }, "http_request_bytes_count": common.MapStr{ - "counter": uint64(366), - "rate": uint64(300), + "counter": float64(366), + "rate": float64(300), }, "labels": labels2, } From 234727b361840ac1dceea305f89cafa1bcb85139 Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 20 Jul 2020 13:28:49 +0300 Subject: [PATCH 14/24] Add pattern config option Signed-off-by: chrismark --- metricbeat/helper/prometheus/prometheus.go | 27 +++++ .../module/prometheus/collector/collector.go | 34 +----- .../prometheus/collector/collector_test.go | 17 +-- .../module/prometheus/collector/counter.go | 4 +- .../prometheus/collector/counter_test.go | 4 +- .../module/prometheus/collector/data.go | 26 ++--- .../module/prometheus/remote_write/config.go | 16 ++- .../module/prometheus/remote_write/data.go | 102 ++++++++++++------ .../remote_write/remote_write_test.go | 48 ++++----- 9 files changed, 163 insertions(+), 115 deletions(-) diff --git a/metricbeat/helper/prometheus/prometheus.go b/metricbeat/helper/prometheus/prometheus.go index 2859178d98f..87e21c75083 100644 --- a/metricbeat/helper/prometheus/prometheus.go +++ b/metricbeat/helper/prometheus/prometheus.go @@ -22,6 +22,7 @@ import ( "io" "io/ioutil" "net/http" + "regexp" "github.com/pkg/errors" dto "github.com/prometheus/client_model/go" @@ -284,3 +285,29 @@ func getLabels(metric *dto.Metric) common.MapStr { } return labels } + +func CompilePatternList(patterns *[]string) ([]*regexp.Regexp, error) { + var compiledPatterns []*regexp.Regexp + compiledPatterns = []*regexp.Regexp{} + if patterns != nil { + for _, pattern := range *patterns { + r, err := regexp.Compile(pattern) + if err != nil { + return nil, errors.Wrapf(err, "compiling pattern '%s'", pattern) + } + compiledPatterns = append(compiledPatterns, r) + } + return compiledPatterns, nil + } + return []*regexp.Regexp{}, nil +} + +func MatchMetricFamily(family string, matchMetrics []*regexp.Regexp) bool { + for _, checkMetric := range matchMetrics { + matched := checkMetric.MatchString(family) + if matched { + return true + } + } + return false +} diff --git a/metricbeat/module/prometheus/collector/collector.go b/metricbeat/module/prometheus/collector/collector.go index 6941f30bd8a..ce3cee8cb60 100644 --- a/metricbeat/module/prometheus/collector/collector.go +++ b/metricbeat/module/prometheus/collector/collector.go @@ -111,11 +111,11 @@ func MetricSetBuilder(namespace string, genFactory PromEventsGeneratorFactory) f } // store host here to use it as a pointer when building `up` metric ms.host = ms.Host() - ms.excludeMetrics, err = compilePatternList(config.MetricsFilters.ExcludeMetrics) + ms.excludeMetrics, err = p.CompilePatternList(config.MetricsFilters.ExcludeMetrics) if err != nil { return nil, errors.Wrapf(err, "unable to compile exclude patterns") } - ms.includeMetrics, err = compilePatternList(config.MetricsFilters.IncludeMetrics) + ms.includeMetrics, err = p.CompilePatternList(config.MetricsFilters.IncludeMetrics) if err != nil { return nil, errors.Wrapf(err, "unable to compile include patterns") } @@ -237,39 +237,13 @@ func (m *MetricSet) skipFamilyName(family string) bool { // if include_metrics are defined, check if this metric should be included if len(m.includeMetrics) > 0 { - if !matchMetricFamily(family, m.includeMetrics) { + if !p.MatchMetricFamily(family, m.includeMetrics) { return true } } // now exclude the metric if it matches any of the given patterns if len(m.excludeMetrics) > 0 { - if matchMetricFamily(family, m.excludeMetrics) { - return true - } - } - return false -} - -func compilePatternList(patterns *[]string) ([]*regexp.Regexp, error) { - var compiledPatterns []*regexp.Regexp - compiledPatterns = []*regexp.Regexp{} - if patterns != nil { - for _, pattern := range *patterns { - r, err := regexp.Compile(pattern) - if err != nil { - return nil, errors.Wrapf(err, "compiling pattern '%s'", pattern) - } - compiledPatterns = append(compiledPatterns, r) - } - return compiledPatterns, nil - } - return []*regexp.Regexp{}, nil -} - -func matchMetricFamily(family string, matchMetrics []*regexp.Regexp) bool { - for _, checkMetric := range matchMetrics { - matched := checkMetric.MatchString(family) - if matched { + if p.MatchMetricFamily(family, m.excludeMetrics) { return true } } diff --git a/metricbeat/module/prometheus/collector/collector_test.go b/metricbeat/module/prometheus/collector/collector_test.go index 94477a0aa2b..541b83b8f83 100644 --- a/metricbeat/module/prometheus/collector/collector_test.go +++ b/metricbeat/module/prometheus/collector/collector_test.go @@ -29,6 +29,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/common" + p "github.com/elastic/beats/v7/metricbeat/helper/prometheus" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" _ "github.com/elastic/beats/v7/metricbeat/module/prometheus" @@ -330,8 +331,8 @@ func TestSkipMetricFamily(t *testing.T) { } // test with no filters - ms.includeMetrics, _ = compilePatternList(&[]string{}) - ms.excludeMetrics, _ = compilePatternList(&[]string{}) + ms.includeMetrics, _ = p.CompilePatternList(&[]string{}) + ms.excludeMetrics, _ = p.CompilePatternList(&[]string{}) metricsToKeep := 0 for _, testFamily := range testFamilies { if !ms.skipFamily(testFamily) { @@ -341,8 +342,8 @@ func TestSkipMetricFamily(t *testing.T) { assert.Equal(t, metricsToKeep, len(testFamilies)) // test with only one include filter - ms.includeMetrics, _ = compilePatternList(&[]string{"http_request_duration_microseconds_a_*"}) - ms.excludeMetrics, _ = compilePatternList(&[]string{}) + ms.includeMetrics, _ = p.CompilePatternList(&[]string{"http_request_duration_microseconds_a_*"}) + ms.excludeMetrics, _ = p.CompilePatternList(&[]string{}) metricsToKeep = 0 for _, testFamily := range testFamilies { if !ms.skipFamily(testFamily) { @@ -352,8 +353,8 @@ func TestSkipMetricFamily(t *testing.T) { assert.Equal(t, metricsToKeep, 2) // test with only one exclude filter - ms.includeMetrics, _ = compilePatternList(&[]string{""}) - ms.excludeMetrics, _ = compilePatternList(&[]string{"http_request_duration_microseconds_a_*"}) + ms.includeMetrics, _ = p.CompilePatternList(&[]string{""}) + ms.excludeMetrics, _ = p.CompilePatternList(&[]string{"http_request_duration_microseconds_a_*"}) metricsToKeep = 0 for _, testFamily := range testFamilies { if !ms.skipFamily(testFamily) { @@ -363,8 +364,8 @@ func TestSkipMetricFamily(t *testing.T) { assert.Equal(t, len(testFamilies)-2, metricsToKeep) // test with ine include and one exclude - ms.includeMetrics, _ = compilePatternList(&[]string{"http_request_duration_microseconds_a_*"}) - ms.excludeMetrics, _ = compilePatternList(&[]string{"http_request_duration_microseconds_a_b_*"}) + ms.includeMetrics, _ = p.CompilePatternList(&[]string{"http_request_duration_microseconds_a_*"}) + ms.excludeMetrics, _ = p.CompilePatternList(&[]string{"http_request_duration_microseconds_a_b_*"}) metricsToKeep = 0 for _, testFamily := range testFamilies { if !ms.skipFamily(testFamily) { diff --git a/x-pack/metricbeat/module/prometheus/collector/counter.go b/x-pack/metricbeat/module/prometheus/collector/counter.go index 6f0f72d80eb..ab2c1871b74 100644 --- a/x-pack/metricbeat/module/prometheus/collector/counter.go +++ b/x-pack/metricbeat/module/prometheus/collector/counter.go @@ -10,7 +10,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" ) -// CounterCache keeps a cache of the last value of all given counters +// counterCache keeps a cache of the last value of all given counters // and allows to calculate their rate since the last call. // All methods are thread-unsafe and must not be called concurrently type CounterCache interface { @@ -38,7 +38,7 @@ type counterCache struct { timeout time.Duration } -// NewCounterCache initializes and returns a CounterCache. The timeout parameter will be +// NewCounterCache initializes and returns a counterCache. The timeout parameter will be // used to automatically expire counters that hasn't been updated in a whole timeout period func NewCounterCache(timeout time.Duration) CounterCache { return &counterCache{ diff --git a/x-pack/metricbeat/module/prometheus/collector/counter_test.go b/x-pack/metricbeat/module/prometheus/collector/counter_test.go index 57bae0052cf..dc4e9cd6423 100644 --- a/x-pack/metricbeat/module/prometheus/collector/counter_test.go +++ b/x-pack/metricbeat/module/prometheus/collector/counter_test.go @@ -51,13 +51,13 @@ func Test_CounterCache(t *testing.T) { for i, val := range tt.valuesUint64 { want := tt.expectedUin64[i] if got, _ := tt.counterCache.RateUint64(tt.counterName, val); got != want { - t.Errorf("CounterCache.RateUint64() = %v, want %v", got, want) + t.Errorf("counterCache.RateUint64() = %v, want %v", got, want) } } for i, val := range tt.valuesFloat64 { want := tt.expectedFloat64[i] if got, _ := tt.counterCache.RateFloat64(tt.counterName, val); got != want { - t.Errorf("CounterCache.RateFloat64() = %v, want %v", got, want) + t.Errorf("counterCache.RateFloat64() = %v, want %v", got, want) } } }) diff --git a/x-pack/metricbeat/module/prometheus/collector/data.go b/x-pack/metricbeat/module/prometheus/collector/data.go index 855c2616b0e..1dd83a82980 100644 --- a/x-pack/metricbeat/module/prometheus/collector/data.go +++ b/x-pack/metricbeat/module/prometheus/collector/data.go @@ -29,8 +29,8 @@ func promEventsGeneratorFactory(base mb.BaseMetricSet) (collector.PromEventsGene counters := NewCounterCache(base.Module().Config().Period * 5) g := typedGenerator{ - CounterCache: counters, - RateCounters: config.RateCounters, + counterCache: counters, + rateCounters: config.RateCounters, } return &g, nil @@ -40,23 +40,23 @@ func promEventsGeneratorFactory(base mb.BaseMetricSet) (collector.PromEventsGene } type typedGenerator struct { - CounterCache CounterCache - RateCounters bool + counterCache CounterCache + rateCounters bool } func (g *typedGenerator) Start() { cfgwarn.Beta("Prometheus 'use_types' setting is beta") - if g.RateCounters { + if g.rateCounters { cfgwarn.Experimental("Prometheus 'rate_counters' setting is experimental") } - g.CounterCache.Start() + g.counterCache.Start() } func (g *typedGenerator) Stop() { - logp.Debug("prometheus.collector.cache", "stopping CounterCache") - g.CounterCache.Stop() + logp.Debug("prometheus.collector.cache", "stopping counterCache") + g.counterCache.Stop() } // GeneratePromEvents stores all Prometheus metrics using @@ -138,7 +138,7 @@ func (g *typedGenerator) GeneratePromEvents(mf *dto.MetricFamily) []collector.Pr events = append(events, collector.PromEvent{ Data: common.MapStr{ name: common.MapStr{ - "histogram": PromHistogramToES(g.CounterCache, name, labels, histogram), + "histogram": PromHistogramToES(g.counterCache, name, labels, histogram), }, }, Labels: labels, @@ -172,8 +172,8 @@ func (g *typedGenerator) rateCounterUint64(name string, labels common.MapStr, va "counter": value, } - if g.RateCounters { - d["rate"], _ = g.CounterCache.RateUint64(name+labels.String(), value) + if g.rateCounters { + d["rate"], _ = g.counterCache.RateUint64(name+labels.String(), value) } return d @@ -185,8 +185,8 @@ func (g *typedGenerator) rateCounterFloat64(name string, labels common.MapStr, v "counter": value, } - if g.RateCounters { - d["rate"], _ = g.CounterCache.RateFloat64(name+labels.String(), value) + if g.rateCounters { + d["rate"], _ = g.counterCache.RateFloat64(name+labels.String(), value) } return d diff --git a/x-pack/metricbeat/module/prometheus/remote_write/config.go b/x-pack/metricbeat/module/prometheus/remote_write/config.go index 2334d6e5ff9..8c5fe12a659 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/config.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/config.go @@ -7,8 +7,20 @@ package remote_write import "errors" type config struct { - UseTypes bool `config:"use_types"` - RateCounters bool `config:"rate_counters"` + UseTypes bool `config:"use_types"` + RateCounters bool `config:"rate_counters"` + TypesPatterns TypesPatterns `config:"types_patterns" yaml:"types_patterns,omitempty"` +} + +type TypesPatterns struct { + CounterPatterns *[]string `config:"counter_patterns" yaml:"include,omitempty"` + HistogramPatterns *[]string `config:"histogram_patterns" yaml:"exclude,omitempty"` +} + +var defaultConfig = config{ + TypesPatterns: TypesPatterns{ + CounterPatterns: nil, + HistogramPatterns: nil}, } func (c *config) Validate() error { diff --git a/x-pack/metricbeat/module/prometheus/remote_write/data.go b/x-pack/metricbeat/module/prometheus/remote_write/data.go index 15ef54d74be..18fe0d343d5 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/data.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/data.go @@ -6,21 +6,31 @@ package remote_write import ( "math" + "regexp" "strconv" "strings" "time" + "github.com/pkg/errors" + dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/model" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/logp" + p "github.com/elastic/beats/v7/metricbeat/helper/prometheus" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/prometheus/remote_write" "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus/collector" ) +const ( + counterType = "counter_type" + histogramType = "histgram_type" + otherType = "other_type" +) + type histogram struct { timestamp time.Time buckets []*dto.Bucket @@ -29,8 +39,9 @@ type histogram struct { } func remoteWriteEventsGeneratorFactory(base mb.BaseMetricSet) (remote_write.RemoteWriteEventsGenerator, error) { - config := config{} - if err := base.Module().UnpackConfig(&config); err != nil { + var err error + config := defaultConfig + if err = base.Module().UnpackConfig(&config); err != nil { return nil, err } @@ -39,9 +50,18 @@ func remoteWriteEventsGeneratorFactory(base mb.BaseMetricSet) (remote_write.Remo // to make sure that all counters are available between fetches counters := collector.NewCounterCache(base.Module().Config().Period * 5) - g := RemoteWriteTypedGenerator{ - CounterCache: counters, - RateCounters: config.RateCounters, + g := remoteWriteTypedGenerator{ + counterCache: counters, + rateCounters: config.RateCounters, + } + + g.counterPatterns, err = p.CompilePatternList(config.TypesPatterns.CounterPatterns) + if err != nil { + return nil, errors.Wrapf(err, "unable to compile counter patterns") + } + g.histogramPatterns, err = p.CompilePatternList(config.TypesPatterns.HistogramPatterns) + if err != nil { + return nil, errors.Wrapf(err, "unable to compile histogram patterns") } return &g, nil @@ -50,24 +70,26 @@ func remoteWriteEventsGeneratorFactory(base mb.BaseMetricSet) (remote_write.Remo return remote_write.DefaultRemoteWriteEventsGeneratorFactory(base) } -type RemoteWriteTypedGenerator struct { - CounterCache collector.CounterCache - RateCounters bool +type remoteWriteTypedGenerator struct { + counterCache collector.CounterCache + rateCounters bool + counterPatterns []*regexp.Regexp + histogramPatterns []*regexp.Regexp } -func (g *RemoteWriteTypedGenerator) Start() { +func (g *remoteWriteTypedGenerator) Start() { cfgwarn.Beta("Prometheus 'use_types' setting is beta") - if g.RateCounters { + if g.rateCounters { cfgwarn.Experimental("Prometheus 'rate_counters' setting is experimental") } - g.CounterCache.Start() + g.counterCache.Start() } -func (g *RemoteWriteTypedGenerator) Stop() { - logp.Debug("prometheus.remote_write.cache", "stopping CounterCache") - g.CounterCache.Stop() +func (g *remoteWriteTypedGenerator) Stop() { + logp.Debug("prometheus.remote_write.cache", "stopping counterCache") + g.counterCache.Stop() } // GenerateEvents receives a list of Sample and: @@ -75,7 +97,7 @@ func (g *RemoteWriteTypedGenerator) Stop() { // 2. handle it properly using "types" logic // 3. if metrics of histogram type then it is converted to ES histogram // 4. metrics with the same set of labels are grouped into same events -func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[string]mb.Event { +func (g remoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[string]mb.Event { var data common.MapStr histograms := map[string]histogram{} eventList := map[string]mb.Event{} @@ -93,14 +115,14 @@ func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str labels[string(k)] = v } - promType := findType(name, labels) + promType := g.findMetricType(name, labels) val := float64(metric.Value) if !math.IsNaN(val) && !math.IsInf(val, 0) { // join metrics with same labels in a single event labelsHash := labels.String() labelsClone := labels.Clone() labelsClone.Delete("le") - if promType == "histogram" { + if promType == histogramType { labelsHash = labelsClone.String() } if _, ok := eventList[labelsHash]; !ok { @@ -110,7 +132,7 @@ func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str // Add labels if len(labels) > 0 { - if promType == "histogram" { + if promType == histogramType { eventList[labelsHash].ModuleFields["labels"] = labelsClone } else { eventList[labelsHash].ModuleFields["labels"] = labels @@ -121,17 +143,17 @@ func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str e := eventList[labelsHash] e.Timestamp = metric.Timestamp.Time() switch promType { - case "counter": + case counterType: data = common.MapStr{ name: g.rateCounterFloat64(name, labels, val), } - case "other": + case otherType: data = common.MapStr{ name: common.MapStr{ "value": val, }, } - case "histogram": + case histogramType: histKey := name + labelsClone.String() le, _ := labels.GetValue("le") @@ -167,32 +189,32 @@ func (g RemoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str } // rateCounterUint64 fills a counter value and optionally adds the rate if rate_counters is enabled -func (g *RemoteWriteTypedGenerator) rateCounterUint64(name string, labels common.MapStr, value uint64) common.MapStr { +func (g *remoteWriteTypedGenerator) rateCounterUint64(name string, labels common.MapStr, value uint64) common.MapStr { d := common.MapStr{ "counter": value, } - if g.RateCounters { - d["rate"], _ = g.CounterCache.RateUint64(name+labels.String(), value) + if g.rateCounters { + d["rate"], _ = g.counterCache.RateUint64(name+labels.String(), value) } return d } // rateCounterFloat64 fills a counter value and optionally adds the rate if rate_counters is enabled -func (g *RemoteWriteTypedGenerator) rateCounterFloat64(name string, labels common.MapStr, value float64) common.MapStr { +func (g *remoteWriteTypedGenerator) rateCounterFloat64(name string, labels common.MapStr, value float64) common.MapStr { d := common.MapStr{ "counter": value, } - if g.RateCounters { - d["rate"], _ = g.CounterCache.RateFloat64(name+labels.String(), value) + if g.rateCounters { + d["rate"], _ = g.counterCache.RateFloat64(name+labels.String(), value) } return d } // processPromHistograms receives a group of Histograms and converts each one to ES histogram -func (g *RemoteWriteTypedGenerator) processPromHistograms(eventList map[string]mb.Event, histograms map[string]histogram) { +func (g *remoteWriteTypedGenerator) processPromHistograms(eventList map[string]mb.Event, histograms map[string]histogram) { for _, histogram := range histograms { labelsHash := histogram.labels.String() if _, ok := eventList[labelsHash]; !ok { @@ -215,24 +237,36 @@ func (g *RemoteWriteTypedGenerator) processPromHistograms(eventList map[string]m name := strings.TrimSuffix(histogram.metricName, "_bucket") data := common.MapStr{ name: common.MapStr{ - "histogram": collector.PromHistogramToES(g.CounterCache, histogram.metricName, histogram.labels, &hist), + "histogram": collector.PromHistogramToES(g.counterCache, histogram.metricName, histogram.labels, &hist), }, } e.ModuleFields.Update(data) } } -// findType evaluates the type of the metric by check the metricname format in order to handle it properly -func findType(metricName string, labels common.MapStr) string { +// findMetricType evaluates the type of the metric by check the metricname format in order to handle it properly +func (g *remoteWriteTypedGenerator) findMetricType(metricName string, labels common.MapStr) string { leLabel := false if _, ok := labels["le"]; ok { leLabel = true } if strings.Contains(metricName, "_total") || strings.Contains(metricName, "_sum") || strings.Contains(metricName, "_count") { - return "counter" + return counterType } else if strings.Contains(metricName, "_bucket") && leLabel { - return "histogram" + return histogramType + } + + // handle user provided patterns + if len(g.counterPatterns) > 0 { + if !p.MatchMetricFamily(metricName, g.counterPatterns) { + return counterType + } + } + if len(g.histogramPatterns) > 0 { + if !p.MatchMetricFamily(metricName, g.histogramPatterns) { + return histogramType + } } - return "other" + return otherType } diff --git a/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go b/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go index 92f16bc8c39..24c240e3e06 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go @@ -23,11 +23,11 @@ func TestGenerateEventsCounter(t *testing.T) { counters := xcollector.NewCounterCache(1 * time.Second) - g := RemoteWriteTypedGenerator{ - CounterCache: counters, - RateCounters: true, + g := remoteWriteTypedGenerator{ + counterCache: counters, + rateCounters: true, } - g.CounterCache.Start() + g.counterCache.Start() labels := common.MapStr{ "listener_name": model.LabelValue("http"), } @@ -89,11 +89,11 @@ func TestGenerateEventsCounterSameLabels(t *testing.T) { counters := xcollector.NewCounterCache(1 * time.Second) - g := RemoteWriteTypedGenerator{ - CounterCache: counters, - RateCounters: true, + g := remoteWriteTypedGenerator{ + counterCache: counters, + rateCounters: true, } - g.CounterCache.Start() + g.counterCache.Start() labels := common.MapStr{ "listener_name": model.LabelValue("http"), } @@ -179,11 +179,11 @@ func TestGenerateEventsCounterDifferentLabels(t *testing.T) { counters := xcollector.NewCounterCache(1 * time.Second) - g := RemoteWriteTypedGenerator{ - CounterCache: counters, - RateCounters: true, + g := remoteWriteTypedGenerator{ + counterCache: counters, + rateCounters: true, } - g.CounterCache.Start() + g.counterCache.Start() labels := common.MapStr{ "listener_name": model.LabelValue("http"), } @@ -309,11 +309,11 @@ func TestGenerateEventsGaugeDifferentLabels(t *testing.T) { counters := xcollector.NewCounterCache(1 * time.Second) - g := RemoteWriteTypedGenerator{ - CounterCache: counters, - RateCounters: true, + g := remoteWriteTypedGenerator{ + counterCache: counters, + rateCounters: true, } - g.CounterCache.Start() + g.counterCache.Start() labels := common.MapStr{ "listener_name": model.LabelValue("http"), } @@ -463,11 +463,11 @@ func TestGenerateEventsQuantilesDifferentLabels(t *testing.T) { counters := xcollector.NewCounterCache(1 * time.Second) - g := RemoteWriteTypedGenerator{ - CounterCache: counters, - RateCounters: true, + g := remoteWriteTypedGenerator{ + counterCache: counters, + rateCounters: true, } - g.CounterCache.Start() + g.counterCache.Start() labels := common.MapStr{ "runtime": model.LabelValue("linux"), "quantile": model.LabelValue("0.25"), @@ -653,11 +653,11 @@ func TestGenerateEventsHistogramsDifferentLabels(t *testing.T) { counters := xcollector.NewCounterCache(1 * time.Second) - g := RemoteWriteTypedGenerator{ - CounterCache: counters, - RateCounters: true, + g := remoteWriteTypedGenerator{ + counterCache: counters, + rateCounters: true, } - g.CounterCache.Start() + g.counterCache.Start() labels := common.MapStr{ "runtime": model.LabelValue("linux"), } From 9cc353be618b402830d7c9d43193b8e3237aa7a1 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 21 Jul 2020 11:57:50 +0300 Subject: [PATCH 15/24] Add docs Signed-off-by: chrismark --- metricbeat/docs/fields.asciidoc | 2 +- .../remote_write/_meta/docs.asciidoc | 101 ++++++++++++++++++ x-pack/metricbeat/metricbeat.reference.yml | 12 +++ .../module/prometheus/_meta/config.yml | 12 +++ x-pack/metricbeat/module/prometheus/fields.go | 2 +- .../prometheus/remote_write/_meta/fields.yml | 1 + .../module/prometheus/remote_write/data.go | 4 +- .../modules.d/prometheus.yml.disabled | 12 +++ 8 files changed, 142 insertions(+), 4 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 2c9c7c00124..c92ceddb5b2 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -35084,7 +35084,7 @@ type: object *`prometheus.*.histogram`*:: + -- -Prometheus histogram metric - release: ga +Prometheus histogram metric - release: ga - release: ga type: object diff --git a/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc b/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc index 99f5e120d1a..3183ba6f2c1 100644 --- a/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc @@ -61,3 +61,104 @@ remote_write: # Disable validation of the server certificate. #insecure_skip_verify: true ------------------------------------------------------------------------------ + + +[float] +[role="xpack"] +=== Histograms and types + +beta[] + +[source,yaml] +------------------------------------------------------------------------------------- +metricbeat.modules: +- module: prometheus + metricsets: ["remote_write"] + host: "localhost" + port: "9201" +------------------------------------------------------------------------------------- + +`use_types` paramater (default: false) enables a different layout for metrics storage, leveraging Elasticsearch +types, including https://www.elastic.co/guide/en/elasticsearch/reference/current/histogram.html[histograms]. + +`rate_counters` paramater (default: false) enables calculating a rate out of Prometheus counters. When enabled, Metricbeat stores +the counter increment since the last collection. This metric should make some aggregations easier and with better +performance. This parameter can only be enabled in combination with `use_types`. + +When `use_types` and `rate_counters` are enabled, metrics are stored like this: + +[source,json] +---- +{ + "prometheus": { + "labels": { + "instance": "172.27.0.2:9090", + "job": "prometheus" + }, + "prometheus_target_interval_length_seconds_count": { + "counter": 1, + "rate": 0 + }, + "prometheus_target_interval_length_seconds_sum": { + "counter": 15.000401344, + "rate": 0 + } + "prometheus_tsdb_compaction_chunk_range_seconds_bucket": { + "histogram": { + "values": [50, 300, 1000, 4000, 16000], + "counts": [10, 2, 34, 7] + } + } + }, +} +---- + + +[float] +==== Types' patterns + +Unline `collector` metricset, `remote_write` receives metrics in raw form from the prometheus server. +In this, the module has to internally use a heuristic in order to identify efficiently the type of each raw metric. +For these purpose some name patterns are used in order to identify the type of each metric. +The default patterns are the following: + +. `_total` suffix: the metric is of Counter type +. `_sum` suffix: the metric is of Counter type +. `_count` suffix: the metric is of Counter type +. `_bucket` suffix and `le` in labels: the metric is of Histogram type + +Everything else is handled as a Gauge. + +Users have the flexibility to add their own patterns using the following configuration: + +[source,yaml] +------------------------------------------------------------------------------------- +metricbeat.modules: +- module: prometheus + metricsets: ["remote_write"] + host: "localhost" + port: "9201" + types_patterns: + counter_patterns: ["*_my_counter_suffix"] + histogram_patterns: ["*_my_histogram_suffix"] +------------------------------------------------------------------------------------- + +The configuration above will consider metrics with names that match `*_my_counter_suffix` as Counters +and those that match match `*_my_histogram_suffix` (and have `le` in their labels) as Histograms. + + +To match only specific metrics, anchor the start and the end of the regexp of each metric: + +- the caret ^ matches the beginning of a text or line, +- the dollar sign $ matches the end of a text. + +[source,yaml] +------------------------------------------------------------------------------------- +metricbeat.modules: +- module: prometheus + metricsets: ["remote_write"] + host: "localhost" + port: "9201" + types_patterns: + histogram_patterns: ["^my_histogram_metirc$"] +------------------------------------------------------------------------------------- diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index ee5d7893bc0..3bb51771337 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1095,6 +1095,18 @@ metricbeat.modules: # Store counter rates instead of original cumulative counters (experimental, default: false) #rate_counters: true + # Use Elasticsearch histogram type to store histograms (beta, default: false) + # This will change the default layout and put metric type in the field name + #use_types: true + + # Store counter rates instead of original cumulative counters (experimental, default: false) + #rate_counters: true + + #types_patterns: + # counter_patterns: [] + # histogram_patterns: [] + + # Metrics sent by a Prometheus server using remote_write option #- module: prometheus # metricsets: ["remote_write"] diff --git a/x-pack/metricbeat/module/prometheus/_meta/config.yml b/x-pack/metricbeat/module/prometheus/_meta/config.yml index 6fd4e582c8e..c845c9ceff5 100644 --- a/x-pack/metricbeat/module/prometheus/_meta/config.yml +++ b/x-pack/metricbeat/module/prometheus/_meta/config.yml @@ -20,6 +20,18 @@ # Store counter rates instead of original cumulative counters (experimental, default: false) #rate_counters: true + # Use Elasticsearch histogram type to store histograms (beta, default: false) + # This will change the default layout and put metric type in the field name + #use_types: true + + # Store counter rates instead of original cumulative counters (experimental, default: false) + #rate_counters: true + + #types_patterns: + # counter_patterns: [] + # histogram_patterns: [] + + # Metrics sent by a Prometheus server using remote_write option #- module: prometheus # metricsets: ["remote_write"] diff --git a/x-pack/metricbeat/module/prometheus/fields.go b/x-pack/metricbeat/module/prometheus/fields.go index 63330de9110..750e87397ee 100644 --- a/x-pack/metricbeat/module/prometheus/fields.go +++ b/x-pack/metricbeat/module/prometheus/fields.go @@ -19,5 +19,5 @@ func init() { // AssetPrometheus returns asset data. // This is the base64 encoded gzipped contents of module/prometheus. func AssetPrometheus() string { - return "eJzEkktqAzEMhvdzih8vQ5MDeNEzFLosJSi24rjxC0tTmtuXZIYw9EGzKMQ7+/+QPgmvceSTRes1sx54lPVHI3ccAI2a2MI8XSPoqbFHZu3RiRkAz+J6bBprsXgcAOBZSQXiOp3Zfa8ZhEUNLr7VWHQzAJ0Tk7BFoAEQVo0liMWLEUnmAeag2szrAOwjJy/20mGNQpmXzpvV5p3SyJcYF02Luntjp/PTdNlOia/jLvH3ZJuptVjCjJmVmZkfxjyfxVSBxsDzZn6XdHUsyv1+mrPAn6Kd9I7LPHf3N7seomgNnfKNwl/5/3G+Vl36Ts7LX/4ZAAD//xMCFRw=" + return "eJzEkk1qAzEMhfdzioeXockBvOgZCl2WEhRbcdz4D0tTmtuXJEOYtinNohDvrPeQPj20xJ4PFq3XzLrjUZYfjdx+ADRqYgvzdJGgh8YembVHJ2YAPIvrsWmsxeJxAIBnJRWI63T0bnvNIMx6cPGtxqKrAeicmIQtAg2AsGosQSxejEgyDzA71WZeB2AbOXmxpwlLFMo8Z14tVu+URj7JOGFa1M0bO51K58/6rPg6bhL/VNaZWoslTDazMJPnyprHN9sq0Bh4SuZ3SFfHotzvhzkB/AnaSe8Y5nG6v5l1F0Vr6JRvBP7u/x/mS9c575n565Vfq34GAAD//xfAGnY=" } diff --git a/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml b/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml index e69de29bb2d..a927f3fc9f8 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml +++ b/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml @@ -0,0 +1 @@ +- release: ga diff --git a/x-pack/metricbeat/module/prometheus/remote_write/data.go b/x-pack/metricbeat/module/prometheus/remote_write/data.go index 18fe0d343d5..6c8dcf49772 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/data.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/data.go @@ -259,12 +259,12 @@ func (g *remoteWriteTypedGenerator) findMetricType(metricName string, labels com // handle user provided patterns if len(g.counterPatterns) > 0 { - if !p.MatchMetricFamily(metricName, g.counterPatterns) { + if p.MatchMetricFamily(metricName, g.counterPatterns) { return counterType } } if len(g.histogramPatterns) > 0 { - if !p.MatchMetricFamily(metricName, g.histogramPatterns) { + if p.MatchMetricFamily(metricName, g.histogramPatterns) && leLabel { return histogramType } } diff --git a/x-pack/metricbeat/modules.d/prometheus.yml.disabled b/x-pack/metricbeat/modules.d/prometheus.yml.disabled index f5882aff4fc..5443fa72d8a 100644 --- a/x-pack/metricbeat/modules.d/prometheus.yml.disabled +++ b/x-pack/metricbeat/modules.d/prometheus.yml.disabled @@ -23,6 +23,18 @@ # Store counter rates instead of original cumulative counters (experimental, default: false) #rate_counters: true + # Use Elasticsearch histogram type to store histograms (beta, default: false) + # This will change the default layout and put metric type in the field name + #use_types: true + + # Store counter rates instead of original cumulative counters (experimental, default: false) + #rate_counters: true + + #types_patterns: + # counter_patterns: [] + # histogram_patterns: [] + + # Metrics sent by a Prometheus server using remote_write option #- module: prometheus # metricsets: ["remote_write"] From e4f23c935e3c1a0c720c8ddf841791a23fb2bc0c Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 21 Jul 2020 14:19:35 +0300 Subject: [PATCH 16/24] Add tests for patterns Signed-off-by: chrismark --- .../remote_write/_meta/docs.asciidoc | 8 +- .../module/prometheus/remote_write/data.go | 4 +- .../remote_write/remote_write_test.go | 152 +++++++++++++++++- 3 files changed, 157 insertions(+), 7 deletions(-) diff --git a/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc b/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc index 3183ba6f2c1..abe729a4880 100644 --- a/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc @@ -139,12 +139,12 @@ metricbeat.modules: host: "localhost" port: "9201" types_patterns: - counter_patterns: ["*_my_counter_suffix"] - histogram_patterns: ["*_my_histogram_suffix"] + counter_patterns: ["_my_counter_suffix"] + histogram_patterns: ["_my_histogram_suffix"] ------------------------------------------------------------------------------------- -The configuration above will consider metrics with names that match `*_my_counter_suffix` as Counters -and those that match match `*_my_histogram_suffix` (and have `le` in their labels) as Histograms. +The configuration above will consider metrics with names that match `_my_counter_suffix` as Counters +and those that match match `_my_histogram_suffix` (and have `le` in their labels) as Histograms. To match only specific metrics, anchor the start and the end of the regexp of each metric: diff --git a/x-pack/metricbeat/module/prometheus/remote_write/data.go b/x-pack/metricbeat/module/prometheus/remote_write/data.go index 6c8dcf49772..0a0fdafd277 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/data.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/data.go @@ -250,8 +250,8 @@ func (g *remoteWriteTypedGenerator) findMetricType(metricName string, labels com if _, ok := labels["le"]; ok { leLabel = true } - if strings.Contains(metricName, "_total") || strings.Contains(metricName, "_sum") || - strings.Contains(metricName, "_count") { + if strings.HasSuffix(metricName, "_total") || strings.HasSuffix(metricName, "_sum") || + strings.HasSuffix(metricName, "_count") { return counterType } else if strings.Contains(metricName, "_bucket") && leLabel { return histogramType diff --git a/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go b/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go index 24c240e3e06..d5c07f0d2a9 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/remote_write_test.go @@ -11,10 +11,10 @@ import ( "time" "github.com/prometheus/common/model" - "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/common" + p "github.com/elastic/beats/v7/metricbeat/helper/prometheus" xcollector "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus/collector" ) @@ -1047,3 +1047,153 @@ func TestGenerateEventsHistogramsDifferentLabels(t *testing.T) { e = events[labels2.String()] assert.EqualValues(t, e.ModuleFields, expected2) } + +// TestGenerateEventsCounterWithDefinedPattern tests counter with defined pattern +func TestGenerateEventsCounterWithDefinedPattern(t *testing.T) { + + counters := xcollector.NewCounterCache(1 * time.Second) + + counterPatterns, err := p.CompilePatternList(&[]string{"_mycounter"}) + if err != nil { + panic(err) + } + g := remoteWriteTypedGenerator{ + counterCache: counters, + rateCounters: true, + counterPatterns: counterPatterns, + } + + g.counterCache.Start() + labels := common.MapStr{ + "listener_name": model.LabelValue("http"), + } + + // first fetch + metrics := model.Samples{ + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_closed_mycounter", + "listener_name": "http", + }, + Value: model.SampleValue(42), + Timestamp: model.Time(424242), + }, + } + events := g.GenerateEvents(metrics) + + expected := common.MapStr{ + "net_conntrack_listener_conn_closed_mycounter": common.MapStr{ + "counter": float64(42), + "rate": float64(0), + }, + "labels": labels, + } + + assert.Equal(t, len(events), 1) + e := events[labels.String()] + assert.EqualValues(t, e.ModuleFields, expected) + + // repeat in order to test the rate + metrics = model.Samples{ + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_closed_mycounter", + "listener_name": "http", + }, + Value: model.SampleValue(45), + Timestamp: model.Time(424242), + }, + } + events = g.GenerateEvents(metrics) + + expected = common.MapStr{ + "net_conntrack_listener_conn_closed_mycounter": common.MapStr{ + "counter": float64(45), + "rate": float64(3), + }, + "labels": labels, + } + + assert.Equal(t, len(events), 1) + e = events[labels.String()] + assert.EqualValues(t, e.ModuleFields, expected) + +} + +// TestGenerateEventsHistogramWithDefinedPattern tests histogram with defined pattern +func TestGenerateEventsHistogramWithDefinedPattern(t *testing.T) { + + counters := xcollector.NewCounterCache(1 * time.Second) + + histogramPatterns, err := p.CompilePatternList(&[]string{"_myhistogram"}) + if err != nil { + panic(err) + } + g := remoteWriteTypedGenerator{ + counterCache: counters, + rateCounters: true, + histogramPatterns: histogramPatterns, + } + + g.counterCache.Start() + labels := common.MapStr{ + "listener_name": model.LabelValue("http"), + } + + // first fetch + metrics := model.Samples{ + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_closed_myhistogram", + "listener_name": "http", + "le": "20", + }, + Value: model.SampleValue(42), + Timestamp: model.Time(424242), + }, + } + events := g.GenerateEvents(metrics) + + expected := common.MapStr{ + "net_conntrack_listener_conn_closed_myhistogram": common.MapStr{ + "histogram": common.MapStr{ + "values": []float64{float64(10)}, + "counts": []uint64{uint64(0)}, + }, + }, + "labels": labels, + } + + assert.Equal(t, len(events), 1) + e := events[labels.String()] + assert.EqualValues(t, e.ModuleFields, expected) + + // repeat in order to test the rate + metrics = model.Samples{ + &model.Sample{ + Metric: map[model.LabelName]model.LabelValue{ + "__name__": "net_conntrack_listener_conn_closed_myhistogram", + "listener_name": "http", + "le": "20", + }, + Value: model.SampleValue(45), + Timestamp: model.Time(424242), + }, + } + events = g.GenerateEvents(metrics) + + expected = common.MapStr{ + "net_conntrack_listener_conn_closed_myhistogram": common.MapStr{ + "histogram": common.MapStr{ + "values": []float64{float64(10)}, + "counts": []uint64{uint64(3)}, + }, + }, + "labels": labels, + } + + assert.Equal(t, len(events), 1) + e = events[labels.String()] + assert.EqualValues(t, e.ModuleFields, expected) + +} From f0adc1a61098eece47783d774f3449bb20d5d07e Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 21 Jul 2020 14:23:45 +0300 Subject: [PATCH 17/24] add changelog Signed-off-by: chrismark --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 2 +- .../metricbeat/module/prometheus/remote_write/_meta/fields.yml | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9c88a4e6282..47c8fee49be 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -569,6 +569,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added `performance` and `query` metricsets to `mysql` module. {pull}18955[18955] - The `elasticsearch-xpack/index` metricset now reports hidden indices as such. {issue}18639[18639] {pull}18706[18706] - Adds support for app insights metrics in the azure module. {issue}18570[18570] {pull}18940[18940] +- Infer types in Prometheus remote_write. {pull}19944[19944] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index c92ceddb5b2..2c9c7c00124 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -35084,7 +35084,7 @@ type: object *`prometheus.*.histogram`*:: + -- -Prometheus histogram metric - release: ga - release: ga +Prometheus histogram metric - release: ga type: object diff --git a/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml b/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml index a927f3fc9f8..e69de29bb2d 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml +++ b/x-pack/metricbeat/module/prometheus/remote_write/_meta/fields.yml @@ -1 +0,0 @@ -- release: ga From 6c340fde9fc03478e839c80c7eab9394dd62c4c4 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 22 Jul 2020 14:48:50 +0300 Subject: [PATCH 18/24] make update Signed-off-by: chrismark --- x-pack/metricbeat/module/prometheus/fields.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/prometheus/fields.go b/x-pack/metricbeat/module/prometheus/fields.go index 750e87397ee..63330de9110 100644 --- a/x-pack/metricbeat/module/prometheus/fields.go +++ b/x-pack/metricbeat/module/prometheus/fields.go @@ -19,5 +19,5 @@ func init() { // AssetPrometheus returns asset data. // This is the base64 encoded gzipped contents of module/prometheus. func AssetPrometheus() string { - return "eJzEkk1qAzEMhfdzioeXockBvOgZCl2WEhRbcdz4D0tTmtuXJEOYtinNohDvrPeQPj20xJ4PFq3XzLrjUZYfjdx+ADRqYgvzdJGgh8YembVHJ2YAPIvrsWmsxeJxAIBnJRWI63T0bnvNIMx6cPGtxqKrAeicmIQtAg2AsGosQSxejEgyDzA71WZeB2AbOXmxpwlLFMo8Z14tVu+URj7JOGFa1M0bO51K58/6rPg6bhL/VNaZWoslTDazMJPnyprHN9sq0Bh4SuZ3SFfHotzvhzkB/AnaSe8Y5nG6v5l1F0Vr6JRvBP7u/x/mS9c575n565Vfq34GAAD//xfAGnY=" + return "eJzEkktqAzEMhvdzih8vQ5MDeNEzFLosJSi24rjxC0tTmtuXZIYw9EGzKMQ7+/+QPgmvceSTRes1sx54lPVHI3ccAI2a2MI8XSPoqbFHZu3RiRkAz+J6bBprsXgcAOBZSQXiOp3Zfa8ZhEUNLr7VWHQzAJ0Tk7BFoAEQVo0liMWLEUnmAeag2szrAOwjJy/20mGNQpmXzpvV5p3SyJcYF02Luntjp/PTdNlOia/jLvH3ZJuptVjCjJmVmZkfxjyfxVSBxsDzZn6XdHUsyv1+mrPAn6Kd9I7LPHf3N7seomgNnfKNwl/5/3G+Vl36Ts7LX/4ZAAD//xMCFRw=" } From eace032a5171cd467a66837940dafe557f1895c5 Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 27 Jul 2020 15:14:41 +0300 Subject: [PATCH 19/24] clean ups Signed-off-by: chrismark --- .../module/prometheus/remote_write/_meta/docs.asciidoc | 4 ++-- metricbeat/module/prometheus/remote_write/remote_write.go | 8 ++++---- x-pack/metricbeat/metricbeat.reference.yml | 1 + x-pack/metricbeat/module/prometheus/_meta/config.yml | 1 + x-pack/metricbeat/module/prometheus/collector/counter.go | 4 ++-- x-pack/metricbeat/module/prometheus/remote_write/data.go | 2 +- x-pack/metricbeat/modules.d/prometheus.yml.disabled | 1 + 7 files changed, 12 insertions(+), 9 deletions(-) diff --git a/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc b/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc index abe729a4880..2fef11334ed 100644 --- a/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc @@ -144,7 +144,7 @@ metricbeat.modules: ------------------------------------------------------------------------------------- The configuration above will consider metrics with names that match `_my_counter_suffix` as Counters -and those that match match `_my_histogram_suffix` (and have `le` in their labels) as Histograms. +and those that match `_my_histogram_suffix` (and have `le` in their labels) as Histograms. To match only specific metrics, anchor the start and the end of the regexp of each metric: @@ -160,5 +160,5 @@ metricbeat.modules: host: "localhost" port: "9201" types_patterns: - histogram_patterns: ["^my_histogram_metirc$"] + histogram_patterns: ["^my_histogram_metric$"] ------------------------------------------------------------------------------------- diff --git a/metricbeat/module/prometheus/remote_write/remote_write.go b/metricbeat/module/prometheus/remote_write/remote_write.go index 9c4d3657723..72bd93185f6 100644 --- a/metricbeat/module/prometheus/remote_write/remote_write.go +++ b/metricbeat/module/prometheus/remote_write/remote_write.go @@ -39,19 +39,19 @@ func init() { ) } -// RemoteWriteEventsGenerator converts a Prometheus metric Samples mb.Event map +// RemoteWriteEventsGenerator converts Prometheus Samples to a map of mb.Event type RemoteWriteEventsGenerator interface { // Start must be called before using the generator Start() - // converts a Prometheus metric family into a list of PromEvents + // converts Prometheus Samples to a map of mb.Event GenerateEvents(metrics model.Samples) map[string]mb.Event // Stop must be called when the generator won't be used anymore Stop() } -// RemoteWriteEventsGeneratorFactory creates a PromEventsGenerator when instanciating a metricset +// RemoteWriteEventsGeneratorFactory creates a RemoteWriteEventsGenerator when instanciating a metricset type RemoteWriteEventsGeneratorFactory func(ms mb.BaseMetricSet) (RemoteWriteEventsGenerator, error) type MetricSet struct { @@ -89,7 +89,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return m, nil } -// MetricSetBuilder returns a builder function for a new Prometheus metricset using +// MetricSetBuilder returns a builder function for a new Prometheus remote_write metricset using // the given namespace and event generator func MetricSetBuilder(genFactory RemoteWriteEventsGeneratorFactory) func(base mb.BaseMetricSet) (mb.MetricSet, error) { return func(base mb.BaseMetricSet) (mb.MetricSet, error) { diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 805e8411793..141a1cc79b2 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1104,6 +1104,7 @@ metricbeat.modules: # Store counter rates instead of original cumulative counters (experimental, default: false) #rate_counters: true + # Define patterns for counter and histogram types so as to identify metrics' types according to these patterns #types_patterns: # counter_patterns: [] # histogram_patterns: [] diff --git a/x-pack/metricbeat/module/prometheus/_meta/config.yml b/x-pack/metricbeat/module/prometheus/_meta/config.yml index c845c9ceff5..cd54c01383a 100644 --- a/x-pack/metricbeat/module/prometheus/_meta/config.yml +++ b/x-pack/metricbeat/module/prometheus/_meta/config.yml @@ -27,6 +27,7 @@ # Store counter rates instead of original cumulative counters (experimental, default: false) #rate_counters: true + # Define patterns for counter and histogram types so as to identify metrics' types according to these patterns #types_patterns: # counter_patterns: [] # histogram_patterns: [] diff --git a/x-pack/metricbeat/module/prometheus/collector/counter.go b/x-pack/metricbeat/module/prometheus/collector/counter.go index ab2c1871b74..6f0f72d80eb 100644 --- a/x-pack/metricbeat/module/prometheus/collector/counter.go +++ b/x-pack/metricbeat/module/prometheus/collector/counter.go @@ -10,7 +10,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" ) -// counterCache keeps a cache of the last value of all given counters +// CounterCache keeps a cache of the last value of all given counters // and allows to calculate their rate since the last call. // All methods are thread-unsafe and must not be called concurrently type CounterCache interface { @@ -38,7 +38,7 @@ type counterCache struct { timeout time.Duration } -// NewCounterCache initializes and returns a counterCache. The timeout parameter will be +// NewCounterCache initializes and returns a CounterCache. The timeout parameter will be // used to automatically expire counters that hasn't been updated in a whole timeout period func NewCounterCache(timeout time.Duration) CounterCache { return &counterCache{ diff --git a/x-pack/metricbeat/module/prometheus/remote_write/data.go b/x-pack/metricbeat/module/prometheus/remote_write/data.go index 0a0fdafd277..2b7fdc72541 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/data.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/data.go @@ -118,13 +118,13 @@ func (g remoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str promType := g.findMetricType(name, labels) val := float64(metric.Value) if !math.IsNaN(val) && !math.IsInf(val, 0) { - // join metrics with same labels in a single event labelsHash := labels.String() labelsClone := labels.Clone() labelsClone.Delete("le") if promType == histogramType { labelsHash = labelsClone.String() } + // join metrics with same labels in a single event if _, ok := eventList[labelsHash]; !ok { eventList[labelsHash] = mb.Event{ ModuleFields: common.MapStr{}, diff --git a/x-pack/metricbeat/modules.d/prometheus.yml.disabled b/x-pack/metricbeat/modules.d/prometheus.yml.disabled index 5443fa72d8a..5dbe163c62a 100644 --- a/x-pack/metricbeat/modules.d/prometheus.yml.disabled +++ b/x-pack/metricbeat/modules.d/prometheus.yml.disabled @@ -30,6 +30,7 @@ # Store counter rates instead of original cumulative counters (experimental, default: false) #rate_counters: true + # Define patterns for counter and histogram types so as to identify metrics' types according to these patterns #types_patterns: # counter_patterns: [] # histogram_patterns: [] From 99705c08f11917d0cdf76abfd6c4f228598d0dca Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 28 Jul 2020 18:06:21 +0300 Subject: [PATCH 20/24] review changes Signed-off-by: chrismark --- metricbeat/helper/prometheus/prometheus.go | 3 +++ x-pack/metricbeat/module/prometheus/remote_write/data.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/metricbeat/helper/prometheus/prometheus.go b/metricbeat/helper/prometheus/prometheus.go index 87e21c75083..d5a8ccb9484 100644 --- a/metricbeat/helper/prometheus/prometheus.go +++ b/metricbeat/helper/prometheus/prometheus.go @@ -286,6 +286,7 @@ func getLabels(metric *dto.Metric) common.MapStr { return labels } +// CompilePatternList compiles a patter list and returns the list of the compiled patterns func CompilePatternList(patterns *[]string) ([]*regexp.Regexp, error) { var compiledPatterns []*regexp.Regexp compiledPatterns = []*regexp.Regexp{} @@ -302,6 +303,8 @@ func CompilePatternList(patterns *[]string) ([]*regexp.Regexp, error) { return []*regexp.Regexp{}, nil } + +// MatchMetricFamily checks if the given family/metric name matches any of the given patterns func MatchMetricFamily(family string, matchMetrics []*regexp.Regexp) bool { for _, checkMetric := range matchMetrics { matched := checkMetric.MatchString(family) diff --git a/x-pack/metricbeat/module/prometheus/remote_write/data.go b/x-pack/metricbeat/module/prometheus/remote_write/data.go index 2b7fdc72541..dfbdec70398 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/data.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/data.go @@ -253,7 +253,7 @@ func (g *remoteWriteTypedGenerator) findMetricType(metricName string, labels com if strings.HasSuffix(metricName, "_total") || strings.HasSuffix(metricName, "_sum") || strings.HasSuffix(metricName, "_count") { return counterType - } else if strings.Contains(metricName, "_bucket") && leLabel { + } else if strings.HasSuffix(metricName, "_bucket") && leLabel { return histogramType } From e3fc1254fb04dc73ced80468af0b75c9408ed4a8 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 28 Jul 2020 18:53:50 +0300 Subject: [PATCH 21/24] fmt Signed-off-by: chrismark --- metricbeat/helper/prometheus/prometheus.go | 1 - 1 file changed, 1 deletion(-) diff --git a/metricbeat/helper/prometheus/prometheus.go b/metricbeat/helper/prometheus/prometheus.go index d5a8ccb9484..e6ef00e5fce 100644 --- a/metricbeat/helper/prometheus/prometheus.go +++ b/metricbeat/helper/prometheus/prometheus.go @@ -303,7 +303,6 @@ func CompilePatternList(patterns *[]string) ([]*regexp.Regexp, error) { return []*regexp.Regexp{}, nil } - // MatchMetricFamily checks if the given family/metric name matches any of the given patterns func MatchMetricFamily(family string, matchMetrics []*regexp.Regexp) bool { for _, checkMetric := range matchMetrics { From bdc443166c5388805c2ec14cc373e6c7e79bfcde Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 29 Jul 2020 11:01:49 +0300 Subject: [PATCH 22/24] review changes Signed-off-by: chrismark --- metricbeat/helper/prometheus/prometheus.go | 2 +- .../prometheus/collector/_meta/docs.asciidoc | 8 +- .../remote_write/_meta/docs.asciidoc | 14 +- .../module/prometheus/remote_write/data.go | 44 +++--- .../module/prometheus/remote_write/data.go | 131 +++++++++--------- 5 files changed, 106 insertions(+), 93 deletions(-) diff --git a/metricbeat/helper/prometheus/prometheus.go b/metricbeat/helper/prometheus/prometheus.go index e6ef00e5fce..c520460109d 100644 --- a/metricbeat/helper/prometheus/prometheus.go +++ b/metricbeat/helper/prometheus/prometheus.go @@ -286,7 +286,7 @@ func getLabels(metric *dto.Metric) common.MapStr { return labels } -// CompilePatternList compiles a patter list and returns the list of the compiled patterns +// CompilePatternList compiles a pattern list and returns the list of the compiled patterns func CompilePatternList(patterns *[]string) ([]*regexp.Regexp, error) { var compiledPatterns []*regexp.Regexp compiledPatterns = []*regexp.Regexp{} diff --git a/metricbeat/module/prometheus/collector/_meta/docs.asciidoc b/metricbeat/module/prometheus/collector/_meta/docs.asciidoc index c3609b083dd..022b6172b61 100644 --- a/metricbeat/module/prometheus/collector/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/collector/_meta/docs.asciidoc @@ -40,10 +40,10 @@ metricbeat.modules: rate_counters: false ------------------------------------------------------------------------------------- -`use_types` paramater (default: false) enables a different layout for metrics storage, leveraging Elasticsearch +`use_types` parameter (default: false) enables a different layout for metrics storage, leveraging Elasticsearch types, including https://www.elastic.co/guide/en/elasticsearch/reference/current/histogram.html[histograms]. -`rate_counters` paramater (default: false) enables calculating a rate out of Prometheus counters. When enabled, Metricbeat stores +`rate_counters` parameter (default: false) enables calculating a rate out of Prometheus counters. When enabled, Metricbeat stores the counter increment since the last collection. This metric should make some aggregations easier and with better performance. This parameter can only be enabled in combination with `use_types`. @@ -122,8 +122,8 @@ The configuration above will include only metrics that match `node_filesystem_*` To keep only specific metrics, anchor the start and the end of the regexp of each metric: -- the caret ^ matches the beginning of a text or line, -- the dollar sign $ matches the end of a text. +- the caret `^` matches the beginning of a text or line, +- the dollar sign `$` matches the end of a text. [source,yaml] ------------------------------------------------------------------------------------- diff --git a/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc b/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc index 2fef11334ed..a995c95655e 100644 --- a/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc @@ -78,10 +78,10 @@ metricbeat.modules: port: "9201" ------------------------------------------------------------------------------------- -`use_types` paramater (default: false) enables a different layout for metrics storage, leveraging Elasticsearch +`use_types` parameter (default: false) enables a different layout for metrics storage, leveraging Elasticsearch types, including https://www.elastic.co/guide/en/elasticsearch/reference/current/histogram.html[histograms]. -`rate_counters` paramater (default: false) enables calculating a rate out of Prometheus counters. When enabled, Metricbeat stores +`rate_counters` parameter (default: false) enables calculating a rate out of Prometheus counters. When enabled, Metricbeat stores the counter increment since the last collection. This metric should make some aggregations easier and with better performance. This parameter can only be enabled in combination with `use_types`. @@ -117,7 +117,7 @@ When `use_types` and `rate_counters` are enabled, metrics are stored like this: [float] ==== Types' patterns -Unline `collector` metricset, `remote_write` receives metrics in raw form from the prometheus server. +Unlike `collector` metricset, `remote_write` receives metrics in raw format from the prometheus server. In this, the module has to internally use a heuristic in order to identify efficiently the type of each raw metric. For these purpose some name patterns are used in order to identify the type of each metric. The default patterns are the following: @@ -149,8 +149,8 @@ and those that match `_my_histogram_suffix` (and have `le` in their labels) as H To match only specific metrics, anchor the start and the end of the regexp of each metric: -- the caret ^ matches the beginning of a text or line, -- the dollar sign $ matches the end of a text. +- the caret `^` matches the beginning of a text or line, +- the dollar sign `$` matches the end of a text. [source,yaml] ------------------------------------------------------------------------------------- @@ -162,3 +162,7 @@ metricbeat.modules: types_patterns: histogram_patterns: ["^my_histogram_metric$"] ------------------------------------------------------------------------------------- + +Note that when using `types_patterns`, the provided patterns have higher priority than the default patterns. +For instance if `_histogram_total` is a defined histogram pattern, then a metric like `network_bytes_histogram_total` +will be handled as a histogram even of it has the suffix `_total` which is a default pattern for counters. diff --git a/metricbeat/module/prometheus/remote_write/data.go b/metricbeat/module/prometheus/remote_write/data.go index 8381fafd19e..2eec6aefaa3 100644 --- a/metricbeat/module/prometheus/remote_write/data.go +++ b/metricbeat/module/prometheus/remote_write/data.go @@ -45,6 +45,11 @@ func (p *remoteWriteEventGenerator) GenerateEvents(metrics model.Samples) map[st if metric == nil { continue } + val := float64(metric.Value) + if math.IsNaN(val) || math.IsInf(val, 0) { + continue + } + name := string(metric.Metric["__name__"]) delete(metric.Metric, "__name__") @@ -52,31 +57,28 @@ func (p *remoteWriteEventGenerator) GenerateEvents(metrics model.Samples) map[st labels[string(k)] = v } - val := float64(metric.Value) - if !math.IsNaN(val) && !math.IsInf(val, 0) { - // join metrics with same labels in a single event - labelsHash := labels.String() - if _, ok := eventList[labelsHash]; !ok { - eventList[labelsHash] = mb.Event{ - ModuleFields: common.MapStr{ - "metrics": common.MapStr{}, - }, - } - - // Add labels - if len(labels) > 0 { - eventList[labelsHash].ModuleFields["labels"] = labels - } + // join metrics with same labels in a single event + labelsHash := labels.String() + if _, ok := eventList[labelsHash]; !ok { + eventList[labelsHash] = mb.Event{ + ModuleFields: common.MapStr{ + "metrics": common.MapStr{}, + }, } - // Not checking anything here because we create these maps some lines before - e := eventList[labelsHash] - e.Timestamp = metric.Timestamp.Time() - data := common.MapStr{ - name: val, + // Add labels + if len(labels) > 0 { + eventList[labelsHash].ModuleFields["labels"] = labels } - e.ModuleFields["metrics"].(common.MapStr).Update(data) } + + // Not checking anything here because we create these maps some lines before + e := eventList[labelsHash] + e.Timestamp = metric.Timestamp.Time() + data := common.MapStr{ + name: val, + } + e.ModuleFields["metrics"].(common.MapStr).Update(data) } return eventList diff --git a/x-pack/metricbeat/module/prometheus/remote_write/data.go b/x-pack/metricbeat/module/prometheus/remote_write/data.go index dfbdec70398..ee9cb7d3dd7 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/data.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/data.go @@ -27,7 +27,7 @@ import ( const ( counterType = "counter_type" - histogramType = "histgram_type" + histogramType = "histogram_type" otherType = "other_type" ) @@ -108,6 +108,11 @@ func (g remoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str if metric == nil { continue } + val := float64(metric.Value) + if !math.IsNaN(val) && !math.IsInf(val, 0) { + continue + } + name := string(metric.Metric["__name__"]) delete(metric.Metric, "__name__") @@ -116,71 +121,70 @@ func (g remoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str } promType := g.findMetricType(name, labels) - val := float64(metric.Value) - if !math.IsNaN(val) && !math.IsInf(val, 0) { - labelsHash := labels.String() - labelsClone := labels.Clone() - labelsClone.Delete("le") - if promType == histogramType { - labelsHash = labelsClone.String() + + labelsHash := labels.String() + labelsClone := labels.Clone() + labelsClone.Delete("le") + if promType == histogramType { + labelsHash = labelsClone.String() + } + // join metrics with same labels in a single event + if _, ok := eventList[labelsHash]; !ok { + eventList[labelsHash] = mb.Event{ + ModuleFields: common.MapStr{}, } - // join metrics with same labels in a single event - if _, ok := eventList[labelsHash]; !ok { - eventList[labelsHash] = mb.Event{ - ModuleFields: common.MapStr{}, - } - // Add labels - if len(labels) > 0 { - if promType == histogramType { - eventList[labelsHash].ModuleFields["labels"] = labelsClone - } else { - eventList[labelsHash].ModuleFields["labels"] = labels - } + // Add labels + if len(labels) > 0 { + if promType == histogramType { + eventList[labelsHash].ModuleFields["labels"] = labelsClone + } else { + eventList[labelsHash].ModuleFields["labels"] = labels } } + } - e := eventList[labelsHash] - e.Timestamp = metric.Timestamp.Time() - switch promType { - case counterType: - data = common.MapStr{ - name: g.rateCounterFloat64(name, labels, val), - } - case otherType: - data = common.MapStr{ - name: common.MapStr{ - "value": val, - }, - } - case histogramType: - histKey := name + labelsClone.String() + e := eventList[labelsHash] + e.Timestamp = metric.Timestamp.Time() + switch promType { + case counterType: + data = common.MapStr{ + name: g.rateCounterFloat64(name, labels, val), + } + case otherType: + data = common.MapStr{ + name: common.MapStr{ + "value": val, + }, + } + case histogramType: + histKey := name + labelsClone.String() - le, _ := labels.GetValue("le") - upperBound := string(le.(model.LabelValue)) + le, _ := labels.GetValue("le") + upperBound := string(le.(model.LabelValue)) - bucket, err := strconv.ParseFloat(upperBound, 64) - if err != nil { - continue - } - v := uint64(val) - b := &dto.Bucket{ - CumulativeCount: &v, - UpperBound: &bucket, - } - hist, ok := histograms[histKey] - if !ok { - hist = histogram{} - } - hist.buckets = append(hist.buckets, b) - hist.timestamp = metric.Timestamp.Time() - hist.labels = labelsClone - hist.metricName = name - histograms[histKey] = hist + bucket, err := strconv.ParseFloat(upperBound, 64) + if err != nil { continue } - e.ModuleFields.Update(data) + v := uint64(val) + b := &dto.Bucket{ + CumulativeCount: &v, + UpperBound: &bucket, + } + hist, ok := histograms[histKey] + if !ok { + hist = histogram{} + } + hist.buckets = append(hist.buckets, b) + hist.timestamp = metric.Timestamp.Time() + hist.labels = labelsClone + hist.metricName = name + histograms[histKey] = hist + continue } + e.ModuleFields.Update(data) + } // process histograms together @@ -250,12 +254,6 @@ func (g *remoteWriteTypedGenerator) findMetricType(metricName string, labels com if _, ok := labels["le"]; ok { leLabel = true } - if strings.HasSuffix(metricName, "_total") || strings.HasSuffix(metricName, "_sum") || - strings.HasSuffix(metricName, "_count") { - return counterType - } else if strings.HasSuffix(metricName, "_bucket") && leLabel { - return histogramType - } // handle user provided patterns if len(g.counterPatterns) > 0 { @@ -268,5 +266,14 @@ func (g *remoteWriteTypedGenerator) findMetricType(metricName string, labels com return histogramType } } + + // handle defaults + if strings.HasSuffix(metricName, "_total") || strings.HasSuffix(metricName, "_sum") || + strings.HasSuffix(metricName, "_count") { + return counterType + } else if strings.HasSuffix(metricName, "_bucket") && leLabel { + return histogramType + } + return otherType } From b538355fb35af58d9fecb72ab532b47e9827e4e9 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 29 Jul 2020 13:34:07 +0300 Subject: [PATCH 23/24] Add note about summary in the docs Signed-off-by: chrismark --- metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc b/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc index a995c95655e..39522afbd13 100644 --- a/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/remote_write/_meta/docs.asciidoc @@ -127,7 +127,8 @@ The default patterns are the following: . `_count` suffix: the metric is of Counter type . `_bucket` suffix and `le` in labels: the metric is of Histogram type -Everything else is handled as a Gauge. +Everything else is handled as a Gauge. In addition there is no special handling for Summaries so it is expected that +Summary's quantiles are handled as Gauges and Summary's sum and count as Counters. Users have the flexibility to add their own patterns using the following configuration: From c3712e43099fb3005bffa87cc658d8ea57f84727 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 29 Jul 2020 13:37:23 +0300 Subject: [PATCH 24/24] fix Signed-off-by: chrismark --- x-pack/metricbeat/module/prometheus/remote_write/data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/prometheus/remote_write/data.go b/x-pack/metricbeat/module/prometheus/remote_write/data.go index ee9cb7d3dd7..5d8a101fbdd 100644 --- a/x-pack/metricbeat/module/prometheus/remote_write/data.go +++ b/x-pack/metricbeat/module/prometheus/remote_write/data.go @@ -109,7 +109,7 @@ func (g remoteWriteTypedGenerator) GenerateEvents(metrics model.Samples) map[str continue } val := float64(metric.Value) - if !math.IsNaN(val) && !math.IsInf(val, 0) { + if math.IsNaN(val) || math.IsInf(val, 0) { continue }