diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5928ab7664b4..ee6ce234cf93 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -69,6 +69,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Metricbeat* +- Refactor Prometheus metric mappings {pull}9948[9948] +- Removed Prometheus stats metricset in favor of just using Prometheus collector {pull}9948[9948] + *Packetbeat* - Adjust Packetbeat `http` fields to ECS Beta 2 {pull}9645[9645] diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index c6898a81b65a..256523cd23b5 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -19358,66 +19358,26 @@ alias to: process.executable [[exported-fields-prometheus]] == Prometheus fields -Stats collected from Prometheus. +Stats scraped from a Prometheus endpoint. -[float] -== prometheus fields - - - - -[float] -== stats fields - -Stats about the Prometheus server. - - - -[float] -== notifications fields - -Notification stats. - - - -*`prometheus.stats.notifications.queue_length`*:: -+ --- -type: long - -Current queue length. - - --- - -*`prometheus.stats.notifications.dropped`*:: -+ --- -type: long - -Number of dropped queue events. - - --- - -*`prometheus.stats.processes.open_fds`*:: +*`prometheus.labels`*:: + -- -type: long +type: object -Number of open file descriptors. +Prometheus metric labels -- -*`prometheus.stats.storage.chunks_to_persist`*:: +*`prometheus.metrics`*:: + -- -type: long +type: object -Number of memory chunks that are not yet persisted to disk. +Prometheus metric - release: ga -- diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc index 467e68851304..45a7f37fe586 100644 --- a/metricbeat/docs/modules/prometheus.asciidoc +++ b/metricbeat/docs/modules/prometheus.asciidoc @@ -5,10 +5,8 @@ This file is generated! See scripts/docs_collector.py [[metricbeat-module-prometheus]] == Prometheus module -This module periodically fetches metrics from -https://prometheus.io/docs/[Prometheus]. - -The default metricset is `collector`. +This module periodically scrapes metrics from +https://prometheus.io/docs/instrumenting/exporters/[Prometheus exporters]. [float] @@ -21,20 +19,11 @@ in <>. Here is an example configuration: ---- metricbeat.modules: - module: prometheus - metricsets: ["stats"] - enabled: true period: 10s hosts: ["localhost:9090"] - #metrics_path: /metrics - #namespace: example - -- module: prometheus - metricsets: ["collector"] - enabled: true - period: 10s - hosts: ["localhost:9090"] - #metrics_path: /metrics - #namespace: example + metrics_path: /metrics + #username: "user" + #password: "secret" # This can be used for service account based authorization: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token @@ -52,9 +41,5 @@ The following metricsets are available: * <> -* <> - include::prometheus/collector.asciidoc[] -include::prometheus/stats.asciidoc[] - diff --git a/metricbeat/docs/modules/prometheus/stats.asciidoc b/metricbeat/docs/modules/prometheus/stats.asciidoc deleted file mode 100644 index 4332b1b9dd2a..000000000000 --- a/metricbeat/docs/modules/prometheus/stats.asciidoc +++ /dev/null @@ -1,23 +0,0 @@ -//// -This file is generated! See scripts/docs_collector.py -//// - -[[metricbeat-metricset-prometheus-stats]] -=== Prometheus stats metricset - -beta[] - -include::../../../module/prometheus/stats/_meta/docs.asciidoc[] - - -==== Fields - -For a description of each field in the metricset, see the -<> section. - -Here is an example document generated by this metricset: - -[source,json] ----- -include::../../../module/prometheus/stats/_meta/data.json[] ----- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 6e0d3121cab0..e1757c12e009 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -117,8 +117,7 @@ This file is generated! See scripts/docs_collector.py |<> |<> |<> |image:./images/icon-no.png[No prebuilt dashboards] | -.2+| .2+| |<> -|<> beta[] +.1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .4+| .4+| |<> |<> diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 61a10189b7dc..0a873f9080f1 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -140,7 +140,6 @@ import ( _ "github.com/elastic/beats/metricbeat/module/postgresql/statement" _ "github.com/elastic/beats/metricbeat/module/prometheus" _ "github.com/elastic/beats/metricbeat/module/prometheus/collector" - _ "github.com/elastic/beats/metricbeat/module/prometheus/stats" _ "github.com/elastic/beats/metricbeat/module/rabbitmq" _ "github.com/elastic/beats/metricbeat/module/rabbitmq/connection" _ "github.com/elastic/beats/metricbeat/module/rabbitmq/exchange" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index e3c4f328c9cf..6c89c863f81d 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -585,20 +585,11 @@ metricbeat.modules: #----------------------------- Prometheus Module ----------------------------- - module: prometheus - metricsets: ["stats"] - enabled: true - period: 10s - hosts: ["localhost:9090"] - #metrics_path: /metrics - #namespace: example - -- module: prometheus - metricsets: ["collector"] - enabled: true period: 10s hosts: ["localhost:9090"] - #metrics_path: /metrics - #namespace: example + metrics_path: /metrics + #username: "user" + #password: "secret" # This can be used for service account based authorization: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token diff --git a/metricbeat/module/prometheus/_meta/Dockerfile b/metricbeat/module/prometheus/_meta/Dockerfile index 836c8777f629..55b29cbbb27b 100644 --- a/metricbeat/module/prometheus/_meta/Dockerfile +++ b/metricbeat/module/prometheus/_meta/Dockerfile @@ -1,3 +1,3 @@ -FROM prom/prometheus:v1.5.1 +FROM prom/prometheus:v2.6.0 HEALTHCHECK --interval=1s --retries=90 CMD nc -w 1 localhost 9090 - Stats collected from Prometheus. + Stats scraped from a Prometheus endpoint. short_config: false release: ga settings: ["ssl", "http"] fields: - - name: prometheus - type: group + # Order is important here, labels will match first, the rest are double + - name: prometheus.labels + type: object + object_type: keyword description: > - fields: + Prometheus metric labels + - name: prometheus.metrics + type: object + object_type: double + object_type_mapping_type: "*" + description: > + Prometheus metric \ No newline at end of file diff --git a/metricbeat/module/prometheus/collector/_meta/data.json b/metricbeat/module/prometheus/collector/_meta/data.json index cbb77a3b3fad..6f63e95343c6 100644 --- a/metricbeat/module/prometheus/collector/_meta/data.json +++ b/metricbeat/module/prometheus/collector/_meta/data.json @@ -1,25 +1,28 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { + "agent": { "hostname": "host.example.com", "name": "host.example.com" }, + "event": { + "dataset": "prometheus.collector", + "duration": 115000, + "module": "prometheus" + }, "metricset": { - "host": "prometheus:9090", - "module": "prometheus", - "name": "collector", - "namespace": "collector", - "rtt": 115 + "name": "collector" }, "prometheus": { - "collector": { - "label": { - "event": "add", - "role": "node" - }, - "prometheus_sd_kubernetes_events_total": { - "value": 0 - } + "labels": { + "interval": "15s" + }, + "metrics": { + "prometheus_target_interval_length_seconds_count": 1, + "prometheus_target_interval_length_seconds_sum": 15.000226176 } + }, + "service": { + "address": "172.29.0.2:9090", + "type": "prometheus" } } \ No newline at end of file diff --git a/metricbeat/module/prometheus/collector/_meta/docs.asciidoc b/metricbeat/module/prometheus/collector/_meta/docs.asciidoc index 8dbcf97d0497..97b6126d0efb 100644 --- a/metricbeat/module/prometheus/collector/_meta/docs.asciidoc +++ b/metricbeat/module/prometheus/collector/_meta/docs.asciidoc @@ -1,5 +1,43 @@ -The Prometheus `collector` metricset fetches data from https://prometheus.io/docs/instrumenting/exporters/[prometheus exporters]. +The Prometheus `collector` metricset scrapes data from https://prometheus.io/docs/instrumenting/exporters/[prometheus exporters]. -All events with the same labels are grouped together as one event. The fields -exported by this metricset vary depending on the Prometheus exporter that you're -using. + +[float] +=== Scraping from a Prometheus exporter + +To scrape metrics from a Prometheus exporter, configure the `hosts` field to it. The path +to retrieve the metrics from (`/metrics` by default) can be configured with `metrics_path`. + +[source,yaml] +------------------------------------------------------------------------------------- +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + metrics_path: /metrics + + #username: "user" + #password: "secret" + + # This can be used for service account based authorization: + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt +------------------------------------------------------------------------------------- + + +[float] +=== Scraping all metrics from a Prometheus server + +This module can scrape all metrics stored in a Prometheus server, by using the +https://prometheus.io/docs/prometheus/latest/federation/[federation API]. By pointing this +config to the Prometheus server: + +[source,yaml] +------------------------------------------------------------------------------------- +metricbeat.modules: +- module: prometheus + period: 10s + hosts: ["localhost:9090"] + metrics_path: '/federate' + query: + 'match[]': '{__name__!=""}' +------------------------------------------------------------------------------------- \ No newline at end of file diff --git a/metricbeat/module/prometheus/collector/collector.go b/metricbeat/module/prometheus/collector/collector.go index 72493328c30b..e16bdf6cc531 100644 --- a/metricbeat/module/prometheus/collector/collector.go +++ b/metricbeat/module/prometheus/collector/collector.go @@ -21,7 +21,6 @@ import ( "fmt" "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/common/cfgwarn" p "github.com/elastic/beats/metricbeat/helper/prometheus" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" @@ -50,20 +49,9 @@ func init() { type MetricSet struct { mb.BaseMetricSet prometheus p.Prometheus - namespace string } func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - cfgwarn.Beta("The prometheus collector metricset is beta") - - config := struct { - Namespace string `config:"namespace" validate:"required"` - }{} - err := base.Module().UnpackConfig(&config) - if err != nil { - return nil, err - } - prometheus, err := p.NewPrometheusClient(base) if err != nil { return nil, err @@ -72,42 +60,41 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return &MetricSet{ BaseMetricSet: base, prometheus: prometheus, - namespace: config.Namespace, }, nil } -func (m *MetricSet) Fetch() ([]common.MapStr, error) { +func (m *MetricSet) Fetch(reporter mb.ReporterV2) { families, err := m.prometheus.GetFamilies() if err != nil { - return nil, fmt.Errorf("Unable to decode response from prometheus endpoint") + reporter.Error(fmt.Errorf("Unable to decode response from prometheus endpoint")) + return } eventList := map[string]common.MapStr{} for _, family := range families { - promEvents := GetPromEventsFromMetricFamily(family) + promEvents := getPromEventsFromMetricFamily(family) for _, promEvent := range promEvents { - if _, ok := eventList[promEvent.labelHash]; !ok { - eventList[promEvent.labelHash] = common.MapStr{} + labelsHash := promEvent.LabelsHash() + if _, ok := eventList[labelsHash]; !ok { + eventList[labelsHash] = common.MapStr{} // Add labels if len(promEvent.labels) > 0 { - eventList[promEvent.labelHash]["label"] = promEvent.labels + eventList[labelsHash]["labels"] = promEvent.labels } } - eventList[promEvent.labelHash][promEvent.key] = promEvent.value + eventList[labelsHash].Update(common.MapStr{ + "metrics": promEvent.data, + }) } } // Converts hash list to slice - events := []common.MapStr{} for _, e := range eventList { - e[mb.NamespaceKey] = m.namespace - events = append(events, e) + reporter.Event(mb.Event{ModuleFields: e}) } - - return events, err } diff --git a/metricbeat/module/prometheus/collector/collector_integration_test.go b/metricbeat/module/prometheus/collector/collector_integration_test.go index c09ac37f27a9..98d9169ae1d0 100644 --- a/metricbeat/module/prometheus/collector/collector_integration_test.go +++ b/metricbeat/module/prometheus/collector/collector_integration_test.go @@ -25,33 +25,15 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" - - "github.com/stretchr/testify/assert" ) -// These tests are running with prometheus metrics as an example as this container is already available -// Every prometheus exporter should work here. - -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "prometheus") - - f := mbtest.NewEventsFetcher(t, getConfig()) - event, err := f.Fetch() - if !assert.NoError(t, err) { - t.FailNow() - } - - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) -} - func TestData(t *testing.T) { compose.EnsureUp(t, "prometheus") - f := mbtest.NewEventsFetcher(t, getConfig()) - - err := mbtest.WriteEvents(f, t) + ms := mbtest.NewReportingMetricSetV2(t, getConfig()) + err := mbtest.WriteEventsReporterV2(ms, t, "") if err != nil { - t.Fatal("write", err) + t.Fatal(err) } } diff --git a/metricbeat/module/prometheus/collector/collector_test.go b/metricbeat/module/prometheus/collector/collector_test.go index 9e56dd058abb..362ae0a137ba 100644 --- a/metricbeat/module/prometheus/collector/collector_test.go +++ b/metricbeat/module/prometheus/collector/collector_test.go @@ -35,7 +35,7 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { } tests := []struct { Family *dto.MetricFamily - Event PromEvent + Event []PromEvent }{ { Family: &dto.MetricFamily{ @@ -56,13 +56,13 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { }, }, }, - Event: PromEvent{ - key: "http_request_duration_microseconds", - value: common.MapStr{ - "value": int64(10), + Event: []PromEvent{ + { + data: common.MapStr{ + "http_request_duration_microseconds": float64(10), + }, + labels: labels, }, - labelHash: labels.String(), - labels: labels, }, }, { @@ -78,12 +78,13 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { }, }, }, - Event: PromEvent{ - key: "http_request_duration_microseconds", - value: common.MapStr{ - "value": float64(10), + Event: []PromEvent{ + { + data: common.MapStr{ + "http_request_duration_microseconds": float64(10), + }, + labels: common.MapStr{}, }, - labelHash: "#", }, }, { @@ -106,16 +107,22 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { }, }, }, - Event: PromEvent{ - key: "http_request_duration_microseconds", - value: common.MapStr{ - "count": uint64(10), - "sum": float64(10), - "percentile": common.MapStr{ - "99": float64(10), + Event: []PromEvent{ + { + data: common.MapStr{ + "http_request_duration_microseconds_count": uint64(10), + "http_request_duration_microseconds_sum": float64(10), + }, + labels: common.MapStr{}, + }, + { + data: common.MapStr{ + "http_request_duration_microseconds": float64(10), + }, + labels: common.MapStr{ + "quantile": "0.99", }, }, - labelHash: "#", }, }, { @@ -138,16 +145,20 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { }, }, }, - Event: PromEvent{ - key: "http_request_duration_microseconds", - value: common.MapStr{ - "count": uint64(10), - "sum": float64(10), - "bucket": common.MapStr{ - "0.99": uint64(10), + Event: []PromEvent{ + { + data: common.MapStr{ + "http_request_duration_microseconds_count": uint64(10), + "http_request_duration_microseconds_sum": float64(10), + }, + labels: common.MapStr{}, + }, + { + data: common.MapStr{ + "http_request_duration_microseconds": uint64(10), }, + labels: common.MapStr{"le": "0.99"}, }, - labelHash: "#", }, }, { @@ -169,20 +180,19 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) { }, }, }, - Event: PromEvent{ - key: "http_request_duration_microseconds", - value: common.MapStr{ - "value": float64(10), + Event: []PromEvent{ + { + data: common.MapStr{ + "http_request_duration_microseconds": float64(10), + }, + labels: labels, }, - labelHash: labels.String(), - labels: labels, }, }, } for _, test := range tests { - event := GetPromEventsFromMetricFamily(test.Family) - assert.Equal(t, len(event), 1) - assert.Equal(t, event[0], test.Event) + event := getPromEventsFromMetricFamily(test.Family) + assert.Equal(t, test.Event, event) } } diff --git a/metricbeat/module/prometheus/collector/data.go b/metricbeat/module/prometheus/collector/data.go index 487782500233..0120aa55db30 100644 --- a/metricbeat/module/prometheus/collector/data.go +++ b/metricbeat/module/prometheus/collector/data.go @@ -26,93 +26,111 @@ import ( dto "github.com/prometheus/client_model/go" ) +// PromEvent stores a set of one or more metrics with the same labels type PromEvent struct { - key string - value common.MapStr - labels common.MapStr - labelHash string + data common.MapStr + labels common.MapStr } -func GetPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { +// LabelsHash returns a repeatable string that is unique for the set of labels in this event +func (p *PromEvent) LabelsHash() string { + return p.labels.String() +} + +func getPromEventsFromMetricFamily(mf *dto.MetricFamily) []PromEvent { var events []PromEvent name := *mf.Name metrics := mf.Metric for _, metric := range metrics { - event := PromEvent{ - key: name, - labelHash: "#", - } - value := common.MapStr{} - labels := metric.Label + labels := common.MapStr{} - if len(labels) != 0 { - tagsMap := common.MapStr{} - for _, label := range labels { + if len(metric.Label) != 0 { + for _, label := range metric.Label { if label.GetName() != "" && label.GetValue() != "" { - tagsMap[label.GetName()] = label.GetValue() + labels[label.GetName()] = label.GetValue() } } - event.labels = tagsMap - event.labelHash = tagsMap.String() - } counter := metric.GetCounter() if counter != nil { - value["value"] = int64(counter.GetValue()) + events = append(events, PromEvent{ + data: common.MapStr{ + name: counter.GetValue(), + }, + labels: labels, + }) } gauge := metric.GetGauge() if gauge != nil { - value["value"] = gauge.GetValue() + events = append(events, PromEvent{ + data: common.MapStr{ + name: gauge.GetValue(), + }, + labels: labels, + }) } summary := metric.GetSummary() if summary != nil { - value["sum"] = summary.GetSampleSum() - value["count"] = summary.GetSampleCount() - - quantiles := summary.GetQuantile() - - percentileMap := common.MapStr{} - for _, quantile := range quantiles { - key := strconv.FormatFloat((100 * quantile.GetQuantile()), 'f', -1, 64) - - if math.IsNaN(quantile.GetValue()) == false { - percentileMap[key] = quantile.GetValue() + events = append(events, PromEvent{ + data: common.MapStr{ + name + "_sum": summary.GetSampleSum(), + name + "_count": summary.GetSampleCount(), + }, + labels: labels, + }) + + for _, quantile := range summary.GetQuantile() { + if math.IsNaN(quantile.GetValue()) { + continue } - } - - if len(percentileMap) != 0 { - value["percentile"] = percentileMap + quantileLabels := labels.Clone() + quantileLabels["quantile"] = strconv.FormatFloat(quantile.GetQuantile(), 'f', -1, 64) + events = append(events, PromEvent{ + data: common.MapStr{ + name: quantile.GetValue(), + }, + labels: quantileLabels, + }) } } histogram := metric.GetHistogram() if histogram != nil { - value["sum"] = histogram.GetSampleSum() - value["count"] = histogram.GetSampleCount() - buckets := histogram.GetBucket() - bucketMap := common.MapStr{} - for _, bucket := range buckets { - key := strconv.FormatFloat(bucket.GetUpperBound(), 'f', -1, 64) - bucketMap[key] = bucket.GetCumulativeCount() + events = append(events, PromEvent{ + data: common.MapStr{ + name + "_sum": histogram.GetSampleSum(), + name + "_count": histogram.GetSampleCount(), + }, + labels: labels, + }) + + for _, bucket := range histogram.GetBucket() { + bucketLabels := labels.Clone() + bucketLabels["le"] = strconv.FormatFloat(bucket.GetUpperBound(), 'f', -1, 64) + + events = append(events, PromEvent{ + data: common.MapStr{ + name: bucket.GetCumulativeCount(), + }, + labels: bucketLabels, + }) } - - value["bucket"] = bucketMap } untyped := metric.GetUntyped() if untyped != nil { - value["value"] = untyped.GetValue() + events = append(events, PromEvent{ + data: common.MapStr{ + name: untyped.GetValue(), + }, + labels: labels, + }) } - - event.value = value - - events = append(events, event) - } return events } diff --git a/metricbeat/module/prometheus/fields.go b/metricbeat/module/prometheus/fields.go index d119a4bbb863..d4c3f017f049 100644 --- a/metricbeat/module/prometheus/fields.go +++ b/metricbeat/module/prometheus/fields.go @@ -32,5 +32,5 @@ func init() { // AssetPrometheus returns asset data. // This is the base64 encoded gzipped contents of ../metricbeat/module/prometheus. func AssetPrometheus() string { - return "eJy0krGum0AQRXu+4oo6jw+gSJP+KVLKKEJruMDKyw7ZGZ7kv48INsYEy03elMPsPYfZfcOZlxJjkoHWc9IMMG+BJfLvazPPgIZaJz+al1jiawYAP8yZopYQWBsbtEkG3E8VGaC9JKtqia3vSrQuKDMgMdApS3RunqGZj52W+JmrhvwL8t5szH9lQOsZGi3/4t4Q3cCd7Fx2GeesJNN47RzIPmYteY8et+5C0fnn1u4R5CloqWU77iSTwXpuFgNl+mAqNuOryYnmNv2989YwivnW126G68PEM9sXxnO9b0KXHRS7mSOlrdbviROrwNhZ/8/QzSxI7A4+vpCb69uUEqMtGCyYveJdpkkyjmw+weN9Gk5MkPbGuBrxg3G/tM3LralKLWRkrNrm+NoOpF7d2iozB6P1gesRSU9s1CS5jkXdT/GslUk1MqlX++9SAwdJFywgWO8MLnF+wLjQcMWygQkar+ci+xMAAP//EEQ9qA==" + return "eJyUkU1u6zAMhPc+xUBv95DkAFr0Ci3QZVEEikXbbPQHkkGQ2xeJjdQBsmi1E78RORpucaSLR5OaySY6aQcYWyIP93Yvug6IpL1wM67F46UDgHcLptBeQqOIQWpGwM8rUImtcrFdB+hUxfZ9LQOPHkNISh0glCgoeYzhqiEzLqN6fDjV5DZwk1lznx0wMKWo/jb3H14lkoAVnFsVC8UwkdAGKRwoKc6cEnKwfsLAoraBTQQhNQQhxHo6JLr12qKETOsEdnOPGwXs0sijHr6ot6U0X/YzOdLlXCUu6ElI17PKJJMJ91jNeOJgFv3Wwuo3D2SfQ2tcxkXm/rs/uryT7cOivgMAAP//FvGzhQ==" } diff --git a/metricbeat/module/prometheus/stats/_meta/data.json b/metricbeat/module/prometheus/stats/_meta/data.json deleted file mode 100644 index 9e8d67236911..000000000000 --- a/metricbeat/module/prometheus/stats/_meta/data.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" - }, - "metricset": { - "host": "prometheus:9090", - "module": "prometheus", - "name": "stats", - "rtt": 115 - }, - "prometheus": { - "stats": { - "notifications": { - "dropped": 0, - "queue_length": 0 - }, - "processes": { - "open_fds": 25 - }, - "storage": { - "chunks_to_persist": 0 - } - } - } -} \ No newline at end of file diff --git a/metricbeat/module/prometheus/stats/_meta/docs.asciidoc b/metricbeat/module/prometheus/stats/_meta/docs.asciidoc deleted file mode 100644 index 0f900946053d..000000000000 --- a/metricbeat/module/prometheus/stats/_meta/docs.asciidoc +++ /dev/null @@ -1,2 +0,0 @@ -The Prometheus `stats` metricset collects statistics about the Prometheus -server. diff --git a/metricbeat/module/prometheus/stats/_meta/fields.yml b/metricbeat/module/prometheus/stats/_meta/fields.yml deleted file mode 100644 index 4680a64d976d..000000000000 --- a/metricbeat/module/prometheus/stats/_meta/fields.yml +++ /dev/null @@ -1,27 +0,0 @@ -- name: stats - type: group - description: > - Stats about the Prometheus server. - release: beta - fields: - - name: notifications - type: group - description: > - Notification stats. - fields: - - name: queue_length - type: long - description: > - Current queue length. - - name: dropped - type: long - description: > - Number of dropped queue events. - - name: processes.open_fds - type: long - description: > - Number of open file descriptors. - - name: storage.chunks_to_persist - type: long - description: > - Number of memory chunks that are not yet persisted to disk. diff --git a/metricbeat/module/prometheus/stats/data.go b/metricbeat/module/prometheus/stats/data.go deleted file mode 100644 index c1b379ce1f21..000000000000 --- a/metricbeat/module/prometheus/stats/data.go +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package stats - -import ( - "github.com/elastic/beats/libbeat/common" - s "github.com/elastic/beats/libbeat/common/schema" - c "github.com/elastic/beats/libbeat/common/schema/mapstrstr" -) - -var ( - schema = s.Schema{ - "notifications": s.Object{ - "queue_length": c.Int("prometheus_notifications_queue_length"), - "dropped": c.Int("prometheus_notifications_dropped_total"), - }, - "processes": s.Object{ - "open_fds": c.Int("process_open_fds"), - }, - "storage": s.Object{ - "chunks_to_persist": c.Int("prometheus_local_storage_chunks_to_persist"), - }, - } -) - -func eventMapping(entries map[string]interface{}) (common.MapStr, error) { - data, _ := schema.Apply(entries) - return data, nil -} diff --git a/metricbeat/module/prometheus/stats/stats.go b/metricbeat/module/prometheus/stats/stats.go deleted file mode 100644 index 05c329d89e14..000000000000 --- a/metricbeat/module/prometheus/stats/stats.go +++ /dev/null @@ -1,92 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package stats - -import ( - "strings" - - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/common/cfgwarn" - "github.com/elastic/beats/metricbeat/helper" - "github.com/elastic/beats/metricbeat/mb" - "github.com/elastic/beats/metricbeat/mb/parse" -) - -const ( - defaultScheme = "http" - defaultPath = "/metrics" -) - -var ( - hostParser = parse.URLHostParserBuilder{ - DefaultScheme: defaultScheme, - DefaultPath: defaultPath, - }.Build() -) - -func init() { - mb.Registry.MustAddMetricSet("prometheus", "stats", New, - mb.WithHostParser(hostParser), - ) -} - -type MetricSet struct { - mb.BaseMetricSet - http *helper.HTTP -} - -func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - cfgwarn.Beta("The prometheus stats metricset is beta") - - http, err := helper.NewHTTP(base) - if err != nil { - return nil, err - } - return &MetricSet{ - BaseMetricSet: base, - http: http, - }, nil -} - -func (m *MetricSet) Fetch() (common.MapStr, error) { - scanner, err := m.http.FetchScanner() - if err != nil { - return nil, err - } - - entries := map[string]interface{}{} - - // Iterate through all events to gather data - for scanner.Scan() { - line := scanner.Text() - - // Skip comments and calculated lines - if line[0] == '#' || strings.Contains(line, "quantile=") { - continue - } - - splitPos := strings.LastIndex(line, " ") - split := []string{line[:splitPos], line[splitPos+1:]} - - entries[split[0]] = split[1] - } - - data, err := eventMapping(entries) - - return data, err -} diff --git a/metricbeat/module/prometheus/stats/stats_integration_test.go b/metricbeat/module/prometheus/stats/stats_integration_test.go deleted file mode 100644 index 13154a351b6f..000000000000 --- a/metricbeat/module/prometheus/stats/stats_integration_test.go +++ /dev/null @@ -1,84 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build integration - -package stats - -import ( - "os" - "testing" - - "github.com/elastic/beats/libbeat/common" - "github.com/elastic/beats/libbeat/tests/compose" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" - - "github.com/stretchr/testify/assert" -) - -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "prometheus") - - f := mbtest.NewEventFetcher(t, getConfig()) - event, err := f.Fetch() - if !assert.NoError(t, err) { - t.FailNow() - } - - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) - - // Check number of fields. - assert.Equal(t, 3, len(event)) - assert.True(t, event["processes"].(common.MapStr)["open_fds"].(int64) > 0) -} - -func TestData(t *testing.T) { - compose.EnsureUp(t, "prometheus") - - f := mbtest.NewEventFetcher(t, getConfig()) - - err := mbtest.WriteEvent(f, t) - if err != nil { - t.Fatal("write", err) - } -} - -func getConfig() map[string]interface{} { - return map[string]interface{}{ - "module": "prometheus", - "metricsets": []string{"stats"}, - "hosts": []string{getPrometheusEnvHost() + ":" + getPrometheusEnvPort()}, - } -} - -func getPrometheusEnvHost() string { - host := os.Getenv("PROMETHEUS_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -func getPrometheusEnvPort() string { - port := os.Getenv("PROMETHEUS_PORT") - - if len(port) == 0 { - port = "9090" - } - return port -} diff --git a/metricbeat/modules.d/prometheus.yml.disabled b/metricbeat/modules.d/prometheus.yml.disabled index c13619278c27..f5dfb80f21ed 100644 --- a/metricbeat/modules.d/prometheus.yml.disabled +++ b/metricbeat/modules.d/prometheus.yml.disabled @@ -2,12 +2,9 @@ # Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-prometheus.html - module: prometheus - #metricsets: - # - stats period: 10s hosts: ["localhost:9090"] metrics_path: /metrics - #namespace: example #username: "user" #password: "secret" diff --git a/metricbeat/tests/system/test_prometheus.py b/metricbeat/tests/system/test_prometheus.py index 535395460df8..07fec23705e0 100644 --- a/metricbeat/tests/system/test_prometheus.py +++ b/metricbeat/tests/system/test_prometheus.py @@ -16,7 +16,7 @@ def test_stats(self): """ self.render_config_template(modules=[{ "name": "prometheus", - "metricsets": ["stats"], + "metricsets": ["collector"], "hosts": self.get_hosts(), "period": "5s" }]) @@ -26,7 +26,6 @@ def test_stats(self): self.assert_no_logged_warnings() output = self.read_output_json() - self.assertEqual(len(output), 1) evt = output[0] self.assertItemsEqual(self.de_dot(PROMETHEUS_FIELDS), evt.keys(), evt) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 39ada9fd1734..c9fd68bc2baf 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -603,20 +603,11 @@ metricbeat.modules: #------------------------------ Prometheus Module ------------------------------ - module: prometheus - metricsets: ["stats"] - enabled: true - period: 10s - hosts: ["localhost:9090"] - #metrics_path: /metrics - #namespace: example - -- module: prometheus - metricsets: ["collector"] - enabled: true period: 10s hosts: ["localhost:9090"] - #metrics_path: /metrics - #namespace: example + metrics_path: /metrics + #username: "user" + #password: "secret" # This can be used for service account based authorization: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token