Refactor Prometheus metric mappings

metricbeat/docs/fields.asciidoc

@@ -18270,66 +18270,26 @@ Total number of temp block cache written by the query.
 [[exported-fields-prometheus]]
 == Prometheus fields
 
-Stats collected from Prometheus.
+Stats scraped from a Prometheus endpoint.
 
 
 
-[float]
-== prometheus fields
-
-
-
-
-[float]
-== stats fields
-
-Stats about the Prometheus server.
-
-
-
-[float]
-== notifications fields
-
-Notification stats.
-
-
-
-*`prometheus.stats.notifications.queue_length`*::
-+
---
-type: long
-
-Current queue length.
-
-
---
-
-*`prometheus.stats.notifications.dropped`*::
-+
---
-type: long
-
-Number of dropped queue events.
-
-
---
-
-*`prometheus.stats.processes.open_fds`*::
+*`prometheus.labels.*`*::
 +
 --
-type: long
+type: object
 
-Number of open file descriptors.
+Prometheus metric labels
 
 
 --
 
-*`prometheus.stats.storage.chunks_to_persist`*::
+*`prometheus.*`*::
 +
 --
-type: long
+type: object
 
-Number of memory chunks that are not yet persisted to disk.
+Prometheus metric - release: ga
 
 
 --

metricbeat/docs/modules/prometheus.asciidoc

@@ -52,9 +52,5 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-prometheus-collector,collector>>
 
-* <<metricbeat-metricset-prometheus-stats,stats>>
-
 include::prometheus/collector.asciidoc[]
 
-include::prometheus/stats.asciidoc[]
-

metricbeat/docs/modules_list.asciidoc

@@ -111,8 +111,7 @@ This file is generated! See scripts/docs_collector.py
 |<<metricbeat-metricset-postgresql-database,database>>   
 |<<metricbeat-metricset-postgresql-statement,statement>> beta[]  
 |<<metricbeat-module-prometheus,Prometheus>>     |image:./images/icon-no.png[No prebuilt dashboards]    |  
-.2+| .2+|  |<<metricbeat-metricset-prometheus-collector,collector>>   
-|<<metricbeat-metricset-prometheus-stats,stats>> beta[]  
+.1+| .1+|  |<<metricbeat-metricset-prometheus-collector,collector>>   
 |<<metricbeat-module-rabbitmq,RabbitMQ>>  beta[]   |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .4+| .4+|  |<<metricbeat-metricset-rabbitmq-connection,connection>> beta[]  
 |<<metricbeat-metricset-rabbitmq-exchange,exchange>> beta[]  

metricbeat/module/prometheus/_meta/config.yml

@@ -4,7 +4,6 @@
   period: 10s
   hosts: ["localhost:9090"]
   metrics_path: /metrics
-  #namespace: example
   #username: "user"
   #password: "secret"
 

metricbeat/module/prometheus/_meta/fields.yml

@@ -1,12 +1,20 @@
 - key: prometheus
   title: "Prometheus"
   description: >
-    Stats collected from Prometheus.
+    Stats scraped from a Prometheus endpoint.
   short_config: false
   release: ga
   settings: ["ssl", "http"]
   fields:
-    - name: prometheus
-      type: group
+    # Order is important here, labels will match first, the rest are double
+    - name: prometheus.labels.*
+      type: object
+      object_type: keyword
       description: >
-      fields:
+        Prometheus metric labels
+    - name: prometheus.*
+      type: object
+      object_type: double
+      object_type_mapping_type: "*"
+      description: >
+        Prometheus metric

metricbeat/module/prometheus/collector/collector.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 
 	"github.com/elastic/beats/libbeat/common"
-	"github.com/elastic/beats/libbeat/common/cfgwarn"
 	p "github.com/elastic/beats/metricbeat/helper/prometheus"
 	"github.com/elastic/beats/metricbeat/mb"
 	"github.com/elastic/beats/metricbeat/mb/parse"
@@ -50,20 +49,9 @@ func init() {
 type MetricSet struct {
 	mb.BaseMetricSet
 	prometheus p.Prometheus
-	namespace  string
 }
 
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
-	cfgwarn.Beta("The prometheus collector metricset is beta")
-
-	config := struct {
-		Namespace string `config:"namespace" validate:"required"`
-	}{}
-	err := base.Module().UnpackConfig(&config)
-	if err != nil {
-		return nil, err
-	}
-
 	prometheus, err := p.NewPrometheusClient(base)
 	if err != nil {
 		return nil, err
@@ -72,42 +60,39 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	return &MetricSet{
 		BaseMetricSet: base,
 		prometheus:    prometheus,
-		namespace:     config.Namespace,
 	}, nil
 }
 
-func (m *MetricSet) Fetch() ([]common.MapStr, error) {
+func (m *MetricSet) Fetch(reporter mb.ReporterV2) {
 	families, err := m.prometheus.GetFamilies()
 
 	if err != nil {
-		return nil, fmt.Errorf("Unable to decode response from prometheus endpoint")
+		reporter.Error(fmt.Errorf("Unable to decode response from prometheus endpoint"))
+		return
 	}
 
 	eventList := map[string]common.MapStr{}
 
 	for _, family := range families {
-		promEvents := GetPromEventsFromMetricFamily(family)
+		promEvents := getPromEventsFromMetricFamily(family)
 
 		for _, promEvent := range promEvents {
-			if _, ok := eventList[promEvent.labelHash]; !ok {
-				eventList[promEvent.labelHash] = common.MapStr{}
+			labelsHash := promEvent.LabelsHash()
+			if _, ok := eventList[labelsHash]; !ok {
+				eventList[labelsHash] = common.MapStr{}
 
 				// Add labels
 				if len(promEvent.labels) > 0 {
-					eventList[promEvent.labelHash]["label"] = promEvent.labels
+					eventList[labelsHash]["labels"] = promEvent.labels
 				}
 			}
 
-			eventList[promEvent.labelHash][promEvent.key] = promEvent.value
+			eventList[labelsHash].Update(promEvent.data)
 		}
 	}
 
 	// Converts hash list to slice
-	events := []common.MapStr{}
 	for _, e := range eventList {
-		e[mb.NamespaceKey] = m.namespace
-		events = append(events, e)
+		reporter.Event(mb.Event{ModuleFields: e})
 	}
-
-	return events, err
 }

metricbeat/module/prometheus/collector/collector_test.go

@@ -35,7 +35,7 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) {
 	}
 	tests := []struct {
 		Family *dto.MetricFamily
-		Event  PromEvent
+		Event  []PromEvent
 	}{
 		{
 			Family: &dto.MetricFamily{
@@ -56,13 +56,13 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) {
 					},
 				},
 			},
-			Event: PromEvent{
-				key: "http_request_duration_microseconds",
-				value: common.MapStr{
-					"value": int64(10),
+			Event: []PromEvent{
+				{
+					data: common.MapStr{
+						"http_request_duration_microseconds": float64(10),
+					},
+					labels: labels,
 				},
-				labelHash: labels.String(),
-				labels:    labels,
 			},
 		},
 		{
@@ -78,12 +78,13 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) {
 					},
 				},
 			},
-			Event: PromEvent{
-				key: "http_request_duration_microseconds",
-				value: common.MapStr{
-					"value": float64(10),
+			Event: []PromEvent{
+				{
+					data: common.MapStr{
+						"http_request_duration_microseconds": float64(10),
+					},
+					labels: common.MapStr{},
 				},
-				labelHash: "#",
 			},
 		},
 		{
@@ -106,16 +107,22 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) {
 					},
 				},
 			},
-			Event: PromEvent{
-				key: "http_request_duration_microseconds",
-				value: common.MapStr{
-					"count": uint64(10),
-					"sum":   float64(10),
-					"percentile": common.MapStr{
-						"99": float64(10),
+			Event: []PromEvent{
+				{
+					data: common.MapStr{
+						"http_request_duration_microseconds_count": uint64(10),
+						"http_request_duration_microseconds_sum":   float64(10),
+					},
+					labels: common.MapStr{},
+				},
+				{
+					data: common.MapStr{
+						"http_request_duration_microseconds": float64(10),
+					},
+					labels: common.MapStr{
+						"quantile": float64(0.99),
 					},
 				},
-				labelHash: "#",
 			},
 		},
 		{
@@ -138,16 +145,20 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) {
 					},
 				},
 			},
-			Event: PromEvent{
-				key: "http_request_duration_microseconds",
-				value: common.MapStr{
-					"count": uint64(10),
-					"sum":   float64(10),
-					"bucket": common.MapStr{
-						"0.99": uint64(10),
+			Event: []PromEvent{
+				{
+					data: common.MapStr{
+						"http_request_duration_microseconds_count": uint64(10),
+						"http_request_duration_microseconds_sum":   float64(10),
+					},
+					labels: common.MapStr{},
+				},
+				{
+					data: common.MapStr{
+						"http_request_duration_microseconds": uint64(10),
 					},
+					labels: common.MapStr{"le": float64(0.99)},
 				},
-				labelHash: "#",
 			},
 		},
 		{
@@ -169,20 +180,19 @@ func TestGetPromEventsFromMetricFamily(t *testing.T) {
 					},
 				},
 			},
-			Event: PromEvent{
-				key: "http_request_duration_microseconds",
-				value: common.MapStr{
-					"value": float64(10),
+			Event: []PromEvent{
+				{
+					data: common.MapStr{
+						"http_request_duration_microseconds": float64(10),
+					},
+					labels: labels,
 				},
-				labelHash: labels.String(),
-				labels:    labels,
 			},
 		},
 	}
 
 	for _, test := range tests {
-		event := GetPromEventsFromMetricFamily(test.Family)
-		assert.Equal(t, len(event), 1)
-		assert.Equal(t, event[0], test.Event)
+		event := getPromEventsFromMetricFamily(test.Family)
+		assert.Equal(t, test.Event, event)
 	}
 }