diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 372b8e35244..1e1e430e995 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -161,6 +161,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix behavior of cgroups path discovery when monitoring the host system from within a container {pull}39627[39627] - Fix issue where beats may report incorrect metrics for its own process when running inside a container {pull}39627[39627] - Fix for MySQL/Performance - Query failure for MySQL versions below v8.0.1, for performance metric `quantile_95`. {pull}38710[38710] +- Fix Prometheus helper text parser to store each metric family type. {pull}39743[39743] - Normalize AWS RDS CPU Utilization values before making the metadata API call. {pull}39664[39664] - Fix query logic for temp and non-temp tablespaces in Oracle module. {issue}38051[38051] {pull}39787[39787] diff --git a/metricbeat/helper/prometheus/textparse.go b/metricbeat/helper/prometheus/textparse.go index 4dca85c3aa5..5430bab07ef 100644 --- a/metricbeat/helper/prometheus/textparse.go +++ b/metricbeat/helper/prometheus/textparse.go @@ -488,7 +488,8 @@ func ParseMetricFamilies(b []byte, contentType string, ts time.Time, logger *log summariesByName = map[string]map[string]*OpenMetric{} histogramsByName = map[string]map[string]*OpenMetric{} fam *MetricFamily - mt = textparse.MetricTypeUnknown + // metricTypes stores the metric type for each metric name. + metricTypes = make(map[string]textparse.MetricType) ) var err error @@ -530,7 +531,8 @@ func ParseMetricFamilies(b []byte, contentType string, ts time.Time, logger *log } else { fam.Type = t } - mt = t + // Store the metric type for each base metric name. + metricTypes[s] = t continue case textparse.EntryHelp: buf, t := parser.Help() @@ -611,6 +613,23 @@ func ParseMetricFamilies(b []byte, contentType string, ts time.Time, logger *log lookupMetricName := metricName var exm *exemplar.Exemplar + mt, ok := metricTypes[metricName] + if !ok { + // Splitting is necessary to find the base metric name type in the metricTypes map. + // This allows us to group related metrics together under the same base metric name. + // For example, the metric family `summary_metric` can have the metrics + // `summary_metric_count` and `summary_metric_sum`, all having the same metric type. + parts := strings.Split(metricName, "_") + baseMetricNamekey := strings.Join(parts[:len(parts)-1], "_") + + // If the metric type is not found, default to unknown + if metricTypeFound, ok := metricTypes[baseMetricNamekey]; ok { + mt = metricTypeFound + } else { + mt = textparse.MetricTypeUnknown + } + } + // Suffixes - https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#suffixes switch mt { case textparse.MetricTypeCounter: diff --git a/metricbeat/helper/prometheus/textparse_test.go b/metricbeat/helper/prometheus/textparse_test.go index cd76e14691f..e6c4fff4452 100644 --- a/metricbeat/helper/prometheus/textparse_test.go +++ b/metricbeat/helper/prometheus/textparse_test.go @@ -144,6 +144,166 @@ process_cpu 20 require.ElementsMatch(t, expected, result) } +func TestGroupWithHeaderBlockPrometheus(t *testing.T) { + input := ` +# HELP nginx_sts_server_bytes_total The request/response bytes +# TYPE nginx_sts_server_bytes_total counter +# HELP nginx_sts_server_connects_total The connects counter +# TYPE nginx_sts_server_connects_total counter +# HELP nginx_sts_server_session_seconds_total The session duration time +# TYPE nginx_sts_server_session_seconds_total counter +# HELP nginx_sts_server_session_seconds The average of session duration time in seconds +# TYPE nginx_sts_server_session_seconds gauge +# HELP nginx_sts_server_session_duration_seconds The histogram of session duration in seconds +# TYPE nginx_sts_server_session_duration_seconds histogram +nginx_sts_server_bytes_total{listen="TCP:8091:127.0.0.1",port="8091",protocol="TCP",direction="in"} 0 +nginx_sts_server_bytes_total{listen="TCP:8091:127.0.0.1",port="8091",protocol="TCP",direction="out"} 0 +nginx_sts_server_connects_total{listen="TCP:8091:127.0.0.1",port="8091",protocol="TCP",code="1xx"} 0 +nginx_sts_server_connects_total{listen="TCP:8091:127.0.0.1",port="8091",protocol="TCP",code="2xx"} 0 +nginx_sts_server_connects_total{listen="TCP:8091:127.0.0.1",port="8091",protocol="TCP",code="3xx"} 0 +nginx_sts_server_connects_total{listen="TCP:8091:127.0.0.1",port="8091",protocol="TCP",code="4xx"} 0 +nginx_sts_server_connects_total{listen="TCP:8091:127.0.0.1",port="8091",protocol="TCP",code="5xx"} 171 +nginx_sts_server_connects_total{listen="TCP:8091:127.0.0.1",port="8091",protocol="TCP",code="total"} 171 +nginx_sts_server_session_seconds_total{listen="TCP:8091:127.0.0.1",port="8091",protocol="TCP"} 0.016 +nginx_sts_server_session_seconds{listen="TCP:8091:127.0.0.1",port="8091",protocol="TCP"} 0.000 +` + + expected := []*MetricFamily{ + { + Name: stringp("nginx_sts_server_bytes_total"), + Help: stringp("The request/response bytes"), + Type: "counter", + Metric: []*OpenMetric{ + { + Name: stringp("nginx_sts_server_bytes_total"), + Label: []*labels.Label{ + {Name: "direction", Value: "in"}, + {Name: "listen", Value: "TCP:8091:127.0.0.1"}, + {Name: "port", Value: "8091"}, + {Name: "protocol", Value: "TCP"}, + }, + Counter: &Counter{Value: float64p(0)}, + }, + { + Name: stringp("nginx_sts_server_bytes_total"), + Label: []*labels.Label{ + {Name: "direction", Value: "out"}, + {Name: "listen", Value: "TCP:8091:127.0.0.1"}, + {Name: "port", Value: "8091"}, + {Name: "protocol", Value: "TCP"}, + }, + Counter: &Counter{Value: float64p(0)}, + }, + }, + }, + { + Name: stringp("nginx_sts_server_connects_total"), + Help: stringp("The connects counter"), + Type: "counter", + Metric: []*OpenMetric{ + { + Name: stringp("nginx_sts_server_connects_total"), + Label: []*labels.Label{ + {Name: "code", Value: "1xx"}, + {Name: "listen", Value: "TCP:8091:127.0.0.1"}, + {Name: "port", Value: "8091"}, + {Name: "protocol", Value: "TCP"}, + }, + Counter: &Counter{Value: float64p(0)}, + }, + { + Name: stringp("nginx_sts_server_connects_total"), + Label: []*labels.Label{ + {Name: "code", Value: "2xx"}, + {Name: "listen", Value: "TCP:8091:127.0.0.1"}, + {Name: "port", Value: "8091"}, + {Name: "protocol", Value: "TCP"}, + }, + Counter: &Counter{Value: float64p(0)}, + }, + { + Name: stringp("nginx_sts_server_connects_total"), + Label: []*labels.Label{ + {Name: "code", Value: "3xx"}, + {Name: "listen", Value: "TCP:8091:127.0.0.1"}, + {Name: "port", Value: "8091"}, + {Name: "protocol", Value: "TCP"}, + }, + Counter: &Counter{Value: float64p(0)}, + }, + { + Name: stringp("nginx_sts_server_connects_total"), + Label: []*labels.Label{ + {Name: "code", Value: "4xx"}, + {Name: "listen", Value: "TCP:8091:127.0.0.1"}, + {Name: "port", Value: "8091"}, + {Name: "protocol", Value: "TCP"}, + }, + Counter: &Counter{Value: float64p(0)}, + }, + { + Name: stringp("nginx_sts_server_connects_total"), + Label: []*labels.Label{ + {Name: "code", Value: "5xx"}, + {Name: "listen", Value: "TCP:8091:127.0.0.1"}, + {Name: "port", Value: "8091"}, + {Name: "protocol", Value: "TCP"}, + }, + Counter: &Counter{Value: float64p(171)}, + }, + { + Name: stringp("nginx_sts_server_connects_total"), + Label: []*labels.Label{ + {Name: "code", Value: "total"}, + {Name: "listen", Value: "TCP:8091:127.0.0.1"}, + {Name: "port", Value: "8091"}, + {Name: "protocol", Value: "TCP"}, + }, + Counter: &Counter{Value: float64p(171)}, + }, + }, + }, + { + Name: stringp("nginx_sts_server_session_seconds_total"), + Help: stringp("The session duration time"), + Type: "counter", + Metric: []*OpenMetric{ + { + Name: stringp("nginx_sts_server_session_seconds_total"), + Label: []*labels.Label{ + {Name: "listen", Value: "TCP:8091:127.0.0.1"}, + {Name: "port", Value: "8091"}, + {Name: "protocol", Value: "TCP"}, + }, + Counter: &Counter{Value: float64p(0.016)}, + }, + }, + }, + { + Name: stringp("nginx_sts_server_session_seconds"), + Help: stringp("The average of session duration time in seconds"), + Type: "gauge", + Metric: []*OpenMetric{ + { + Name: stringp("nginx_sts_server_session_seconds"), + Label: []*labels.Label{ + {Name: "listen", Value: "TCP:8091:127.0.0.1"}, + {Name: "port", Value: "8091"}, + {Name: "protocol", Value: "TCP"}, + }, + Gauge: &Gauge{Value: float64p(0.000)}, + }, + }, + }, + } + + result, err := ParseMetricFamilies([]byte(input), ContentTypeTextFormat, time.Now(), nil) + if err != nil { + t.Fatalf("ParseMetricFamilies for content type %s returned an error: %v", ContentTypeTextFormat, err) + } + require.ElementsMatch(t, expected, result) +} + func TestGaugeOpenMetrics(t *testing.T) { input := ` # TYPE first_metric gauge