Add support for dynamic histogram metrics (#5239)

* model/metricset: add "histogram" dynamic_template * ingest/pipeline: handle metric descriptions * model: add metric type and unit * processor/otel: translate histogram metrics * Update changelog * processor/otel: fix histogram translation Handle histogram bucket counts correctly, following the approach taken by Prometheus. * processor/otel: fix merge * processor/otel: use appropriate slice cap (cherry picked from commit 91645bd) # Conflicts: # apmpackage/apm/0.2.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.json # apmpackage/apm/0.2.0/data_stream/error_logs/elasticsearch/ingest_pipeline/default.json # apmpackage/apm/0.2.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.json # apmpackage/apm/0.2.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/default.json # apmpackage/apm/0.2.0/data_stream/traces/elasticsearch/ingest_pipeline/default.json # apmpackage/apm/data_stream/app_metrics/elasticsearch/ingest_pipeline/apm_metrics_dynamic_template.json # apmpackage/apm/data_stream/error_logs/elasticsearch/ingest_pipeline/apm_metrics_dynamic_template.json # apmpackage/apm/data_stream/internal_metrics/elasticsearch/ingest_pipeline/apm_metrics_dynamic_template.json # apmpackage/apm/data_stream/profile_metrics/elasticsearch/ingest_pipeline/apm_metrics_dynamic_template.json # apmpackage/apm/data_stream/traces/elasticsearch/ingest_pipeline/apm_metrics_dynamic_template.json # changelogs/head.asciidoc # include/fields.go # processor/otel/metrics.go # processor/otel/metrics_test.go
elastic · Jul 9, 2021 · 632d3de · 632d3de
1 parent e369a0e
commit 632d3de
Show file tree

Hide file tree

Showing 9 changed files with 247 additions and 0 deletions.
diff --git a/apmpackage/apm/0.2.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.json b/apmpackage/apm/0.2.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.json
@@ -0,0 +1,37 @@
+{
+  "description": "Default enrichment for APM events",
+  "processors": [
+    {
+      "pipeline": {
+        "name": "metrics-apm.app-0.2.0-apm_ingest_timestamp"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.app-0.2.0-apm_user_agent"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.app-0.2.0-apm_user_geo"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.app-0.2.0-apm_remove_span_metadata"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.app-0.2.0-apm_error_grouping_name",
+        "if": "ctx.processor?.event == 'error'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.app-0.2.0-apm_metrics_dynamic_template",
+        "if": "ctx.processor?.event == 'metric'"
+      }
+    }
+  ]
+}
diff --git a/apmpackage/apm/0.2.0/data_stream/error_logs/elasticsearch/ingest_pipeline/default.json b/apmpackage/apm/0.2.0/data_stream/error_logs/elasticsearch/ingest_pipeline/default.json
@@ -0,0 +1,37 @@
+{
+  "description": "Default enrichment for APM events",
+  "processors": [
+    {
+      "pipeline": {
+        "name": "logs-apm.error-0.2.0-apm_ingest_timestamp"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-apm.error-0.2.0-apm_user_agent"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-apm.error-0.2.0-apm_user_geo"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-apm.error-0.2.0-apm_remove_span_metadata"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-apm.error-0.2.0-apm_error_grouping_name",
+        "if": "ctx.processor?.event == 'error'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "logs-apm.error-0.2.0-apm_metrics_dynamic_template",
+        "if": "ctx.processor?.event == 'metric'"
+      }
+    }
+  ]
+}
diff --git a/apmpackage/apm/0.2.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.json b/apmpackage/apm/0.2.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.json
@@ -0,0 +1,37 @@
+{
+  "description": "Default enrichment for APM events",
+  "processors": [
+    {
+      "pipeline": {
+        "name": "metrics-apm.internal-0.2.0-apm_ingest_timestamp"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.internal-0.2.0-apm_user_agent"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.internal-0.2.0-apm_user_geo"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.internal-0.2.0-apm_remove_span_metadata"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.internal-0.2.0-apm_error_grouping_name",
+        "if": "ctx.processor?.event == 'error'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.internal-0.2.0-apm_metrics_dynamic_template",
+        "if": "ctx.processor?.event == 'metric'"
+      }
+    }
+  ]
+}
diff --git a/apmpackage/apm/0.2.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/default.json b/apmpackage/apm/0.2.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/default.json
@@ -0,0 +1,37 @@
+{
+  "description": "Default enrichment for APM events",
+  "processors": [
+    {
+      "pipeline": {
+        "name": "metrics-apm.profiling-0.2.0-apm_ingest_timestamp"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.profiling-0.2.0-apm_user_agent"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.profiling-0.2.0-apm_user_geo"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.profiling-0.2.0-apm_remove_span_metadata"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.profiling-0.2.0-apm_error_grouping_name",
+        "if": "ctx.processor?.event == 'error'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.profiling-0.2.0-apm_metrics_dynamic_template",
+        "if": "ctx.processor?.event == 'metric'"
+      }
+    }
+  ]
+}
diff --git a/apmpackage/apm/0.2.0/data_stream/traces/elasticsearch/ingest_pipeline/default.json b/apmpackage/apm/0.2.0/data_stream/traces/elasticsearch/ingest_pipeline/default.json
@@ -0,0 +1,37 @@
+{
+  "description": "Default enrichment for APM events",
+  "processors": [
+    {
+      "pipeline": {
+        "name": "traces-apm-0.2.0-apm_ingest_timestamp"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "traces-apm-0.2.0-apm_user_agent"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "traces-apm-0.2.0-apm_user_geo"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "traces-apm-0.2.0-apm_remove_span_metadata"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "traces-apm-0.2.0-apm_error_grouping_name",
+        "if": "ctx.processor?.event == 'error'"
+      }
+    },
+    {
+      "pipeline": {
+        "name": "traces-apm-0.2.0-apm_metrics_dynamic_template",
+        "if": "ctx.processor?.event == 'metric'"
+      }
+    }
+  ]
+}
diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc
@@ -0,0 +1,29 @@
+[[release-notes-head]]
+== APM Server version HEAD
+
+https://github.com/elastic/apm-server/compare/7.13\...master[View commits]
+
+[float]
+==== Breaking Changes
+
+[float]
+==== Bug fixes
+* Don't auto-disable ILM due to a failure to communicate with Elasticsearch {pull}5264[5264]
+* Fix panic due to misaligned 64-bit access on 32-bit architectures {pull}5277[5277]
+* Fixed tail-based sampling pubsub to use _seq_no correctly {pull}5126[5126]
+* Fix document grouping of translated OpenTelemetry Java metrics {pull}5309[5309]
+* OpenTelemetry: record array attributes as labels {pull}5286[5286]
+* model/modeldecoder: fix 32-bit timestamp decoding {pull}5308[5308]
+
+[float]
+==== Intake API Changes
+
+[float]
+==== Added
+* Support setting agent configuration from apm-server.yml {pull}5177[5177]
+* Add metric_type and unit to field metadata of system metrics {pull}5230[5230]
+* Upgrade Go to 1.15.12 {pull}[]
+* Add support for dynamic histogram metrics {pull}5239[5239]
+
+[float]
+==== Deprecated
diff --git a/include/fields.go b/include/fields.go
diff --git a/processor/otel/metrics.go b/processor/otel/metrics.go
@@ -186,9 +186,15 @@ func (c *Consumer) addMetric(metric pdata.Metric, ms *metricsets) bool {
 			}
 		}
 		return !anyDropped
+<<<<<<< HEAD
 	case pdata.MetricDataTypeHistogram:
 		anyDropped := false
 		dps := metric.Histogram().DataPoints()
+=======
+	case pdata.MetricDataTypeDoubleHistogram:
+		anyDropped := false
+		dps := metric.DoubleHistogram().DataPoints()
+>>>>>>> 91645bde (Add support for dynamic histogram metrics (#5239))
 		for i := 0; i < dps.Len(); i++ {
 			dp := dps.At(i)
 			if sample, ok := histogramSample(metric.Name(), dp.BucketCounts(), dp.ExplicitBounds()); ok {
@@ -198,7 +204,11 @@ func (c *Consumer) addMetric(metric pdata.Metric, ms *metricsets) bool {
 			}
 		}
 		return !anyDropped
+<<<<<<< HEAD
 	case pdata.MetricDataTypeSummary:
+=======
+	case pdata.MetricDataTypeDoubleSummary:
+>>>>>>> 91645bde (Add support for dynamic histogram metrics (#5239))
 		// TODO(axw) https://github.com/elastic/apm-server/issues/3195
 		// (Not quite the same issue, but the solution would also enable
 		// aggregate metrics, which would be appropriate for summaries.)

diff --git a/processor/otel/metrics_test.go b/processor/otel/metrics_test.go
@@ -122,8 +122,13 @@ func TestConsumeMetrics(t *testing.T) {
 	doubleSum.DataPoints().At(2).SetValue(14)
 	doubleSum.DataPoints().At(2).LabelsMap().InitFromMap(map[string]string{"k2": "v"})
 
+<<<<<<< HEAD
 	metric = appendMetric("histogram_metric", pdata.MetricDataTypeHistogram)
 	doubleHistogram := metric.Histogram()
+=======
+	metric = appendMetric("double_histogram_metric", pdata.MetricDataTypeDoubleHistogram)
+	doubleHistogram := metric.DoubleHistogram()
+>>>>>>> 91645bde (Add support for dynamic histogram metrics (#5239))
 	doubleHistogram.DataPoints().Resize(1)
 	doubleHistogram.DataPoints().At(0).SetTimestamp(pdata.TimestampFromTime(timestamp0))
 	doubleHistogram.DataPoints().At(0).SetBucketCounts([]uint64{1, 1, 2, 3})
@@ -136,17 +141,27 @@ func TestConsumeMetrics(t *testing.T) {
 	intHistogram.DataPoints().At(0).SetBucketCounts([]uint64{0, 1, 2, 3})
 	intHistogram.DataPoints().At(0).SetExplicitBounds([]float64{1.0, 2.0, 3.0})
 
+<<<<<<< HEAD
 	metric = appendMetric("invalid_histogram_metric", pdata.MetricDataTypeHistogram)
 	invalidHistogram := metric.Histogram()
+=======
+	metric = appendMetric("invalid_histogram_metric", pdata.MetricDataTypeDoubleHistogram)
+	invalidHistogram := metric.DoubleHistogram()
+>>>>>>> 91645bde (Add support for dynamic histogram metrics (#5239))
 	invalidHistogram.DataPoints().Resize(1)
 	invalidHistogram.DataPoints().At(0).SetTimestamp(pdata.TimestampFromTime(timestamp0))
 	invalidHistogram.DataPoints().At(0).SetBucketCounts([]uint64{1, 2, 3}) // should be one more bucket count than bounds
 	invalidHistogram.DataPoints().At(0).SetExplicitBounds([]float64{1, 2, 3})
 	expectDropped++
 
 	// Summary metrics are not yet supported, and will be dropped.
+<<<<<<< HEAD
 	metric = appendMetric("double_summary_metric", pdata.MetricDataTypeSummary)
 	metric.Summary().DataPoints().Resize(1)
+=======
+	metric = appendMetric("double_summary_metric", pdata.MetricDataTypeDoubleSummary)
+	metric.DoubleSummary().DataPoints().Resize(1)
+>>>>>>> 91645bde (Add support for dynamic histogram metrics (#5239))
 	expectDropped++
 
 	metadata := model.Metadata{
@@ -175,7 +190,11 @@ func TestConsumeMetrics(t *testing.T) {
 			{Name: "double_sum_metric", Value: 12, Type: "counter"},
 
 			{
+<<<<<<< HEAD
 				Name: "histogram_metric", Type: "histogram",
+=======
+				Name: "double_histogram_metric", Type: "histogram",
+>>>>>>> 91645bde (Add support for dynamic histogram metrics (#5239))
 				Counts: []int64{1, 1, 2, 3},
 				Values: []float64{-1, 0.5, 2.75, 3.5},
 			},