diff --git a/apmpackage/apm/changelog.yml b/apmpackage/apm/changelog.yml index 63caa756153..6c7445ababd 100644 --- a/apmpackage/apm/changelog.yml +++ b/apmpackage/apm/changelog.yml @@ -6,6 +6,9 @@ - description: define index sorting for internal metrics type: enhancement link: https://github.com/elastic/apm-server/pull/6116 + - description: add histogram dynamic_template to app metrics data stream + type: enhancement + link: https://github.com/elastic/apm-server/pull/6043 - version: "0.4.0" changes: - description: add anonymous auth config, replace some RUM config diff --git a/apmpackage/apm/data_stream/app_metrics/fields/fields.yml b/apmpackage/apm/data_stream/app_metrics/fields/fields.yml index 6445c808c71..8ff494dc7af 100644 --- a/apmpackage/apm/data_stream/app_metrics/fields/fields.yml +++ b/apmpackage/apm/data_stream/app_metrics/fields/fields.yml @@ -1,5 +1,3 @@ -- name: histogram - type: histogram - name: kubernetes title: Kubernetes type: group diff --git a/apmpackage/apm/data_stream/app_metrics/manifest.yml b/apmpackage/apm/data_stream/app_metrics/manifest.yml index e3ad2aa7b64..c625fd1b7c7 100644 --- a/apmpackage/apm/data_stream/app_metrics/manifest.yml +++ b/apmpackage/apm/data_stream/app_metrics/manifest.yml @@ -10,3 +10,9 @@ elasticsearch: # as their names are application-specific and not # known ahead of time. dynamic: true + # Install dynamic templates for use in dynamically + # mapping complex application metrics. + dynamic_templates: + - histogram: + mapping: + type: histogram diff --git a/apmpackage/apm/data_stream/internal_metrics/fields/fields.yml b/apmpackage/apm/data_stream/internal_metrics/fields/fields.yml index c4197e121af..063b7806e91 100644 --- a/apmpackage/apm/data_stream/internal_metrics/fields/fields.yml +++ b/apmpackage/apm/data_stream/internal_metrics/fields/fields.yml @@ -1,5 +1,3 @@ -- name: histogram - type: histogram - name: kubernetes title: Kubernetes type: group diff --git a/apmpackage/apm/docs/README.md b/apmpackage/apm/docs/README.md index 77a71e95c7b..52d22a085ee 100644 --- a/apmpackage/apm/docs/README.md +++ b/apmpackage/apm/docs/README.md @@ -453,7 +453,6 @@ Metrics are written to `metrics-apm.app.*`, `metrics-apm.internal.*`, and `metri |destination.ip|IP addess of the destination. Can be one of multiple IPv4 or IPv6 addresses.|ip| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) | |destination.port|Port of the destination.|long| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) | |ecs.version|ECS version the event conforms to.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) | -|histogram||histogram| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) | |host.architecture|The architecture of the host the event was recorded on.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) | |host.hostname|The hostname of the host the event was recorded on.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) | |host.ip|IP of the host that records the event.|ip| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) | diff --git a/apmpackage/cmd/gen-package/field.go b/apmpackage/cmd/gen-package/field.go index a42f41148b1..a32ef4480f3 100644 --- a/apmpackage/cmd/gen-package/field.go +++ b/apmpackage/cmd/gen-package/field.go @@ -34,6 +34,7 @@ type field struct { // opt *in* to dynamically mapping where needed. Dynamic bool `yaml:"dynamic,omitempty"` + DynamicTemplate bool `yaml:"dynamic_template,omitempty"` ObjectTypeParams interface{} `yaml:"object_type_params,omitempty"` Release string `yaml:"release,omitempty"` Alias string `yaml:"alias,omitempty"` diff --git a/apmpackage/cmd/gen-package/genfields.go b/apmpackage/cmd/gen-package/genfields.go index 9763a4afaad..0d98971a0db 100644 --- a/apmpackage/cmd/gen-package/genfields.go +++ b/apmpackage/cmd/gen-package/genfields.go @@ -193,6 +193,13 @@ type fieldMapItem struct { } func (m fieldMap) update(f field) { + if f.DynamicTemplate { + // We don't add dynamic_template "fields" to the + // integration package; they are manually defined + // in the data stream manifest. + return + } + item := m[f.Name] item.field = f if item.fields == nil { diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc index d06ab57f63e..ac29a334ca8 100644 --- a/changelogs/head.asciidoc +++ b/changelogs/head.asciidoc @@ -35,6 +35,7 @@ https://github.com/elastic/apm-server/compare/7.13\...master[View commits] - Data streams now define a default `dynamic` mapping parameter, overridable in the `@custom` template {pull}5947[5947] - The `error.log.message` or `error.exception.message` field of errors will be copied to the ECS field `message` {pull}5974[5974] - Define index sorting for internal metrics data stream {pull}6116[6116] +- Add histogram dynamic_template to app metrics data stream {pull}6043[6043] [float] ==== Deprecated diff --git a/systemtest/approvals/TestApprovedMetrics.approved.json b/systemtest/approvals/TestApprovedMetrics/data_streams_disabled.approved.json similarity index 100% rename from systemtest/approvals/TestApprovedMetrics.approved.json rename to systemtest/approvals/TestApprovedMetrics/data_streams_disabled.approved.json diff --git a/systemtest/approvals/TestApprovedMetrics/data_streams_enabled.approved.json b/systemtest/approvals/TestApprovedMetrics/data_streams_enabled.approved.json new file mode 100644 index 00000000000..94d49477047 --- /dev/null +++ b/systemtest/approvals/TestApprovedMetrics/data_streams_enabled.approved.json @@ -0,0 +1,317 @@ +{ + "events": [ + { + "@timestamp": "2017-05-30T18:53:41.364Z", + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "data_stream.dataset": "apm.app.1234_service_12a3", + "data_stream.namespace": "default", + "data_stream.type": "metrics", + "ecs": { + "version": "dynamic" + }, + "event": { + "agent_id_status": "missing", + "ingested": "dynamic" + }, + "go.memstats.heap.sys.bytes": 6520832, + "host": { + "ip": "127.0.0.1" + }, + "labels": { + "tag1": "one", + "tag2": 2 + }, + "metricset.name": "app", + "observer": { + "ephemeral_id": "dynamic", + "hostname": "dynamic", + "id": "dynamic", + "type": "apm-server", + "version": "dynamic", + "version_major": "dynamic" + }, + "process": { + "pid": 1234 + }, + "processor": { + "event": "metric", + "name": "metric" + }, + "service": { + "language": { + "name": "ecmascript" + }, + "name": "1234_service-12a3", + "node": { + "name": "node-1" + } + }, + "user": { + "email": "user@mail.com", + "id": "axb123hg", + "name": "logged-in-user" + } + }, + { + "@timestamp": "2017-05-30T18:53:41.366Z", + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "data_stream.dataset": "apm.app.1234_service_12a3", + "data_stream.namespace": "default", + "data_stream.type": "metrics", + "ecs": { + "version": "dynamic" + }, + "event": { + "agent_id_status": "missing", + "ingested": "dynamic" + }, + "host": { + "ip": "127.0.0.1" + }, + "labels": { + "tag1": "one", + "tag2": 2 + }, + "metricset.name": "app", + "observer": { + "ephemeral_id": "dynamic", + "hostname": "dynamic", + "id": "dynamic", + "type": "apm-server", + "version": "dynamic", + "version_major": "dynamic" + }, + "process": { + "pid": 1234 + }, + "processor": { + "event": "metric", + "name": "metric" + }, + "service": { + "language": { + "name": "ecmascript" + }, + "name": "1234_service-12a3", + "node": { + "name": "node-1" + } + }, + "system.process.cgroup.memory.mem.limit.bytes": 2048, + "system.process.cgroup.memory.mem.usage.bytes": 1024, + "user": { + "email": "user@mail.com", + "id": "axb123hg", + "name": "logged-in-user" + } + }, + { + "@timestamp": "2017-05-30T18:53:41.366Z", + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "data_stream.dataset": "apm.app.1234_service_12a3", + "data_stream.namespace": "default", + "data_stream.type": "metrics", + "ecs": { + "version": "dynamic" + }, + "event": { + "agent_id_status": "missing", + "ingested": "dynamic" + }, + "host": { + "ip": "127.0.0.1" + }, + "labels": { + "tag1": "one", + "tag2": 2 + }, + "metricset.name": "app", + "observer": { + "ephemeral_id": "dynamic", + "hostname": "dynamic", + "id": "dynamic", + "type": "apm-server", + "version": "dynamic", + "version_major": "dynamic" + }, + "process": { + "pid": 1234 + }, + "processor": { + "event": "metric", + "name": "metric" + }, + "service": { + "language": { + "name": "ecmascript" + }, + "name": "1234_service-12a3", + "node": { + "name": "node-1" + } + }, + "system.process.cgroup.cpu.cfs.period.us": 1024, + "system.process.cgroup.cpu.cfs.quota.us": 2048, + "system.process.cgroup.cpu.id": 2048, + "system.process.cgroup.cpu.stats.periods": 2048, + "system.process.cgroup.cpu.stats.throttled.ns": 2048, + "system.process.cgroup.cpu.stats.throttled.periods": 2048, + "system.process.cgroup.cpuacct.id": 2048, + "system.process.cgroup.cpuacct.total.ns": 2048, + "user": { + "email": "user@mail.com", + "id": "axb123hg", + "name": "logged-in-user" + } + }, + { + "@timestamp": "2017-05-30T18:53:41.366Z", + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "data_stream.dataset": "apm.app.1234_service_12a3", + "data_stream.namespace": "default", + "data_stream.type": "metrics", + "ecs": { + "version": "dynamic" + }, + "event": { + "agent_id_status": "missing", + "ingested": "dynamic" + }, + "host": { + "ip": "127.0.0.1" + }, + "labels": { + "tag1": "one", + "tag2": 2 + }, + "latency_distribution": { + "counts": [ + 1, + 2, + 3 + ], + "values": [ + 1.1, + 2.2, + 3.3 + ] + }, + "metricset.name": "app", + "observer": { + "ephemeral_id": "dynamic", + "hostname": "dynamic", + "id": "dynamic", + "type": "apm-server", + "version": "dynamic", + "version_major": "dynamic" + }, + "process": { + "pid": 1234 + }, + "processor": { + "event": "metric", + "name": "metric" + }, + "service": { + "language": { + "name": "ecmascript" + }, + "name": "1234_service-12a3", + "node": { + "name": "node-1" + } + }, + "user": { + "email": "user@mail.com", + "id": "axb123hg", + "name": "logged-in-user" + } + }, + { + "@timestamp": "2017-05-30T18:53:42.281Z", + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "data_stream.dataset": "apm.internal", + "data_stream.namespace": "default", + "data_stream.type": "metrics", + "ecs": { + "version": "dynamic" + }, + "event": { + "agent_id_status": "missing", + "ingested": "dynamic" + }, + "host": { + "ip": "127.0.0.1" + }, + "labels": { + "code": 200, + "some": "abc", + "success": true, + "tag1": "one", + "tag2": 2 + }, + "metricset.name": "span_breakdown", + "observer": { + "ephemeral_id": "dynamic", + "hostname": "dynamic", + "id": "dynamic", + "type": "apm-server", + "version": "dynamic", + "version_major": "dynamic" + }, + "process": { + "pid": 1234 + }, + "processor": { + "event": "metric", + "name": "metric" + }, + "service": { + "language": { + "name": "ecmascript" + }, + "name": "1234_service-12a3", + "node": { + "name": "node-1" + } + }, + "span": { + "self_time": { + "count": 1, + "sum.us": 633 + }, + "subtype": "mysql", + "type": "db" + }, + "transaction": { + "breakdown.count": 12, + "duration": { + "count": 2, + "sum.us": 12 + }, + "name": "GET /", + "type": "request" + }, + "user": { + "email": "user@mail.com", + "id": "axb123hg", + "name": "logged-in-user" + } + } + ] +} diff --git a/systemtest/estest/search.go b/systemtest/estest/search.go index 7d495dde4cd..a594393b350 100644 --- a/systemtest/estest/search.go +++ b/systemtest/estest/search.go @@ -20,6 +20,7 @@ package estest import ( "context" "encoding/json" + "strings" "testing" "github.com/elastic/go-elasticsearch/v7/esapi" @@ -62,7 +63,7 @@ func (es *Client) ExpectMinDocs(t testing.TB, min int, index string, query inter func (es *Client) Search(index string) *SearchRequest { req := &SearchRequest{es: es} - req.Index = []string{index} + req.Index = strings.Split(index, ",") return req } diff --git a/systemtest/helpers_test.go b/systemtest/helpers_test.go new file mode 100644 index 00000000000..3fa94c8a7b3 --- /dev/null +++ b/systemtest/helpers_test.go @@ -0,0 +1,46 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package systemtest_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/apm-server/systemtest" + "github.com/elastic/apm-server/systemtest/apmservertest" +) + +// withDataStreams runs two sub-tests, calling f with and without data streams enabled. +func withDataStreams(t *testing.T, f func(t *testing.T, unstartedServer *apmservertest.Server)) { + t.Run("data_streams_disabled", func(t *testing.T) { + systemtest.CleanupElasticsearch(t) + srv := apmservertest.NewUnstartedServer(t) + f(t, srv) + }) + t.Run("data_streams_enabled", func(t *testing.T) { + systemtest.CleanupElasticsearch(t) + cleanupFleet(t, systemtest.Fleet) + integrationPackage := getAPMIntegrationPackage(t, systemtest.Fleet) + err := systemtest.Fleet.InstallPackage(integrationPackage.Name, integrationPackage.Version) + require.NoError(t, err) + srv := apmservertest.NewUnstartedServer(t) + srv.Config.DataStreams = &apmservertest.DataStreamsConfig{Enabled: true} + f(t, srv) + }) +} diff --git a/systemtest/metrics_test.go b/systemtest/metrics_test.go index b3add0d3245..91ecc42bff4 100644 --- a/systemtest/metrics_test.go +++ b/systemtest/metrics_test.go @@ -39,8 +39,12 @@ import ( ) func TestApprovedMetrics(t *testing.T) { - systemtest.CleanupElasticsearch(t) - srv := apmservertest.NewServer(t) + withDataStreams(t, testApprovedMetrics) +} + +func testApprovedMetrics(t *testing.T, srv *apmservertest.Server) { + err := srv.Start() + require.NoError(t, err) eventsPayload, err := ioutil.ReadFile("../testdata/intake-v2/metricsets.ndjson") require.NoError(t, err) @@ -58,14 +62,15 @@ func TestApprovedMetrics(t *testing.T) { assert.NoError(t, err) // Check the metrics documents are exactly as we expect. - result := systemtest.Elasticsearch.ExpectMinDocs(t, ingestResult.Accepted, "apm-*", estest.TermQuery{ + indices := []string{"apm-*", "metrics-apm.*"} + result := systemtest.Elasticsearch.ExpectMinDocs(t, ingestResult.Accepted, strings.Join(indices, ","), estest.TermQuery{ Field: "processor.event", Value: "metric", }) systemtest.ApproveEvents(t, t.Name(), result.Hits.Hits) // Check dynamic mapping of histograms. - mappings := getFieldMappings(t, []string{"apm-*"}, []string{"latency_distribution"}) + mappings := getFieldMappings(t, indices, []string{"latency_distribution"}) assert.Equal(t, map[string]interface{}{ "latency_distribution": map[string]interface{}{ "full_name": "latency_distribution",