From 735d3bae8c4e77e4990427a6fcd19630bc43c658 Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Tue, 7 Jul 2020 11:19:59 -0500 Subject: [PATCH] [APM] Improvements to data telemetry (#70524) Make some changes to how we deal with data telemetry in APM and reduce the number of fields we're storing in Saved Objects in the .kibana index. Add a telemetry doc in dev_docs explaining how telemetry is collected and how to make updates. (In this PR the docs only cover data telemetry, but there's a space for the behavioral telemetry docs.) Stop storing the mapping for the data telemetry in the Saved Object but instead use `{ dynamic: false }`. This reduces the number of fields used by APM in the .kibana index (as requested in #43673.) Before: ```bash > curl -s -X GET "admin:changeme@localhost:9200/.kibana/_field_caps?fields=*&pretty=true" | jq '.fields|length' 653 ``` After: ```bash > curl -s -X GET "admin:changeme@localhost:9200/.kibana/_field_caps?fields=*&pretty=true" | jq '.fields|length' 415 ``` We don't need the mapping anymore for storing the saved object, but we still do need to update the telemetry repository when the mapping changes, and the `upload-telemetry-data` script uses that mapping when generating data. For these purposes the mapping in now defined in TypeScript in a function in common/apm_telemetry.ts. It's broken down into some variables that and put together as the same mapping object that was there before, but having it in this form should make it easier to update. A new script, `merge-telemetry-mapping`, takes the telemetry repository's xpack-phone-home.json mapping, merges in the result of our mapping and replaces the file. The result can be committed to the telemetry repo, making it easier to make changes to the mapping. References #61583 Fixes #67032 --- .../__snapshots__/apm_telemetry.test.ts.snap | 913 ++++++++++++++++++ x-pack/plugins/apm/common/agent_name.ts | 5 +- .../plugins/apm/common/apm_telemetry.test.ts | 51 + x-pack/plugins/apm/common/apm_telemetry.ts | 229 +++++ x-pack/plugins/apm/dev_docs/telemetry.md | 69 ++ x-pack/plugins/apm/public/plugin.ts | 5 +- x-pack/plugins/apm/readme.md | 1 + .../apm/scripts/merge-telemetry-mapping.js | 21 + .../scripts/merge-telemetry-mapping/index.ts | 30 + .../scripts/shared/create-or-update-index.ts | 11 +- .../scripts/upload-telemetry-data/index.ts | 30 +- .../apm/server/saved_objects/apm_telemetry.ts | 912 +---------------- 12 files changed, 1349 insertions(+), 928 deletions(-) create mode 100644 x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap create mode 100644 x-pack/plugins/apm/common/apm_telemetry.test.ts create mode 100644 x-pack/plugins/apm/common/apm_telemetry.ts create mode 100644 x-pack/plugins/apm/dev_docs/telemetry.md create mode 100644 x-pack/plugins/apm/scripts/merge-telemetry-mapping.js create mode 100644 x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap new file mode 100644 index 0000000000000..3ac20a05639fb --- /dev/null +++ b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap @@ -0,0 +1,913 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the telemetry mapping 1`] = ` +Object { + "properties": Object { + "agents": Object { + "properties": Object { + "dotnet": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "go": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "java": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "js-base": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "nodejs": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "python": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "ruby": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + "rum-js": Object { + "properties": Object { + "agent": Object { + "properties": Object { + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "service": Object { + "properties": Object { + "framework": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "language": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + "runtime": Object { + "properties": Object { + "composite": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "name": Object { + "ignore_above": 1024, + "type": "keyword", + }, + "version": Object { + "ignore_above": 1024, + "type": "keyword", + }, + }, + }, + }, + }, + }, + }, + }, + }, + "cardinality": Object { + "properties": Object { + "transaction": Object { + "properties": Object { + "name": Object { + "properties": Object { + "all_agents": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "rum": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + }, + }, + }, + }, + "user_agent": Object { + "properties": Object { + "original": Object { + "properties": Object { + "all_agents": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "rum": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + }, + }, + }, + }, + }, + }, + "counts": Object { + "properties": Object { + "agent_configuration": Object { + "properties": Object { + "all": Object { + "type": "long", + }, + }, + }, + "error": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + "max_error_groups_per_service": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "max_transaction_groups_per_service": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "metric": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + "onboarding": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + "services": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "sourcemap": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + "span": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + "traces": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + }, + }, + "transaction": Object { + "properties": Object { + "1d": Object { + "type": "long", + }, + "all": Object { + "type": "long", + }, + }, + }, + }, + }, + "has_any_services": Object { + "type": "boolean", + }, + "indices": Object { + "properties": Object { + "all": Object { + "properties": Object { + "total": Object { + "properties": Object { + "docs": Object { + "properties": Object { + "count": Object { + "type": "long", + }, + }, + }, + "store": Object { + "properties": Object { + "size_in_bytes": Object { + "type": "long", + }, + }, + }, + }, + }, + }, + }, + "shards": Object { + "properties": Object { + "total": Object { + "type": "long", + }, + }, + }, + }, + }, + "integrations": Object { + "properties": Object { + "ml": Object { + "properties": Object { + "all_jobs_count": Object { + "type": "long", + }, + }, + }, + }, + }, + "retainment": Object { + "properties": Object { + "error": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + "metric": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + "onboarding": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + "span": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + "transaction": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "services_per_agent": Object { + "properties": Object { + "dotnet": Object { + "null_value": 0, + "type": "long", + }, + "go": Object { + "null_value": 0, + "type": "long", + }, + "java": Object { + "null_value": 0, + "type": "long", + }, + "js-base": Object { + "null_value": 0, + "type": "long", + }, + "nodejs": Object { + "null_value": 0, + "type": "long", + }, + "python": Object { + "null_value": 0, + "type": "long", + }, + "ruby": Object { + "null_value": 0, + "type": "long", + }, + "rum-js": Object { + "null_value": 0, + "type": "long", + }, + }, + }, + "tasks": Object { + "properties": Object { + "agent_configuration": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "agents": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "cardinality": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "groupings": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "indices_stats": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "integrations": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "processor_events": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "services": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + "versions": Object { + "properties": Object { + "took": Object { + "properties": Object { + "ms": Object { + "type": "long", + }, + }, + }, + }, + }, + }, + }, + "version": Object { + "properties": Object { + "apm_server": Object { + "properties": Object { + "major": Object { + "type": "long", + }, + "minor": Object { + "type": "long", + }, + "patch": Object { + "type": "long", + }, + }, + }, + }, + }, + }, +} +`; diff --git a/x-pack/plugins/apm/common/agent_name.ts b/x-pack/plugins/apm/common/agent_name.ts index 9d462dad87ec0..8b479d1d82fe7 100644 --- a/x-pack/plugins/apm/common/agent_name.ts +++ b/x-pack/plugins/apm/common/agent_name.ts @@ -15,15 +15,14 @@ import { AgentName } from '../typings/es_schemas/ui/fields/agent'; */ export const AGENT_NAMES: AgentName[] = [ - 'java', - 'js-base', - 'rum-js', 'dotnet', 'go', 'java', + 'js-base', 'nodejs', 'python', 'ruby', + 'rum-js', ]; export function isAgentName(agentName: string): agentName is AgentName { diff --git a/x-pack/plugins/apm/common/apm_telemetry.test.ts b/x-pack/plugins/apm/common/apm_telemetry.test.ts new file mode 100644 index 0000000000000..1612716142ce7 --- /dev/null +++ b/x-pack/plugins/apm/common/apm_telemetry.test.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + getApmTelemetryMapping, + mergeApmTelemetryMapping, +} from './apm_telemetry'; + +describe('APM telemetry helpers', () => { + describe('getApmTelemetry', () => { + it('generates a JSON object with the telemetry mapping', () => { + expect(getApmTelemetryMapping()).toMatchSnapshot(); + }); + }); + + describe('mergeApmTelemetryMapping', () => { + describe('with an invalid mapping', () => { + it('throws an error', () => { + expect(() => mergeApmTelemetryMapping({})).toThrowError(); + }); + }); + + describe('with a valid mapping', () => { + it('merges the mapping', () => { + // This is "valid" in the sense that it has all of the deep fields + // needed to merge. It's not a valid mapping opbject. + const validTelemetryMapping = { + mappings: { + properties: { + stack_stats: { + properties: { + kibana: { + properties: { plugins: { properties: { apm: {} } } }, + }, + }, + }, + }, + }, + }; + + expect( + mergeApmTelemetryMapping(validTelemetryMapping)?.mappings.properties + .stack_stats.properties.kibana.properties.plugins.properties.apm + ).toEqual(getApmTelemetryMapping()); + }); + }); + }); +}); diff --git a/x-pack/plugins/apm/common/apm_telemetry.ts b/x-pack/plugins/apm/common/apm_telemetry.ts new file mode 100644 index 0000000000000..1532058adf64f --- /dev/null +++ b/x-pack/plugins/apm/common/apm_telemetry.ts @@ -0,0 +1,229 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { produce } from 'immer'; +import { AGENT_NAMES } from './agent_name'; + +/** + * Generate an object containing the mapping used for APM telemetry. Can be used + * with the `upload-telemetry-data` script or to update the mapping in the + * telemetry repository. + * + * This function breaks things up to make the mapping easier to understand. + */ +export function getApmTelemetryMapping() { + const keyword = { + type: 'keyword', + ignore_above: 1024, + }; + + const long = { + type: 'long', + }; + + const allProperties = { + properties: { + all: long, + }, + }; + + const oneDayProperties = { + properties: { + '1d': long, + }, + }; + + const oneDayAllProperties = { + properties: { + '1d': long, + all: long, + }, + }; + + const msProperties = { + properties: { + ms: long, + }, + }; + + const tookProperties = { + properties: { + took: msProperties, + }, + }; + + const compositeNameVersionProperties = { + properties: { + composite: keyword, + name: keyword, + version: keyword, + }, + }; + + const agentProperties = { + properties: { version: keyword }, + }; + + const serviceProperties = { + properties: { + framework: compositeNameVersionProperties, + language: compositeNameVersionProperties, + runtime: compositeNameVersionProperties, + }, + }; + + return { + properties: { + agents: { + properties: AGENT_NAMES.reduce>( + (previousValue, currentValue) => { + previousValue[currentValue] = { + properties: { + agent: agentProperties, + service: serviceProperties, + }, + }; + + return previousValue; + }, + {} + ), + }, + counts: { + properties: { + agent_configuration: allProperties, + error: oneDayAllProperties, + max_error_groups_per_service: oneDayProperties, + max_transaction_groups_per_service: oneDayProperties, + metric: oneDayAllProperties, + onboarding: oneDayAllProperties, + services: oneDayProperties, + sourcemap: oneDayAllProperties, + span: oneDayAllProperties, + traces: oneDayProperties, + transaction: oneDayAllProperties, + }, + }, + cardinality: { + properties: { + user_agent: { + properties: { + original: { + properties: { + all_agents: oneDayProperties, + rum: oneDayProperties, + }, + }, + }, + }, + transaction: { + properties: { + name: { + properties: { + all_agents: oneDayProperties, + rum: oneDayProperties, + }, + }, + }, + }, + }, + }, + has_any_services: { + type: 'boolean', + }, + indices: { + properties: { + all: { + properties: { + total: { + properties: { + docs: { + properties: { + count: long, + }, + }, + store: { + properties: { + size_in_bytes: long, + }, + }, + }, + }, + }, + }, + shards: { + properties: { + total: long, + }, + }, + }, + }, + integrations: { + properties: { + ml: { + properties: { + all_jobs_count: long, + }, + }, + }, + }, + retainment: { + properties: { + error: msProperties, + metric: msProperties, + onboarding: msProperties, + span: msProperties, + transaction: msProperties, + }, + }, + services_per_agent: { + properties: AGENT_NAMES.reduce>( + (previousValue, currentValue) => { + previousValue[currentValue] = { ...long, null_value: 0 }; + return previousValue; + }, + {} + ), + }, + tasks: { + properties: { + agent_configuration: tookProperties, + agents: tookProperties, + cardinality: tookProperties, + groupings: tookProperties, + indices_stats: tookProperties, + integrations: tookProperties, + processor_events: tookProperties, + services: tookProperties, + versions: tookProperties, + }, + }, + version: { + properties: { + apm_server: { + properties: { + major: long, + minor: long, + patch: long, + }, + }, + }, + }, + }, + }; +} + +/** + * Merge a telemetry mapping object (from https://github.com/elastic/telemetry/blob/master/config/templates/xpack-phone-home.json) + * with the output from `getApmTelemetryMapping`. + */ +export function mergeApmTelemetryMapping( + xpackPhoneHomeMapping: Record +) { + return produce(xpackPhoneHomeMapping, (draft: Record) => { + draft.mappings.properties.stack_stats.properties.kibana.properties.plugins.properties.apm = getApmTelemetryMapping(); + return draft; + }); +} diff --git a/x-pack/plugins/apm/dev_docs/telemetry.md b/x-pack/plugins/apm/dev_docs/telemetry.md new file mode 100644 index 0000000000000..9674d39e57177 --- /dev/null +++ b/x-pack/plugins/apm/dev_docs/telemetry.md @@ -0,0 +1,69 @@ +# APM Telemetry + +In order to learn about our customers' usage and experience of APM, we collect +two types of telemetry, which we'll refer to here as "Data Telemetry" and +"Behavioral Telemetry." + +This document will explain how they are collected and how to make changes to +them. + +[The telemetry repository has information about accessing the clusters](https://github.com/elastic/telemetry#kibana-access). +Telemetry data is uploaded to the "xpack-phone-home" indices. + +## Data Telemetry + +Information that can be derived from a cluster's APM indices is queried and sent +to the telemetry cluster using the +[Usage Collection plugin](../../../../src/plugins/usage_collection/README.md). + +During the APM server-side plugin's setup phase a +[Saved Object](https://www.elastic.co/guide/en/kibana/master/managing-saved-objects.html) +for APM telemetry is registered and a +[task manager](../../task_manager/server/README.md) task is registered and started. +The task periodically queries the APM indices and saves the results in the Saved +Object, and the usage collector periodically gets the data from the saved object +and uploads it to the telemetry cluster. + +Once uploaded to the telemetry cluster, the data telemetry is stored in +`stack_stats.kibana.plugins.apm` in the xpack-phone-home index. + +### Generating sample data + +The script in `scripts/upload-telemetry-data` can generate sample telemetry data and upload it to a cluster of your choosing. + +You'll need to set the `GITHUB_TOKEN` environment variable to a token that has `repo` scope so it can read from the +[elastic/telemetry](https://github.com/elastic/telemetry) repository. (You probably have a token that works for this in +~/.backport/config.json.) + +The script will run as the `elastic` user using the elasticsearch hosts and password settings from the config/kibana.yml +and/or config/kibana.dev.yml files. + +Running the script with `--clear` will delete the index first. + +After running the script you should see sample telemetry data in the "xpack-phone-home" index. + +### Updating Data Telemetry Mappings + +In order for fields to be searchable on the telemetry cluster, they need to be +added to the cluster's mapping. The mapping is defined in +[the telemetry repository's xpack-phone-home template](https://github.com/elastic/telemetry/blob/master/config/templates/xpack-phone-home.json). + +The mapping for the telemetry data is here under `stack_stats.kibana.plugins.apm`. + +The mapping used there can be generated with the output of the [`getTelemetryMapping`](../common/apm_telemetry.ts) function. + +To make a change to the mapping, edit this function, run the tests to update the snapshots, then use the `merge_telemetry_mapping` script to merge the data into the telemetry repository. + +If the [telemetry repository](https://github.com/elastic/telemetry) is cloned as a sibling to the kibana directory, you can run the following from x-pack/plugins/apm: + +```bash +node ./scripts/merge-telemetry-mapping.js ../../../../telemetry/config/templates/xpack-phone-home.json +``` + +this will replace the contents of the mapping in the repository checkout with the updated mapping. You can then [follow the telemetry team's instructions](https://github.com/elastic/telemetry#mappings) for opening a pull request with the mapping changes. + +## Behavioral Telemetry + +Behavioral telemetry is recorded with the ui_metrics and application_usage methods from the Usage Collection plugin. + +Please fill this in with more details. diff --git a/x-pack/plugins/apm/public/plugin.ts b/x-pack/plugins/apm/public/plugin.ts index d24cb29eaf24f..f31ad83666a17 100644 --- a/x-pack/plugins/apm/public/plugin.ts +++ b/x-pack/plugins/apm/public/plugin.ts @@ -66,8 +66,9 @@ export interface ApmPluginStartDeps { } export class ApmPlugin implements Plugin { - private readonly initializerContext: PluginInitializerContext; - constructor(initializerContext: PluginInitializerContext) { + constructor( + private readonly initializerContext: PluginInitializerContext + ) { this.initializerContext = initializerContext; } public setup(core: CoreSetup, plugins: ApmPluginSetupDeps) { diff --git a/x-pack/plugins/apm/readme.md b/x-pack/plugins/apm/readme.md index f460ff6ff9bf2..9b02972d35302 100644 --- a/x-pack/plugins/apm/readme.md +++ b/x-pack/plugins/apm/readme.md @@ -162,3 +162,4 @@ You can access the development environment at http://localhost:9001. - [Cypress integration tests](./e2e/README.md) - [VSCode setup instructions](./dev_docs/vscode_setup.md) - [Github PR commands](./dev_docs/github_commands.md) +- [Telemetry](./dev_docs/telemetry.md) diff --git a/x-pack/plugins/apm/scripts/merge-telemetry-mapping.js b/x-pack/plugins/apm/scripts/merge-telemetry-mapping.js new file mode 100644 index 0000000000000..741df981a9cb0 --- /dev/null +++ b/x-pack/plugins/apm/scripts/merge-telemetry-mapping.js @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// compile typescript on the fly +// eslint-disable-next-line import/no-extraneous-dependencies +require('@babel/register')({ + extensions: ['.ts'], + plugins: [ + '@babel/plugin-proposal-optional-chaining', + '@babel/plugin-proposal-nullish-coalescing-operator', + ], + presets: [ + '@babel/typescript', + ['@babel/preset-env', { targets: { node: 'current' } }], + ], +}); + +require('./merge-telemetry-mapping/index.ts'); diff --git a/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts b/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts new file mode 100644 index 0000000000000..c06d4cec150dc --- /dev/null +++ b/x-pack/plugins/apm/scripts/merge-telemetry-mapping/index.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { readFileSync, truncateSync, writeFileSync } from 'fs'; +import { resolve } from 'path'; +import { argv } from 'yargs'; +import { mergeApmTelemetryMapping } from '../../common/apm_telemetry'; + +function errorExit(error?: Error) { + console.error(`usage: ${argv.$0} /path/to/xpack-phone-home.json`); // eslint-disable-line no-console + if (error) { + throw error; + } + process.exit(1); +} + +try { + const filename = resolve(argv._[0]); + const xpackPhoneHomeMapping = JSON.parse(readFileSync(filename, 'utf-8')); + + const newMapping = mergeApmTelemetryMapping(xpackPhoneHomeMapping); + + truncateSync(filename); + writeFileSync(filename, JSON.stringify(newMapping, null, 2)); +} catch (error) { + errorExit(error); +} diff --git a/x-pack/plugins/apm/scripts/shared/create-or-update-index.ts b/x-pack/plugins/apm/scripts/shared/create-or-update-index.ts index 3f88b73f55984..6d44e12fb00a2 100644 --- a/x-pack/plugins/apm/scripts/shared/create-or-update-index.ts +++ b/x-pack/plugins/apm/scripts/shared/create-or-update-index.ts @@ -30,6 +30,11 @@ export async function createOrUpdateIndex({ } } + // Some settings are non-updateable and need to be removed. + const settings = { ...template.settings }; + delete settings?.index?.number_of_shards; + delete settings?.index?.sort; + const indexExists = ( await client.indices.exists({ index: indexName, @@ -42,6 +47,7 @@ export async function createOrUpdateIndex({ body: template, }); } else { + await client.indices.close({ index: indexName }); await Promise.all([ template.mappings ? client.indices.putMapping({ @@ -49,12 +55,13 @@ export async function createOrUpdateIndex({ body: template.mappings, }) : Promise.resolve(undefined as any), - template.settings + settings ? client.indices.putSettings({ index: indexName, - body: template.settings, + body: settings, }) : Promise.resolve(undefined as any), ]); + await client.indices.open({ index: indexName }); } } diff --git a/x-pack/plugins/apm/scripts/upload-telemetry-data/index.ts b/x-pack/plugins/apm/scripts/upload-telemetry-data/index.ts index 5f9c72810fc91..a44fad82f20e6 100644 --- a/x-pack/plugins/apm/scripts/upload-telemetry-data/index.ts +++ b/x-pack/plugins/apm/scripts/upload-telemetry-data/index.ts @@ -11,7 +11,7 @@ // - Easier testing of the telemetry tasks // - Validate whether we can run the queries we want to on the telemetry data -import { merge, chunk, flatten } from 'lodash'; +import { merge, chunk, flatten, omit } from 'lodash'; import { Client } from '@elastic/elasticsearch'; import { argv } from 'yargs'; import { Logger } from 'kibana/server'; @@ -20,7 +20,7 @@ import { stampLogger } from '../shared/stamp-logger'; import { CollectTelemetryParams } from '../../server/lib/apm_telemetry/collect_data_telemetry'; import { downloadTelemetryTemplate } from '../shared/download-telemetry-template'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { apmTelemetry } from '../../server/saved_objects/apm_telemetry'; +import { mergeApmTelemetryMapping } from '../../common/apm_telemetry'; import { generateSampleDocuments } from './generate-sample-documents'; import { readKibanaConfig } from '../shared/read-kibana-config'; import { getHttpAuth } from '../shared/get-http-auth'; @@ -40,8 +40,6 @@ async function uploadData() { githubToken, }); - const kibanaMapping = apmTelemetry.mappings; - const config = readKibanaConfig(); const httpAuth = getHttpAuth(config); @@ -50,19 +48,25 @@ async function uploadData() { nodes: [config['elasticsearch.hosts']], ...(httpAuth ? { - auth: httpAuth, + auth: { ...httpAuth, username: 'elastic' }, } : {}), }); - const newTemplate = merge(telemetryTemplate, { - settings: { - index: { mapping: { total_fields: { limit: 10000 } } }, - }, - }); - - // override apm mapping instead of merging - newTemplate.mappings.properties.stack_stats.properties.kibana.properties.plugins.properties.apm = kibanaMapping; + // The new template is the template downloaded from the telemetry repo, with + // our current telemetry mapping merged in, with the "index_patterns" key + // (which cannot be used when creating an index) removed. + const newTemplate = omit( + mergeApmTelemetryMapping( + merge(telemetryTemplate, { + index_patterns: undefined, + settings: { + index: { mapping: { total_fields: { limit: 10000 } } }, + }, + }) + ), + 'index_patterns' + ); await createOrUpdateIndex({ indexName: xpackTelemetryIndexName, diff --git a/x-pack/plugins/apm/server/saved_objects/apm_telemetry.ts b/x-pack/plugins/apm/server/saved_objects/apm_telemetry.ts index f741b61fc7a04..411f453042b93 100644 --- a/x-pack/plugins/apm/server/saved_objects/apm_telemetry.ts +++ b/x-pack/plugins/apm/server/saved_objects/apm_telemetry.ts @@ -4,918 +4,14 @@ * you may not use this file except in compliance with the Elastic License. */ import { SavedObjectsType } from 'src/core/server'; +import { APM_TELEMETRY_SAVED_OBJECT_ID } from '../../common/apm_saved_object_constants'; export const apmTelemetry: SavedObjectsType = { - name: 'apm-telemetry', + name: APM_TELEMETRY_SAVED_OBJECT_ID, hidden: false, namespaceType: 'agnostic', mappings: { - properties: { - agents: { - properties: { - dotnet: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - go: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - java: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - 'js-base': { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - nodejs: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - python: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - ruby: { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - 'rum-js': { - properties: { - agent: { - properties: { - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - service: { - properties: { - framework: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - language: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - runtime: { - properties: { - composite: { - type: 'keyword', - ignore_above: 1024, - }, - name: { - type: 'keyword', - ignore_above: 1024, - }, - version: { - type: 'keyword', - ignore_above: 1024, - }, - }, - }, - }, - }, - }, - }, - }, - }, - counts: { - properties: { - agent_configuration: { - properties: { - all: { - type: 'long', - }, - }, - }, - error: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - max_error_groups_per_service: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - max_transaction_groups_per_service: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - metric: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - onboarding: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - services: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - sourcemap: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - span: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - traces: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - transaction: { - properties: { - '1d': { - type: 'long', - }, - all: { - type: 'long', - }, - }, - }, - }, - }, - cardinality: { - properties: { - user_agent: { - properties: { - original: { - properties: { - all_agents: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - rum: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - }, - }, - }, - }, - transaction: { - properties: { - name: { - properties: { - all_agents: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - rum: { - properties: { - '1d': { - type: 'long', - }, - }, - }, - }, - }, - }, - }, - }, - }, - has_any_services: { - type: 'boolean', - }, - indices: { - properties: { - all: { - properties: { - total: { - properties: { - docs: { - properties: { - count: { - type: 'long', - }, - }, - }, - store: { - properties: { - size_in_bytes: { - type: 'long', - }, - }, - }, - }, - }, - }, - }, - shards: { - properties: { - total: { - type: 'long', - }, - }, - }, - }, - }, - integrations: { - properties: { - ml: { - properties: { - all_jobs_count: { - type: 'long', - }, - }, - }, - }, - }, - retainment: { - properties: { - error: { - properties: { - ms: { - type: 'long', - }, - }, - }, - metric: { - properties: { - ms: { - type: 'long', - }, - }, - }, - onboarding: { - properties: { - ms: { - type: 'long', - }, - }, - }, - span: { - properties: { - ms: { - type: 'long', - }, - }, - }, - transaction: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - services_per_agent: { - properties: { - dotnet: { - type: 'long', - null_value: 0, - }, - go: { - type: 'long', - null_value: 0, - }, - java: { - type: 'long', - null_value: 0, - }, - 'js-base': { - type: 'long', - null_value: 0, - }, - nodejs: { - type: 'long', - null_value: 0, - }, - python: { - type: 'long', - null_value: 0, - }, - ruby: { - type: 'long', - null_value: 0, - }, - 'rum-js': { - type: 'long', - null_value: 0, - }, - }, - }, - tasks: { - properties: { - agent_configuration: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - agents: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - cardinality: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - groupings: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - indices_stats: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - integrations: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - processor_events: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - services: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - versions: { - properties: { - took: { - properties: { - ms: { - type: 'long', - }, - }, - }, - }, - }, - }, - }, - version: { - properties: { - apm_server: { - properties: { - major: { - type: 'long', - }, - minor: { - type: 'long', - }, - patch: { - type: 'long', - }, - }, - }, - }, - }, - } as SavedObjectsType['mappings']['properties'], + dynamic: false, + properties: {}, }, };