From f54244df1331e4864b90138110dcd00d3f0bfc26 Mon Sep 17 00:00:00 2001 From: Oliver Gupte Date: Tue, 20 Nov 2018 11:30:12 -0800 Subject: [PATCH] [APM] adds telemetry to APM (#25513) (#25943) * [APM] adds telemetry to APM * [APM] Code and readability improvements for APM Telemetry * [APM] fixed failing tests for apm-telemetry and service routes * [APM] fix lint issues for APM Telemetry --- x-pack/plugins/apm/index.js | 11 +- x-pack/plugins/apm/mappings.json | 37 +++++ .../__test__/apm_telemetry.test.ts | 152 ++++++++++++++++++ .../server/lib/apm_telemetry/apm_telemetry.ts | 59 +++++++ .../apm/server/lib/apm_telemetry/index.ts | 14 ++ .../apm_telemetry/make_apm_usage_collector.ts | 42 +++++ x-pack/plugins/apm/server/routes/services.ts | 23 ++- 7 files changed, 335 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/apm/mappings.json create mode 100644 x-pack/plugins/apm/server/lib/apm_telemetry/__test__/apm_telemetry.test.ts create mode 100644 x-pack/plugins/apm/server/lib/apm_telemetry/apm_telemetry.ts create mode 100644 x-pack/plugins/apm/server/lib/apm_telemetry/index.ts create mode 100644 x-pack/plugins/apm/server/lib/apm_telemetry/make_apm_usage_collector.ts diff --git a/x-pack/plugins/apm/index.js b/x-pack/plugins/apm/index.js index da81a5bfa6aee..f9685467b655d 100644 --- a/x-pack/plugins/apm/index.js +++ b/x-pack/plugins/apm/index.js @@ -10,6 +10,8 @@ import { initServicesApi } from './server/routes/services'; import { initErrorsApi } from './server/routes/errors'; import { initStatusApi } from './server/routes/status_check'; import { initTracesApi } from './server/routes/traces'; +import mappings from './mappings'; +import { makeApmUsageCollector } from './server/lib/apm_telemetry'; export function apm(kibana) { return new kibana.Plugin({ @@ -35,7 +37,13 @@ export function apm(kibana) { apmIndexPattern: config.get('apm_oss.indexPattern') }; }, - hacks: ['plugins/apm/hacks/toggle_app_link_in_nav'] + hacks: ['plugins/apm/hacks/toggle_app_link_in_nav'], + savedObjectSchemas: { + 'apm-telemetry': { + isNamespaceAgnostic: true + } + }, + mappings }, config(Joi) { @@ -60,6 +68,7 @@ export function apm(kibana) { initServicesApi(server); initErrorsApi(server); initStatusApi(server); + makeApmUsageCollector(server); } }); } diff --git a/x-pack/plugins/apm/mappings.json b/x-pack/plugins/apm/mappings.json new file mode 100644 index 0000000000000..3ed52b9c8d639 --- /dev/null +++ b/x-pack/plugins/apm/mappings.json @@ -0,0 +1,37 @@ +{ + "apm-telemetry": { + "properties": { + "has_any_services": { + "type": "boolean" + }, + "services_per_agent": { + "properties": { + "python": { + "type": "long", + "null_value": 0 + }, + "java": { + "type": "long", + "null_value": 0 + }, + "nodejs": { + "type": "long", + "null_value": 0 + }, + "js-base": { + "type": "long", + "null_value": 0 + }, + "ruby": { + "type": "long", + "null_value": 0 + }, + "go": { + "type": "long", + "null_value": 0 + } + } + } + } + } +} diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/__test__/apm_telemetry.test.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/__test__/apm_telemetry.test.ts new file mode 100644 index 0000000000000..9da683b48b618 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/__test__/apm_telemetry.test.ts @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + AgentName, + APM_TELEMETRY_DOC_ID, + ApmTelemetry, + createApmTelementry, + getSavedObjectsClient, + storeApmTelemetry +} from '../apm_telemetry'; + +describe('apm_telemetry', () => { + describe('createApmTelementry', () => { + it('should create a ApmTelemetry object with boolean flag and frequency map of the given list of AgentNames', () => { + const apmTelemetry = createApmTelementry([ + AgentName.GoLang, + AgentName.NodeJs, + AgentName.GoLang, + AgentName.JsBase + ]); + expect(apmTelemetry.has_any_services).toBe(true); + expect(apmTelemetry.services_per_agent).toMatchObject({ + [AgentName.GoLang]: 2, + [AgentName.NodeJs]: 1, + [AgentName.JsBase]: 1 + }); + }); + it('should ignore undefined or unknown AgentName values', () => { + const apmTelemetry = createApmTelementry([ + AgentName.GoLang, + AgentName.NodeJs, + AgentName.GoLang, + AgentName.JsBase, + 'example-platform' as any, + undefined as any + ]); + expect(apmTelemetry.services_per_agent).toMatchObject({ + [AgentName.GoLang]: 2, + [AgentName.NodeJs]: 1, + [AgentName.JsBase]: 1 + }); + }); + }); + + describe('storeApmTelemetry', () => { + let server: any; + let apmTelemetry: ApmTelemetry; + let savedObjectsClientInstance: any; + + beforeEach(() => { + savedObjectsClientInstance = { create: jest.fn() }; + const callWithInternalUser = jest.fn(); + const internalRepository = jest.fn(); + server = { + savedObjects: { + SavedObjectsClient: jest.fn(() => savedObjectsClientInstance), + getSavedObjectsRepository: jest.fn(() => internalRepository) + }, + plugins: { + elasticsearch: { + getCluster: jest.fn(() => ({ callWithInternalUser })) + } + } + }; + apmTelemetry = { + has_any_services: true, + services_per_agent: { + [AgentName.GoLang]: 2, + [AgentName.NodeJs]: 1, + [AgentName.JsBase]: 1 + } + }; + }); + + it('should call savedObjectsClient create with the given ApmTelemetry object', () => { + storeApmTelemetry(server, apmTelemetry); + expect(savedObjectsClientInstance.create.mock.calls[0][1]).toBe( + apmTelemetry + ); + }); + + it('should call savedObjectsClient create with the apm-telemetry document type and ID', () => { + storeApmTelemetry(server, apmTelemetry); + expect(savedObjectsClientInstance.create.mock.calls[0][0]).toBe( + 'apm-telemetry' + ); + expect(savedObjectsClientInstance.create.mock.calls[0][2].id).toBe( + APM_TELEMETRY_DOC_ID + ); + }); + + it('should call savedObjectsClient create with overwrite: true', () => { + storeApmTelemetry(server, apmTelemetry); + expect(savedObjectsClientInstance.create.mock.calls[0][2].overwrite).toBe( + true + ); + }); + }); + + describe('getSavedObjectsClient', () => { + let server: any; + let savedObjectsClientInstance: any; + let callWithInternalUser: any; + let internalRepository: any; + + beforeEach(() => { + savedObjectsClientInstance = { create: jest.fn() }; + callWithInternalUser = jest.fn(); + internalRepository = jest.fn(); + server = { + savedObjects: { + SavedObjectsClient: jest.fn(() => savedObjectsClientInstance), + getSavedObjectsRepository: jest.fn(() => internalRepository) + }, + plugins: { + elasticsearch: { + getCluster: jest.fn(() => ({ callWithInternalUser })) + } + } + }; + }); + + it('should use internal user "admin"', () => { + getSavedObjectsClient(server); + + expect(server.plugins.elasticsearch.getCluster).toHaveBeenCalledWith( + 'admin' + ); + }); + + it('should call getSavedObjectsRepository with a cluster using the internal user context', () => { + getSavedObjectsClient(server); + + expect( + server.savedObjects.getSavedObjectsRepository + ).toHaveBeenCalledWith(callWithInternalUser); + }); + + it('should return a SavedObjectsClient initialized with the saved objects internal repository', () => { + const result = getSavedObjectsClient(server); + + expect(result).toBe(savedObjectsClientInstance); + expect(server.savedObjects.SavedObjectsClient).toHaveBeenCalledWith( + internalRepository + ); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/apm_telemetry.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/apm_telemetry.ts new file mode 100644 index 0000000000000..f136030dd5652 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/apm_telemetry.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Server } from 'hapi'; +import { countBy } from 'lodash'; + +// Support telemetry for additional agent types by appending definitions in +// mappings.json and the AgentName enum. + +export enum AgentName { + Python = 'python', + Java = 'java', + NodeJs = 'nodejs', + JsBase = 'js-base', + Ruby = 'ruby', + GoLang = 'go' +} + +export interface ApmTelemetry { + has_any_services: boolean; + services_per_agent: { [agentName in AgentName]?: number }; +} + +export const APM_TELEMETRY_DOC_ID = 'apm-telemetry'; + +export function createApmTelementry( + agentNames: AgentName[] = [] +): ApmTelemetry { + const validAgentNames = agentNames.filter(agentName => + Object.values(AgentName).includes(agentName) + ); + return { + has_any_services: validAgentNames.length > 0, + services_per_agent: countBy(validAgentNames) + }; +} + +export function storeApmTelemetry( + server: Server, + apmTelemetry: ApmTelemetry +): void { + const savedObjectsClient = getSavedObjectsClient(server); + savedObjectsClient.create('apm-telemetry', apmTelemetry, { + id: APM_TELEMETRY_DOC_ID, + overwrite: true + }); +} + +export function getSavedObjectsClient(server: Server): any { + const { SavedObjectsClient, getSavedObjectsRepository } = server.savedObjects; + const { callWithInternalUser } = server.plugins.elasticsearch.getCluster( + 'admin' + ); + const internalRepository = getSavedObjectsRepository(callWithInternalUser); + return new SavedObjectsClient(internalRepository); +} diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts new file mode 100644 index 0000000000000..96325952fb1a7 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { + ApmTelemetry, + AgentName, + storeApmTelemetry, + createApmTelementry, + APM_TELEMETRY_DOC_ID +} from './apm_telemetry'; +export { makeApmUsageCollector } from './make_apm_usage_collector'; diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/make_apm_usage_collector.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/make_apm_usage_collector.ts new file mode 100644 index 0000000000000..d09ea4cdab51e --- /dev/null +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/make_apm_usage_collector.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Server } from 'hapi'; +import { + APM_TELEMETRY_DOC_ID, + ApmTelemetry, + createApmTelementry, + getSavedObjectsClient +} from './apm_telemetry'; + +// TODO this type should be defined by the platform +interface KibanaHapiServer extends Server { + usage: { + collectorSet: { + makeUsageCollector: any; + register: any; + }; + }; +} + +export function makeApmUsageCollector(server: KibanaHapiServer): void { + const apmUsageCollector = server.usage.collectorSet.makeUsageCollector({ + type: 'apm', + fetch: async (): Promise => { + const savedObjectsClient = getSavedObjectsClient(server); + try { + const apmTelemetrySavedObject = await savedObjectsClient.get( + 'apm-telemetry', + APM_TELEMETRY_DOC_ID + ); + return apmTelemetrySavedObject.attributes; + } catch (err) { + return createApmTelementry(); + } + } + }); + server.usage.collectorSet.register(apmUsageCollector); +} diff --git a/x-pack/plugins/apm/server/routes/services.ts b/x-pack/plugins/apm/server/routes/services.ts index f45cd21a7e625..ef7bbf6c86c86 100644 --- a/x-pack/plugins/apm/server/routes/services.ts +++ b/x-pack/plugins/apm/server/routes/services.ts @@ -6,6 +6,11 @@ import Boom from 'boom'; import { Server } from 'hapi'; +import { + AgentName, + createApmTelementry, + storeApmTelemetry +} from '../lib/apm_telemetry'; import { withDefaultValidators } from '../lib/helpers/input_validation'; import { setupRequest } from '../lib/helpers/setup_request'; import { getService } from '../lib/services/get_service'; @@ -30,9 +35,23 @@ export function initServicesApi(server: Server) { query: withDefaultValidators() } }, - handler: req => { + handler: async req => { const { setup } = req.pre; - return getServices(setup).catch(defaultErrorHandler); + + let serviceBucketList; + try { + serviceBucketList = await getServices(setup); + } catch (error) { + return defaultErrorHandler(error); + } + + // Store telemetry data derived from serviceBucketList + const apmTelemetry = createApmTelementry( + serviceBucketList.map(({ agentName }) => agentName as AgentName) + ); + storeApmTelemetry(server, apmTelemetry); + + return serviceBucketList; } });