[APM] adds telemetry to APM (#25513) (#25943)

* [APM] adds telemetry to APM

* [APM] Code and readability improvements for APM Telemetry

* [APM] fixed failing tests for apm-telemetry and service routes

* [APM] fix lint issues for APM Telemetry

x-pack/plugins/apm/index.js

@@ -10,6 +10,8 @@ import { initServicesApi } from './server/routes/services';
 import { initErrorsApi } from './server/routes/errors';
 import { initStatusApi } from './server/routes/status_check';
 import { initTracesApi } from './server/routes/traces';
+import mappings from './mappings';
+import { makeApmUsageCollector } from './server/lib/apm_telemetry';
 
 export function apm(kibana) {
   return new kibana.Plugin({
@@ -35,7 +37,13 @@ export function apm(kibana) {
           apmIndexPattern: config.get('apm_oss.indexPattern')
         };
       },
-      hacks: ['plugins/apm/hacks/toggle_app_link_in_nav']
+      hacks: ['plugins/apm/hacks/toggle_app_link_in_nav'],
+      savedObjectSchemas: {
+        'apm-telemetry': {
+          isNamespaceAgnostic: true
+        }
+      },
+      mappings
     },
 
     config(Joi) {
@@ -60,6 +68,7 @@ export function apm(kibana) {
       initServicesApi(server);
       initErrorsApi(server);
       initStatusApi(server);
+      makeApmUsageCollector(server);
     }
   });
 }

x-pack/plugins/apm/mappings.json

@@ -0,0 +1,37 @@
+{
+  "apm-telemetry": {
+    "properties": {
+      "has_any_services": {
+        "type": "boolean"
+      },
+      "services_per_agent": {
+        "properties": {
+          "python": {
+            "type": "long",
+            "null_value": 0
+          },
+          "java": {
+            "type": "long",
+            "null_value": 0
+          },
+          "nodejs": {
+            "type": "long",
+            "null_value": 0
+          },
+          "js-base": {
+            "type": "long",
+            "null_value": 0
+          },
+          "ruby": {
+            "type": "long",
+            "null_value": 0
+          },
+          "go": {
+            "type": "long",
+            "null_value": 0
+          }
+        }
+      }
+    }
+  }
+}

x-pack/plugins/apm/server/lib/apm_telemetry/__test__/apm_telemetry.test.ts

@@ -0,0 +1,152 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import {
+  AgentName,
+  APM_TELEMETRY_DOC_ID,
+  ApmTelemetry,
+  createApmTelementry,
+  getSavedObjectsClient,
+  storeApmTelemetry
+} from '../apm_telemetry';
+
+describe('apm_telemetry', () => {
+  describe('createApmTelementry', () => {
+    it('should create a ApmTelemetry object with boolean flag and frequency map of the given list of AgentNames', () => {
+      const apmTelemetry = createApmTelementry([
+        AgentName.GoLang,
+        AgentName.NodeJs,
+        AgentName.GoLang,
+        AgentName.JsBase
+      ]);
+      expect(apmTelemetry.has_any_services).toBe(true);
+      expect(apmTelemetry.services_per_agent).toMatchObject({
+        [AgentName.GoLang]: 2,
+        [AgentName.NodeJs]: 1,
+        [AgentName.JsBase]: 1
+      });
+    });
+    it('should ignore undefined or unknown AgentName values', () => {
+      const apmTelemetry = createApmTelementry([
+        AgentName.GoLang,
+        AgentName.NodeJs,
+        AgentName.GoLang,
+        AgentName.JsBase,
+        'example-platform' as any,
+        undefined as any
+      ]);
+      expect(apmTelemetry.services_per_agent).toMatchObject({
+        [AgentName.GoLang]: 2,
+        [AgentName.NodeJs]: 1,
+        [AgentName.JsBase]: 1
+      });
+    });
+  });
+
+  describe('storeApmTelemetry', () => {
+    let server: any;
+    let apmTelemetry: ApmTelemetry;
+    let savedObjectsClientInstance: any;
+
+    beforeEach(() => {
+      savedObjectsClientInstance = { create: jest.fn() };
+      const callWithInternalUser = jest.fn();
+      const internalRepository = jest.fn();
+      server = {
+        savedObjects: {
+          SavedObjectsClient: jest.fn(() => savedObjectsClientInstance),
+          getSavedObjectsRepository: jest.fn(() => internalRepository)
+        },
+        plugins: {
+          elasticsearch: {
+            getCluster: jest.fn(() => ({ callWithInternalUser }))
+          }
+        }
+      };
+      apmTelemetry = {
+        has_any_services: true,
+        services_per_agent: {
+          [AgentName.GoLang]: 2,
+          [AgentName.NodeJs]: 1,
+          [AgentName.JsBase]: 1
+        }
+      };
+    });
+
+    it('should call savedObjectsClient create with the given ApmTelemetry object', () => {
+      storeApmTelemetry(server, apmTelemetry);
+      expect(savedObjectsClientInstance.create.mock.calls[0][1]).toBe(
+        apmTelemetry
+      );
+    });
+
+    it('should call savedObjectsClient create with the apm-telemetry document type and ID', () => {
+      storeApmTelemetry(server, apmTelemetry);
+      expect(savedObjectsClientInstance.create.mock.calls[0][0]).toBe(
+        'apm-telemetry'
+      );
+      expect(savedObjectsClientInstance.create.mock.calls[0][2].id).toBe(
+        APM_TELEMETRY_DOC_ID
+      );
+    });
+
+    it('should call savedObjectsClient create with overwrite: true', () => {
+      storeApmTelemetry(server, apmTelemetry);
+      expect(savedObjectsClientInstance.create.mock.calls[0][2].overwrite).toBe(
+        true
+      );
+    });
+  });
+
+  describe('getSavedObjectsClient', () => {
+    let server: any;
+    let savedObjectsClientInstance: any;
+    let callWithInternalUser: any;
+    let internalRepository: any;
+
+    beforeEach(() => {
+      savedObjectsClientInstance = { create: jest.fn() };
+      callWithInternalUser = jest.fn();
+      internalRepository = jest.fn();
+      server = {
+        savedObjects: {
+          SavedObjectsClient: jest.fn(() => savedObjectsClientInstance),
+          getSavedObjectsRepository: jest.fn(() => internalRepository)
+        },
+        plugins: {
+          elasticsearch: {
+            getCluster: jest.fn(() => ({ callWithInternalUser }))
+          }
+        }
+      };
+    });
+
+    it('should use internal user "admin"', () => {
+      getSavedObjectsClient(server);
+
+      expect(server.plugins.elasticsearch.getCluster).toHaveBeenCalledWith(
+        'admin'
+      );
+    });
+
+    it('should call getSavedObjectsRepository with a cluster using the internal user context', () => {
+      getSavedObjectsClient(server);
+
+      expect(
+        server.savedObjects.getSavedObjectsRepository
+      ).toHaveBeenCalledWith(callWithInternalUser);
+    });
+
+    it('should return a SavedObjectsClient initialized with the saved objects internal repository', () => {
+      const result = getSavedObjectsClient(server);
+
+      expect(result).toBe(savedObjectsClientInstance);
+      expect(server.savedObjects.SavedObjectsClient).toHaveBeenCalledWith(
+        internalRepository
+      );
+    });
+  });
+});

x-pack/plugins/apm/server/lib/apm_telemetry/apm_telemetry.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Server } from 'hapi';
+import { countBy } from 'lodash';
+
+// Support telemetry for additional agent types by appending definitions in
+// mappings.json and the AgentName enum.
+
+export enum AgentName {
+  Python = 'python',
+  Java = 'java',
+  NodeJs = 'nodejs',
+  JsBase = 'js-base',
+  Ruby = 'ruby',
+  GoLang = 'go'
+}
+
+export interface ApmTelemetry {
+  has_any_services: boolean;
+  services_per_agent: { [agentName in AgentName]?: number };
+}
+
+export const APM_TELEMETRY_DOC_ID = 'apm-telemetry';
+
+export function createApmTelementry(
+  agentNames: AgentName[] = []
+): ApmTelemetry {
+  const validAgentNames = agentNames.filter(agentName =>
+    Object.values(AgentName).includes(agentName)
+  );
+  return {
+    has_any_services: validAgentNames.length > 0,
+    services_per_agent: countBy(validAgentNames)
+  };
+}
+
+export function storeApmTelemetry(
+  server: Server,
+  apmTelemetry: ApmTelemetry
+): void {
+  const savedObjectsClient = getSavedObjectsClient(server);
+  savedObjectsClient.create('apm-telemetry', apmTelemetry, {
+    id: APM_TELEMETRY_DOC_ID,
+    overwrite: true
+  });
+}
+
+export function getSavedObjectsClient(server: Server): any {
+  const { SavedObjectsClient, getSavedObjectsRepository } = server.savedObjects;
+  const { callWithInternalUser } = server.plugins.elasticsearch.getCluster(
+    'admin'
+  );
+  const internalRepository = getSavedObjectsRepository(callWithInternalUser);
+  return new SavedObjectsClient(internalRepository);
+}

x-pack/plugins/apm/server/lib/apm_telemetry/index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export {
+  ApmTelemetry,
+  AgentName,
+  storeApmTelemetry,
+  createApmTelementry,
+  APM_TELEMETRY_DOC_ID
+} from './apm_telemetry';
+export { makeApmUsageCollector } from './make_apm_usage_collector';

x-pack/plugins/apm/server/lib/apm_telemetry/make_apm_usage_collector.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Server } from 'hapi';
+import {
+  APM_TELEMETRY_DOC_ID,
+  ApmTelemetry,
+  createApmTelementry,
+  getSavedObjectsClient
+} from './apm_telemetry';
+
+// TODO this type should be defined by the platform
+interface KibanaHapiServer extends Server {
+  usage: {
+    collectorSet: {
+      makeUsageCollector: any;
+      register: any;
+    };
+  };
+}
+
+export function makeApmUsageCollector(server: KibanaHapiServer): void {
+  const apmUsageCollector = server.usage.collectorSet.makeUsageCollector({
+    type: 'apm',
+    fetch: async (): Promise<ApmTelemetry> => {
+      const savedObjectsClient = getSavedObjectsClient(server);
+      try {
+        const apmTelemetrySavedObject = await savedObjectsClient.get(
+          'apm-telemetry',
+          APM_TELEMETRY_DOC_ID
+        );
+        return apmTelemetrySavedObject.attributes;
+      } catch (err) {
+        return createApmTelementry();
+      }
+    }
+  });
+  server.usage.collectorSet.register(apmUsageCollector);
+}

x-pack/plugins/apm/server/routes/services.ts

@@ -6,6 +6,11 @@
 
 import Boom from 'boom';
 import { Server } from 'hapi';
+import {
+  AgentName,
+  createApmTelementry,
+  storeApmTelemetry
+} from '../lib/apm_telemetry';
 import { withDefaultValidators } from '../lib/helpers/input_validation';
 import { setupRequest } from '../lib/helpers/setup_request';
 import { getService } from '../lib/services/get_service';
@@ -30,9 +35,23 @@ export function initServicesApi(server: Server) {
         query: withDefaultValidators()
       }
     },
-    handler: req => {
+    handler: async req => {
       const { setup } = req.pre;
-      return getServices(setup).catch(defaultErrorHandler);
+
+      let serviceBucketList;
+      try {
+        serviceBucketList = await getServices(setup);
+      } catch (error) {
+        return defaultErrorHandler(error);
+      }
+
+      // Store telemetry data derived from serviceBucketList
+      const apmTelemetry = createApmTelementry(
+        serviceBucketList.map(({ agentName }) => agentName as AgentName)
+      );
+      storeApmTelemetry(server, apmTelemetry);
+
+      return serviceBucketList;
     }
   });