[Telemetry] Pull local Kibana usage stats (#26496) (#26913)

* add kibana stats

* fix tests

* format the stats for telemetry

* fix the os/platform stats

* add version to locally-source kibana telemetry stats

* use callWithInternalUser

* better get_kibana module unit test verification

* separate handleKibanaStats

* variable rename

* fix comment

* fix functional test

* keep the return object literal from handleLocalStats

* validate the payload fields

* add warning log if no kibana stats returned

x-pack/plugins/xpack_main/server/lib/telemetry/local/__tests__/get_local_stats.js

@@ -18,18 +18,24 @@ import {
   handleLocalStats,
 } from '../get_local_stats';
 
+const getMockServer = (getCluster = sinon.stub(), kibanaUsage = {}) => ({
+  log(tags, message) {
+    console.log({ tags, message });
+  },
+  usage: { collectorSet: { bulkFetch: () => kibanaUsage, toObject: data => data } },
+  plugins: {
+    xpack_main: { status: { plugin: { kbnServer: { version: '8675309-snapshot' } } } },
+    elasticsearch: { getCluster },
+  },
+});
+
 function mockGetLocalStats(callCluster, clusterInfo, clusterStats, license, usage) {
   mockGetClusterInfo(callCluster, clusterInfo);
   mockGetClusterStats(callCluster, clusterStats);
   mockGetXPack(callCluster, license, usage);
 }
 
-function dropTimestamp(localStats) {
-  return omit(localStats, 'timestamp');
-}
-
 describe('get_local_stats', () => {
-
   const clusterUuid = 'abc123';
   const clusterName = 'my-cool-cluster';
   const version = '2.3.4';
@@ -47,102 +53,127 @@ describe('get_local_stats', () => {
     nodes: { yup: 'abc' },
     random: 123
   };
-  const license = {
-    fancy: 'license'
-  };
-  const usage = {
-    also: 'fancy'
+  const license = { fancy: 'license' };
+  const xpack = { also: 'fancy' };
+  const kibana = {
+    kibana: {
+      great: 'googlymoogly',
+      versions: [{ version: '8675309', count: 1 }]
+    },
+    kibana_stats: {
+      os: {
+        platform: 'rocky',
+        platformRelease: 'iv',
+      }
+    },
+    sun: { chances: 5 },
+    clouds: { chances: 95 },
+    rain: { chances: 2 },
+    snow: { chances: 0 },
   };
-  const xpack = {
-    license,
-    stack_stats: {
-      xpack: usage
-    }
-  };
-  const localStats = {
+
+  const combinedStatsResult = {
     collection: 'local',
     cluster_uuid: clusterUuid,
     cluster_name: clusterName,
+    license: {
+      fancy: 'license'
+    },
     version,
     cluster_stats: omit(clusterStats, '_nodes', 'cluster_name'),
-    ...xpack
+    stack_stats: {
+      kibana: {
+        great: 'googlymoogly',
+        count: 1,
+        indices: 1,
+        os: {
+          platforms: [{ platform: 'rocky', count: 1 }],
+          platformReleases: [{ platformRelease: 'iv', count: 1 }]
+        },
+        versions: [{ version: '8675309', count: 1 }],
+        plugins: {
+          sun: { chances: 5 },
+          clouds: { chances: 95 },
+          rain: { chances: 2 },
+          snow: { chances: 0 },
+        }
+      },
+      xpack: { also: 'fancy' },
+    }
   };
-  const noXpackLocalStats = omit(localStats, 'license', 'stack_stats');
 
   describe('handleLocalStats', () => {
-
-    it('returns expected object without xpack data', () => {
-      expect(dropTimestamp(handleLocalStats(clusterInfo, clusterStats))).to.eql(noXpackLocalStats);
-      expect(dropTimestamp(handleLocalStats(clusterInfo, clusterStats, { }))).to.eql(noXpackLocalStats);
+    it('returns expected object without xpack and kibana data', () => {
+      const result = handleLocalStats(getMockServer(), clusterInfo, clusterStats);
+      expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
+      expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
+      expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
+      expect(result.version).to.be('2.3.4');
+      expect(result.collection).to.be('local');
+      expect(result.license).to.be(undefined);
+      expect(result.stack_stats).to.eql({ kibana: undefined, xpack: undefined });
     });
 
-    it('returns expected object with xpack data', () => {
-      expect(dropTimestamp(handleLocalStats(clusterInfo, clusterStats, xpack))).to.eql(localStats);
+    it('returns expected object with xpack', () => {
+      const result = handleLocalStats(getMockServer(), clusterInfo, clusterStats, license, xpack);
+      const { stack_stats: stack, ...cluster } = result;
+      expect(cluster.collection).to.be(combinedStatsResult.collection);
+      expect(cluster.cluster_uuid).to.be(combinedStatsResult.cluster_uuid);
+      expect(cluster.cluster_name).to.be(combinedStatsResult.cluster_name);
+      expect(stack.kibana).to.be(undefined); // not mocked for this test
+
+      expect(cluster.version).to.eql(combinedStatsResult.version);
+      expect(cluster.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
+      expect(cluster.license).to.eql(combinedStatsResult.license);
+      expect(stack.xpack).to.eql(combinedStatsResult.stack_stats.xpack);
     });
-
   });
 
   describe('getLocalStatsWithCaller', () => {
-
     it('returns expected object without xpack data when X-Pack fails to respond', async () => {
       const callClusterUsageFailed = sinon.stub();
-      const callClusterLicenseFailed = sinon.stub();
-      const callClusterBothFailed = sinon.stub();
 
       mockGetLocalStats(
         callClusterUsageFailed,
         Promise.resolve(clusterInfo),
         Promise.resolve(clusterStats),
-        Promise.resolve(license), Promise.reject('usage failed')
+        Promise.resolve(license),
+        Promise.reject('usage failed')
       );
 
-      mockGetLocalStats(
-        callClusterLicenseFailed,
-        Promise.resolve(clusterInfo),
-        Promise.resolve(clusterStats),
-        Promise.reject('license failed'), Promise.resolve(usage)
-      );
+      const result = await getLocalStatsWithCaller(getMockServer(), callClusterUsageFailed);
+      expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
+      expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
+      expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
+      expect(result.version).to.be('2.3.4');
+      expect(result.collection).to.be('local');
 
-      mockGetLocalStats(
-        callClusterBothFailed,
-        Promise.resolve(clusterInfo),
-        Promise.resolve(clusterStats),
-        Promise.reject('license failed'), Promise.reject('usage failed')
-      );
-
-      expect(dropTimestamp(await getLocalStatsWithCaller(callClusterUsageFailed))).to.eql(noXpackLocalStats);
-      expect(dropTimestamp(await getLocalStatsWithCaller(callClusterLicenseFailed))).to.eql(noXpackLocalStats);
-      expect(dropTimestamp(await getLocalStatsWithCaller(callClusterBothFailed))).to.eql(noXpackLocalStats);
+      // license and xpack usage info come from the same cluster call
+      expect(result.license).to.be(undefined);
+      expect(result.stack_stats.xpack).to.be(undefined);
     });
 
-    it('returns expected object with xpack data', async () => {
+    it('returns expected object with xpack and kibana data', async () => {
       const callCluster = sinon.stub();
 
       mockGetLocalStats(
         callCluster,
         Promise.resolve(clusterInfo),
         Promise.resolve(clusterStats),
-        Promise.resolve(license), Promise.resolve(usage)
+        Promise.resolve(license),
+        Promise.resolve(xpack)
       );
 
-      expect(dropTimestamp(await getLocalStatsWithCaller(callCluster))).to.eql(localStats);
+      const result = await getLocalStatsWithCaller(getMockServer(callCluster, kibana), callCluster);
+      expect(result.stack_stats.xpack).to.eql(combinedStatsResult.stack_stats.xpack);
+      expect(result.stack_stats.kibana).to.eql(combinedStatsResult.stack_stats.kibana);
     });
-
   });
 
   describe('getLocalStats', () => {
-
     it('uses callWithInternalUser from data cluster', async () => {
       const getCluster = sinon.stub();
-      const req = {
-        server: {
-          plugins: {
-            elasticsearch: {
-              getCluster
-            }
-          }
-        }
-      };
+      const req = { server: getMockServer(getCluster) };
       const callWithInternalUser = sinon.stub();
 
       getCluster.withArgs('data').returns({ callWithInternalUser });
@@ -151,12 +182,15 @@ describe('get_local_stats', () => {
         callWithInternalUser,
         Promise.resolve(clusterInfo),
         Promise.resolve(clusterStats),
-        Promise.resolve(license), Promise.resolve(usage)
+        Promise.resolve(license),
+        Promise.resolve(xpack)
       );
 
-      expect(dropTimestamp(await getLocalStats(req))).to.eql(localStats);
+      const result = await getLocalStats(req);
+      expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
+      expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
+      expect(result.version).to.eql(combinedStatsResult.version);
+      expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
     });
-
   });
-
 });

x-pack/plugins/xpack_main/server/lib/telemetry/local/__tests__/get_xpack.js

@@ -93,15 +93,15 @@ describe('get_xpack', () => {
 
     it('returns the formatted response object', async () => {
       const license = { fancy: 'license' };
-      const usage = { also: 'fancy' };
+      const xpack = { also: 'fancy' };
 
       const callCluster = sinon.stub();
 
-      mockGetXPack(callCluster, Promise.resolve(license), Promise.resolve(usage));
+      mockGetXPack(callCluster, Promise.resolve(license), Promise.resolve(xpack));
 
       const data = await getXPack(callCluster);
 
-      expect(data).to.eql({ license, stack_stats: { xpack: usage } });
+      expect(data).to.eql({ license, xpack });
     });
 
     it('returns empty object upon license failure', async () => {

x-pack/plugins/xpack_main/server/lib/telemetry/local/get_kibana.js

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { get, omit } from 'lodash';
+
+export function handleKibanaStats(server, response) {
+  if (!response) {
+    server.log(['warning', 'telemetry', 'local-stats'], 'No Kibana stats returned from usage collectors');
+    return;
+  }
+
+  const { kibana, kibana_stats: stats, ...plugins } = response;
+
+  const platform = get(stats, 'os.platform', 'unknown');
+  const platformRelease = get(stats, 'os.platformRelease', 'unknown');
+
+  let version;
+  const { kbnServer } = get(server, 'plugins.xpack_main.status.plugin');
+  if (kbnServer) {
+    version = kbnServer.version.replace(/-snapshot/i, '');
+  }
+
+  // combine core stats (os types, saved objects) with plugin usage stats
+  // organize the object into the same format as monitoring-enabled telemetry
+  return {
+    ...omit(kibana, 'index'), // discard index
+    count: 1,
+    indices: 1,
+    os: {
+      platforms: [{ platform, count: 1 }],
+      platformReleases: [{ platformRelease, count: 1 }],
+    },
+    versions: [{ version, count: 1 }],
+    plugins,
+  };
+}
+
+/*
+ * Check user privileges for read access to monitoring
+ * Pass callWithInternalUser to bulkFetchUsage
+ */
+export async function getKibana(server, callWithInternalUser) {
+  const { collectorSet } = server.usage;
+  const usage = await collectorSet.bulkFetch(callWithInternalUser);
+  return collectorSet.toObject(usage);
+}

x-pack/plugins/xpack_main/server/lib/telemetry/local/get_local_stats.js

@@ -8,6 +8,7 @@ import { get, omit } from 'lodash';
 import { getClusterInfo } from './get_cluster_info';
 import { getClusterStats } from './get_cluster_stats';
 import { getXPack } from './get_xpack';
+import { getKibana, handleKibanaStats } from './get_kibana';
 
 /**
  * Handle the separate local calls by combining them into a single object response that looks like the
@@ -18,15 +19,19 @@ import { getXPack } from './get_xpack';
  * @param {Object} xpack License and X-Pack details
  * @return {Object} A combined object containing the different responses.
  */
-export function handleLocalStats(clusterInfo, clusterStats, xpack) {
+export function handleLocalStats(server, clusterInfo, clusterStats, license, xpack, kibana) {
   return {
     timestamp: (new Date()).toISOString(),
     cluster_uuid: get(clusterInfo, 'cluster_uuid'),
     cluster_name: get(clusterInfo, 'cluster_name'),
     version: get(clusterInfo, 'version.number'),
     cluster_stats: omit(clusterStats, '_nodes', 'cluster_name'),
     collection: 'local',
-    ...xpack
+    license,
+    stack_stats: {
+      kibana: handleKibanaStats(server, kibana),
+      xpack,
+    }
   };
 }
 
@@ -37,13 +42,16 @@ export function handleLocalStats(clusterInfo, clusterStats, xpack) {
  * @param {function} callCluster The callWithInternalUser handler (exposed for testing)
  * @return {Promise} The object containing the current Elasticsearch cluster's telemetry.
  */
-export function getLocalStatsWithCaller(callCluster) {
+export function getLocalStatsWithCaller(server, callCluster) {
   return Promise.all([
     getClusterInfo(callCluster),  // cluster info
     getClusterStats(callCluster), // cluster stats (not to be confused with cluster _state_)
-    getXPack(callCluster),        // license, stack_stats
-  ])
-    .then(([clusterInfo, clusterStats, xpack]) => handleLocalStats(clusterInfo, clusterStats, xpack));
+    getXPack(callCluster),        // { license, xpack }
+    getKibana(server, callCluster)
+  ]).then(([clusterInfo, clusterStats, { license, xpack }, kibana]) => {
+    return handleLocalStats(server, clusterInfo, clusterStats, license, xpack, kibana);
+  }
+  );
 }
 
 /**
@@ -53,7 +61,7 @@ export function getLocalStatsWithCaller(callCluster) {
  * @return {Promise} The cluster object containing telemetry.
  */
 export function getLocalStats(req) {
-  const { callWithInternalUser } = req.server.plugins.elasticsearch.getCluster('data');
-
-  return getLocalStatsWithCaller(callWithInternalUser);
+  const { server } = req;
+  const { callWithInternalUser } = server.plugins.elasticsearch.getCluster('data');
+  return getLocalStatsWithCaller(server, callWithInternalUser);
 }

x-pack/plugins/xpack_main/server/lib/telemetry/local/get_xpack.js

@@ -75,7 +75,12 @@ export function getXPack(callCluster) {
     getXPackLicense(callCluster),
     getXPackUsage(callCluster),
   ])
-    .then(([license, usage]) => handleXPack(license, usage))
+    .then(([license, xpack]) => {
+      return {
+        license,
+        xpack,
+      };
+    })
     .catch(() => { return {}; });
 }
 

x-pack/test/api_integration/apis/xpack_main/telemetry/index.js

@@ -7,5 +7,6 @@
 export default function ({ loadTestFile }) {
   describe('Telemetry', () => {
     loadTestFile(require.resolve('./telemetry'));
+    loadTestFile(require.resolve('./telemetry_local'));
   });
 }