[Usage Collection] Report nodes feature usage

src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js

@@ -19,11 +19,12 @@
 
 import expect from '@kbn/expect';
 import sinon from 'sinon';
+import { merge, omit } from 'lodash';
 
+import { TIMEOUT } from '../constants';
 import { mockGetClusterInfo } from './get_cluster_info';
 import { mockGetClusterStats } from './get_cluster_stats';
 
-import { omit } from 'lodash';
 import { getLocalStats, handleLocalStats } from '../get_local_stats';
 
 const mockUsageCollection = (kibanaUsage = {}) => ({
@@ -51,10 +52,26 @@ const getMockServer = (getCluster = sinon.stub()) => ({
     elasticsearch: { getCluster },
   },
 });
+function mockGetNodesUsage(callCluster, nodesUsage, req) {
+  callCluster
+    .withArgs(
+      req,
+      {
+        method: 'GET',
+        path: '/_nodes/usage',
+        query: {
+          timeout: TIMEOUT,
+        },
+      },
+      'transport.request'
+    )
+    .returns(nodesUsage);
+}
 
-function mockGetLocalStats(callCluster, clusterInfo, clusterStats, req) {
+function mockGetLocalStats(callCluster, clusterInfo, clusterStats, nodesUsage, req) {
   mockGetClusterInfo(callCluster, clusterInfo, req);
   mockGetClusterStats(callCluster, clusterStats, req);
+  mockGetNodesUsage(callCluster, nodesUsage, req);
 }
 
 describe('get_local_stats', () => {
@@ -68,13 +85,36 @@ describe('get_local_stats', () => {
       number: version,
     },
   };
+  const nodesUsage = [
+    {
+      node_id: 'some_node_id',
+      timestamp: 1588617023177,
+      since: 1588616945163,
+      rest_actions: {
+        nodes_usage_action: 1,
+        create_index_action: 1,
+        document_get_action: 1,
+        search_action: 19,
+        nodes_info_action: 36,
+      },
+      aggregations: {
+        terms: {
+          bytes: 2,
+        },
+        scripted_metric: {
+          other: 7,
+        },
+      },
+    },
+  ];
   const clusterStats = {
     _nodes: { failed: 123 },
     cluster_name: 'real-cool',
     indices: { totally: 456 },
     nodes: { yup: 'abc' },
     random: 123,
   };
+
   const kibana = {
     kibana: {
       great: 'googlymoogly',
@@ -97,12 +137,16 @@ describe('get_local_stats', () => {
     snow: { chances: 0 },
   };
 
+  const clusterStatsWithNodesUsage = {
+    ...clusterStats,
+    nodes: merge(clusterStats.nodes, { usage: nodesUsage }),
+  };
   const combinedStatsResult = {
     collection: 'local',
     cluster_uuid: clusterUuid,
     cluster_name: clusterName,
     version,
-    cluster_stats: omit(clusterStats, '_nodes', 'cluster_name'),
+    cluster_stats: omit(clusterStatsWithNodesUsage, '_nodes', 'cluster_name'),
     stack_stats: {
       kibana: {
         great: 'googlymoogly',
@@ -135,7 +179,7 @@ describe('get_local_stats', () => {
 
   describe('handleLocalStats', () => {
     it('returns expected object without xpack and kibana data', () => {
-      const result = handleLocalStats(clusterInfo, clusterStats, void 0, context);
+      const result = handleLocalStats(clusterInfo, clusterStatsWithNodesUsage, void 0, context);
       expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
       expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
       expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
@@ -146,7 +190,7 @@ describe('get_local_stats', () => {
     });
 
     it('returns expected object with xpack', () => {
-      const result = handleLocalStats(clusterInfo, clusterStats, void 0, context);
+      const result = handleLocalStats(clusterInfo, clusterStatsWithNodesUsage, void 0, context);
       const { stack_stats: stack, ...cluster } = result;
       expect(cluster.collection).to.be(combinedStatsResult.collection);
       expect(cluster.cluster_uuid).to.be(combinedStatsResult.cluster_uuid);
@@ -167,7 +211,8 @@ describe('get_local_stats', () => {
       mockGetLocalStats(
         callClusterUsageFailed,
         Promise.resolve(clusterInfo),
-        Promise.resolve(clusterStats)
+        Promise.resolve(clusterStats),
+        Promise.resolve(nodesUsage)
       );
       const result = await getLocalStats([], {
         server: getMockServer(),
@@ -177,6 +222,7 @@ describe('get_local_stats', () => {
       expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
       expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
       expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
+      expect(result.cluster_stats.nodes).to.eql(combinedStatsResult.cluster_stats.nodes);
       expect(result.version).to.be('2.3.4');
       expect(result.collection).to.be('local');
 
@@ -188,7 +234,12 @@ describe('get_local_stats', () => {
     it('returns expected object with xpack and kibana data', async () => {
       const callCluster = sinon.stub();
       const usageCollection = mockUsageCollection(kibana);
-      mockGetLocalStats(callCluster, Promise.resolve(clusterInfo), Promise.resolve(clusterStats));
+      mockGetLocalStats(
+        callCluster,
+        Promise.resolve(clusterInfo),
+        Promise.resolve(clusterStats),
+        Promise.resolve(nodesUsage)
+      );
 
       const result = await getLocalStats([], {
         server: getMockServer(callCluster),

src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts

@@ -21,9 +21,11 @@ import {
   StatsGetter,
   StatsCollectionContext,
 } from 'src/plugins/telemetry_collection_manager/server';
+import { merge } from 'lodash';
 import { getClusterInfo, ESClusterInfo } from './get_cluster_info';
 import { getClusterStats } from './get_cluster_stats';
 import { getKibana, handleKibanaStats, KibanaUsageStats } from './get_kibana';
+import { getNodesUsage } from './get_nodes_usage';
 
 /**
  * Handle the separate local calls by combining them into a single object response that looks like the
@@ -67,12 +69,21 @@ export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async (
 
   return await Promise.all(
     clustersDetails.map(async (clustersDetail) => {
-      const [clusterInfo, clusterStats, kibana] = await Promise.all([
+      const [clusterInfo, clusterStats, nodesUsage, kibana] = await Promise.all([
         getClusterInfo(callCluster), // cluster info
         getClusterStats(callCluster), // cluster stats (not to be confused with cluster _state_)
+        getNodesUsage(callCluster), // nodes_usage info
         getKibana(usageCollection, callCluster),
       ]);
-      return handleLocalStats(clusterInfo, clusterStats, kibana, context);
+      return handleLocalStats(
+        clusterInfo,
+        {
+          ...clusterStats,
+          nodes: merge(clusterStats.nodes, { usage: nodesUsage }),
+        },
+        kibana,
+        context
+      );
     })
   );
 };

src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts

@@ -0,0 +1,108 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { getNodesUsage } from './get_nodes_usage';
+import { TIMEOUT } from './constants';
+
+const mockedNodesResponse = {
+  node_id: 'some_node_id',
+  timestamp: 1588617023177,
+  since: 1588616945163,
+  rest_actions: {
+    nodes_usage_action: 1,
+    create_index_action: 1,
+    document_get_action: 1,
+    search_action: 19,
+    nodes_info_action: 36,
+  },
+  aggregations: {
+    terms: {
+      bytes: 2,
+    },
+    scripted_metric: {
+      other: 7,
+    },
+  },
+};
+const mockedNodesFetchResponse = {
+  cluster_name: 'test cluster',
+  nodes: {
+    some_node_id: {
+      timestamp: 1588617023177,
+      since: 1588616945163,
+      rest_actions: {
+        nodes_usage_action: 1,
+        create_index_action: 1,
+        document_get_action: 1,
+        search_action: 19,
+        nodes_info_action: 36,
+      },
+      aggregations: {
+        terms: {
+          bytes: 2,
+        },
+        scripted_metric: {
+          other: 7,
+        },
+      },
+    },
+  },
+};
+describe('get_nodes_usage', () => {
+  it('calls fetchNodesUsage', async () => {
+    const callCluster = jest.fn();
+    callCluster.mockResolvedValueOnce(mockedNodesFetchResponse);
+    jest.fn(async () => ({
+      cluster_name: 'test',
+      nodes: mockedNodesResponse,
+    }));
+    await getNodesUsage(callCluster);
+    expect(callCluster).toHaveBeenCalledWith('transport.request', {
+      path: '/_nodes/usage',
+      method: 'GET',
+      query: {
+        timeout: TIMEOUT,
+      },
+    });
+  });
+  it('returns a modified array of node usage data', async () => {
+    const callCluster = jest.fn();
+    callCluster.mockResolvedValueOnce(mockedNodesFetchResponse);
+    jest.fn(async () => ({
+      cluster_name: 'test',
+      nodes: mockedNodesResponse,
+    }));
+    const result = await getNodesUsage(callCluster);
+    expect(result.nodes).toEqual([
+      {
+        aggregations: { scripted_metric: { other: 7 }, terms: { bytes: 2 } },
+        node_id: 'some_node_id',
+        rest_actions: {
+          create_index_action: 1,
+          document_get_action: 1,
+          nodes_info_action: 36,
+          nodes_usage_action: 1,
+          search_action: 19,
+        },
+        since: 1588616945163,
+        timestamp: 1588617023177,
+      },
+    ]);
+  });
+});

src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts

@@ -0,0 +1,88 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { APICaller } from 'kibana/server';
+import { TIMEOUT } from './constants';
+
+export interface NodeAggregation {
+  [key: string]: number;
+}
+
+// we set aggregations as an optional type because it was only added in v7.8.0
+export interface NodeObj {
+  node_id?: string;
+  timestamp: number;
+  since: number;
+  rest_actions: {
+    [key: string]: number;
+  };
+  aggregations?: {
+    [key: string]: NodeAggregation;
+  };
+}
+export interface NodesStats {
+  total: number;
+  successful: number;
+  failed: number;
+}
+export interface NodesFeatureUsageResponse {
+  cluster_name: string;
+  nodes: {
+    [key: string]: NodeObj;
+  };
+}
+
+export type NodesUsageFetcher = (callCluster: APICaller) => Promise<NodesFeatureUsageResponse>;
+
+export type NodesUsageGetter = (
+  callCluster: APICaller
+) => Promise<{ nodes: NodeObj[] | Array<{}> }>;
+/**
+ * Get the nodes usage data from the connected cluster.
+ *
+ * This is the equivalent to GET /_nodes/usage?timeout=30s.
+ *
+ * The Nodes usage API was introduced in v6.0.0
+ */
+export async function fetchNodesUsage<NodesUsageFetcher>(callCluster: APICaller) {
+  const response = await callCluster('transport.request', {
+    method: 'GET',
+    path: '/_nodes/usage',
+    query: {
+      timeout: TIMEOUT,
+    },
+  });
+  return response;
+}
+
+/**
+ * Get the nodes usage from the connected cluster
+ * @param callCluster APICaller
+ * @returns Object containing array of modified usage information with the node_id nested within the data for that node.
+ */
+export const getNodesUsage: NodesUsageGetter = async (callCluster) => {
+  const result = await fetchNodesUsage<NodesUsageFetcher>(callCluster);
+  let transformedNodes = [{}];
+  if (result && result.nodes) {
+    transformedNodes = Object.entries(result.nodes).map(([key, value]) => ({
+      ...(value as NodeObj),
+      node_id: key,
+    }));
+  }
+  return { nodes: transformedNodes };
+};

test/api_integration/apis/telemetry/telemetry_local.js

@@ -113,6 +113,7 @@ export default function ({ getService }) {
         'cluster_stats.nodes.plugins',
         'cluster_stats.nodes.process',
         'cluster_stats.nodes.versions',
+        'cluster_stats.nodes.usage',
         'cluster_stats.status',
         'cluster_stats.timestamp',
         'cluster_uuid',