[Telemetry] Report data shippers

src/plugins/telemetry/server/index.ts

@@ -47,4 +47,8 @@ export {
   getLocalLicense,
   getLocalStats,
   TelemetryLocalStats,
+  INGEST_SOLUTIONS_ID,
+  IngestSolutionsIndex,
+  IngestSolutionsPayload,
+  buildIngestSolutionsPayload,
 } from './telemetry_collection';

src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js

@@ -135,23 +135,24 @@ describe('get_local_stats', () => {
 
   describe('handleLocalStats', () => {
     it('returns expected object without xpack and kibana data', () => {
-      const result = handleLocalStats(clusterInfo, clusterStats, void 0, context);
+      const result = handleLocalStats(clusterInfo, clusterStats, void 0, void 0, context);
       expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
       expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
       expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
       expect(result.version).to.be('2.3.4');
       expect(result.collection).to.be('local');
       expect(result.license).to.be(undefined);
-      expect(result.stack_stats).to.eql({ kibana: undefined });
+      expect(result.stack_stats).to.eql({ kibana: undefined, ingest_solutions: undefined });
     });
 
     it('returns expected object with xpack', () => {
-      const result = handleLocalStats(clusterInfo, clusterStats, void 0, context);
+      const result = handleLocalStats(clusterInfo, clusterStats, void 0, void 0, context);
       const { stack_stats: stack, ...cluster } = result;
       expect(cluster.collection).to.be(combinedStatsResult.collection);
       expect(cluster.cluster_uuid).to.be(combinedStatsResult.cluster_uuid);
       expect(cluster.cluster_name).to.be(combinedStatsResult.cluster_name);
       expect(stack.kibana).to.be(undefined); // not mocked for this test
+      expect(stack.ingest_solutions).to.be(undefined); // not mocked for this test
 
       expect(cluster.version).to.eql(combinedStatsResult.version);
       expect(cluster.cluster_stats).to.eql(combinedStatsResult.cluster_stats);

src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts

@@ -24,6 +24,11 @@ import {
 import { getClusterInfo, ESClusterInfo } from './get_cluster_info';
 import { getClusterStats } from './get_cluster_stats';
 import { getKibana, handleKibanaStats, KibanaUsageStats } from './get_kibana';
+import {
+  getIngestSolutions,
+  INGEST_SOLUTIONS_ID,
+  IngestSolutionsPayload,
+} from './ingest_solutions';
 
 /**
  * Handle the separate local calls by combining them into a single object response that looks like the
@@ -38,6 +43,7 @@ export function handleLocalStats(
   { cluster_name, cluster_uuid, version }: ESClusterInfo,
   { _nodes, cluster_name: clusterName, ...clusterStats }: any,
   kibana: KibanaUsageStats,
+  ingestSolutions: IngestSolutionsPayload,
   context: StatsCollectionContext
 ) {
   return {
@@ -48,6 +54,7 @@ export function handleLocalStats(
     cluster_stats: clusterStats,
     collection: 'local',
     stack_stats: {
+      [INGEST_SOLUTIONS_ID]: ingestSolutions,
       kibana: handleKibanaStats(context, kibana),
     },
   };
@@ -67,12 +74,13 @@ export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async (
 
   return await Promise.all(
     clustersDetails.map(async clustersDetail => {
-      const [clusterInfo, clusterStats, kibana] = await Promise.all([
+      const [clusterInfo, clusterStats, kibana, ingestSolutions] = await Promise.all([
         getClusterInfo(callCluster), // cluster info
         getClusterStats(callCluster), // cluster stats (not to be confused with cluster _state_)
         getKibana(usageCollection, callCluster),
+        getIngestSolutions(callCluster),
       ]);
-      return handleLocalStats(clusterInfo, clusterStats, kibana, context);
+      return handleLocalStats(clusterInfo, clusterStats, kibana, ingestSolutions, context);
     })
   );
 };

src/plugins/telemetry/server/telemetry_collection/index.ts

@@ -17,6 +17,12 @@
  * under the License.
  */
 
+export {
+  INGEST_SOLUTIONS_ID,
+  IngestSolutionsIndex,
+  IngestSolutionsPayload,
+  buildIngestSolutionsPayload,
+} from './ingest_solutions';
 export { getLocalStats, TelemetryLocalStats } from './get_local_stats';
 export { getLocalLicense } from './get_local_license';
 export { getClusterUuids } from './get_cluster_stats';

src/plugins/telemetry/server/telemetry_collection/ingest_solutions/constants.ts

@@ -0,0 +1,45 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export const INGEST_SOLUTIONS_ID = 'ingest_solutions';
+
+// TODO: Ideally this list should be updated from an external public URL (similar to the newsfeed)
+// But it's good to have a minimum list shipped with the build.
+export const INGEST_SOLUTIONS = [
+  // Observability
+  { name: 'apm', pattern: 'apm-*' },
+  { name: 'metricbeat', pattern: 'metricbeat-*' },
+  { name: 'heartbeat', pattern: 'heartbeat-*' },
+  { name: 'prometheusbeat', pattern: 'prometheusbeat*' },
+  { name: 'filebeat', pattern: 'filebeat-*' },
+  { name: 'functionbeat', pattern: 'functionbeat-*' },
+  { name: 'fluentd', pattern: 'fluentd*' },
+  { name: 'telegraf', pattern: 'telegraf*' },
+  { name: 'fluentbit', pattern: 'fluentbit*' },
+  { name: 'nginx', pattern: 'nginx*' },
+  { name: 'apache', pattern: 'apache*' },
+  { name: 'logs', pattern: '*logs*' },
+  // Security
+  { name: 'auditbeat', pattern: 'auditbeat-*' },
+  { name: 'winlogbeat', pattern: 'winlogbeat-*' },
+  { name: 'packetbeat', pattern: 'packetbeat-*' },
+  // AppSearch
+  { name: 'app-search', pattern: '.ent-search-*' },
+  { name: 'app-search', pattern: '.app-search-*' },
+] as const;

src/plugins/telemetry/server/telemetry_collection/ingest_solutions/index.ts

@@ -0,0 +1,27 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { INGEST_SOLUTIONS_ID } from './constants';
+
+export {
+  IngestSolutionsIndex,
+  IngestSolutionsPayload,
+  getIngestSolutions,
+  buildIngestSolutionsPayload,
+} from './ingest_solutions';

src/plugins/telemetry/server/telemetry_collection/ingest_solutions/ingest_solutions.test.ts

@@ -0,0 +1,118 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { buildIngestSolutionsPayload, getIngestSolutions } from './ingest_solutions';
+
+describe('ingest_solutions', () => {
+  describe('buildIngestSolutionsPayload', () => {
+    test('return the base object when no indices provided', () => {
+      expect(buildIngestSolutionsPayload([])).toStrictEqual({});
+    });
+
+    test('return the base object when no matching indices provided', () => {
+      expect(
+        buildIngestSolutionsPayload([
+          { name: 'no-way-this-can_match_anything', sizeInBytes: 10 },
+          { name: '.kibana-event-log-8.0.0' },
+        ])
+      ).toStrictEqual({});
+    });
+
+    test('matches some indices and puts them in their own category', () => {
+      expect(
+        buildIngestSolutionsPayload([
+          { name: 'apm-1234' },
+          { name: 'apm-5677' },
+          { name: 'filebeat-12314', docCount: 100, sizeInBytes: 10 },
+          { name: 'metricbeat-1234', docCount: 100, sizeInBytes: 10, isECS: false },
+          { name: 'my_logs_custom', docCount: 1000, sizeInBytes: 10 },
+          { name: 'my_logs', docCount: 100, sizeInBytes: 10, isECS: true },
+          { name: 'logs_custom' },
+          { name: '.ent-search-1234' },
+        ])
+      ).toStrictEqual({
+        data_providers: {
+          apm: { index_count: 2 },
+          filebeat: { index_count: 1, doc_count: 100, size_in_bytes: 10 },
+          metricbeat: { index_count: 1, ecs_index_count: 0, doc_count: 100, size_in_bytes: 10 },
+          logs: { index_count: 3, ecs_index_count: 1, doc_count: 1100, size_in_bytes: 20 },
+          'app-search': { index_count: 1 },
+        },
+      });
+    });
+  });
+
+  describe('getIngestSolutions', () => {
+    test('it returns the base payload (all 0s) because no indices are found', async () => {
+      const callCluster = mockCallCluster();
+      await expect(getIngestSolutions(callCluster)).resolves.toStrictEqual({});
+    });
+
+    test('can only see the index in the state, but not the stats', async () => {
+      const callCluster = mockCallCluster(['filebeat-12314']);
+      await expect(getIngestSolutions(callCluster)).resolves.toStrictEqual({
+        data_providers: {
+          filebeat: { index_count: 1, ecs_index_count: 0 },
+        },
+      });
+    });
+
+    test('can see the state and the stats', async () => {
+      const callCluster = mockCallCluster(['filebeat-12314'], true, {
+        indices: {
+          'filebeat-12314': { total: { docs: { count: 100 }, store: { size_in_bytes: 10 } } },
+        },
+      });
+      await expect(getIngestSolutions(callCluster)).resolves.toStrictEqual({
+        data_providers: {
+          filebeat: { index_count: 1, ecs_index_count: 1, doc_count: 100, size_in_bytes: 10 },
+        },
+      });
+    });
+  });
+});
+
+function mockCallCluster(stateIndices: string[] = [], isECS = false, indexStats: any = {}) {
+  return jest.fn().mockImplementation(async (method: string, opts: any) => {
+    if (method === 'cluster.state') {
+      return {
+        metadata: {
+          indices: Object.fromEntries(
+            stateIndices.map((index, version) => [
+              index,
+              {
+                version,
+                ...(isECS
+                  ? {
+                      mappings: {
+                        _doc: {
+                          properties: { ecs: { properties: { version: { type: 'keyword' } } } },
+                        },
+                      },
+                    }
+                  : {}),
+              },
+            ])
+          ),
+        },
+      };
+    }
+    return indexStats;
+  });
+}