From c803068246434bc1562967652c30f2c6181d2782 Mon Sep 17 00:00:00 2001
From: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
Date: Tue, 13 Apr 2021 14:03:51 -0400
Subject: [PATCH] [Telemetry] Fix Logstash telemetry collection for multi node
 clusters (#96831) (#96963)

Prior to this fix, each Logstash node was overwriting the collected list of ephemeral ids used to
collect pipeline details. This meant that pipeline details were only being collected for the
last Logstash node retrieved for each cluster.

Co-authored-by: Rob Bavey <rob.bavey@elastic.co>
---
 .../get_logstash_stats.test.ts                | 140 ++++++++++++++++++
 .../get_logstash_stats.ts                     |   6 +-
 2 files changed, 142 insertions(+), 4 deletions(-)

diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.test.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.test.ts
index f2f0c37255d92..cf1574f8d3f0e 100644
--- a/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.test.ts
+++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.test.ts
@@ -194,6 +194,117 @@ describe('Get Logstash Stats', () => {
       });
     });
 
+    it('should retrieve all ephemeral ids from all hits for the same cluster', () => {
+      const results = {
+        hits: {
+          hits: [
+            {
+              _source: {
+                type: 'logstash_stats',
+                cluster_uuid: 'FlV4ckTxQ0a78hmBkzzc9A',
+                logstash_stats: {
+                  logstash: {
+                    uuid: '0000000-0000-0000-0000-000000000000',
+                  },
+                  pipelines: [
+                    {
+                      id: 'main',
+                      ephemeral_id: 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa',
+                      queue: {
+                        type: 'memory',
+                      },
+                    },
+                  ],
+                },
+              },
+            },
+            {
+              _source: {
+                type: 'logstash_stats',
+                cluster_uuid: 'FlV4ckTxQ0a78hmBkzzc9A',
+                logstash_stats: {
+                  logstash: {
+                    uuid: '11111111-1111-1111-1111-111111111111',
+                  },
+                  pipelines: [
+                    {
+                      id: 'main',
+                      ephemeral_id: 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb',
+                      queue: {
+                        type: 'memory',
+                      },
+                    },
+                  ],
+                },
+              },
+            },
+            {
+              _source: {
+                type: 'logstash_stats',
+                cluster_uuid: '3',
+                logstash_stats: {
+                  logstash: {
+                    uuid: '22222222-2222-2222-2222-222222222222',
+                  },
+                  pipelines: [
+                    {
+                      id: 'main',
+                      ephemeral_id: 'cccccccc-cccc-cccc-cccc-cccccccccccc',
+                      queue: {
+                        type: 'memory',
+                      },
+                    },
+                  ],
+                },
+              },
+            },
+          ],
+        },
+      };
+
+      const options = getBaseOptions();
+      processStatsResults(results as any, options);
+
+      expect(options.allEphemeralIds).toStrictEqual({
+        FlV4ckTxQ0a78hmBkzzc9A: [
+          'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa',
+          'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb',
+        ],
+        '3': ['cccccccc-cccc-cccc-cccc-cccccccccccc'],
+      });
+
+      expect(options.clusters).toStrictEqual({
+        FlV4ckTxQ0a78hmBkzzc9A: {
+          count: 2,
+          cluster_stats: {
+            plugins: [],
+            collection_types: {
+              internal_collection: 2,
+            },
+            pipelines: {},
+            queues: {
+              memory: 2,
+            },
+          },
+          versions: [],
+        },
+        '3': {
+          count: 1,
+          cluster_stats: {
+            plugins: [],
+            collection_types: {
+              internal_collection: 1,
+            },
+            pipelines: {},
+            queues: {
+              memory: 1,
+            },
+          },
+          versions: [],
+        },
+      });
+    });
+
     it('should summarize stats from hits across multiple result objects', () => {
       const options = getBaseOptions();
 
@@ -208,6 +319,35 @@ describe('Get Logstash Stats', () => {
         });
       });
 
+      expect(options.allEphemeralIds).toStrictEqual({
+        '1n1p': ['cf37c6fa-2f1a-41e2-9a89-36b420a8b9a5'],
+        '1nmp': [
+          '47a70feb-3cb5-4618-8670-2c0bada61acd',
+          '5a65d966-0330-4bd7-82f2-ee81040c13cf',
+          '8d33fe25-a2c0-4c54-9ecf-d218cb8dbfe4',
+          'f4167a94-20a8-43e7-828e-4cf38d906187',
+        ],
+        mnmp: [
+          '2fcd4161-e08f-4eea-818b-703ea3ec6389',
+          'c6785d63-6e5f-42c2-839d-5edf139b7c19',
+          'bc6ef6f2-ecce-4328-96a2-002de41a144d',
+          '72058ad1-68a1-45f6-a8e8-10621ffc7288',
+          '18593052-c021-4158-860d-d8122981a0ac',
+          '4207025c-9b00-4bea-a36c-6fbf2d3c215e',
+          '0ec4702d-b5e5-4c60-91e9-6fa6a836f0d1',
+          '41258219-b129-4fad-a629-f244826281f8',
+          'e73bc63d-561a-4acd-a0c4-d5f70c4603df',
+          'ddf882b7-be26-4a93-8144-0aeb35122651',
+          '602936f5-98a3-4f8c-9471-cf389a519f4b',
+          '8b300988-62cc-4bc6-9ee0-9194f3f78e27',
+          '6ab60531-fb6f-478c-9063-82f2b0af2bed',
+          '802a5994-a03c-44b8-a650-47c0f71c2e48',
+          '6070b400-5c10-4c5e-b5c5-a5bd9be6d321',
+          '3193df5f-2a34-4fe3-816e-6b05999aa5ce',
+          '994e68cd-d607-40e6-a54c-02a51caa17e0',
+        ],
+      });
+
       expect(options.clusters).toStrictEqual({
         '1n1p': {
           count: 1,
diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.ts
index 93c69c644c064..f4f67a5582303 100644
--- a/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.ts
+++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.ts
@@ -147,8 +147,6 @@ export function processStatsResults(
       }
       clusterStats.collection_types![thisCollectionType] =
         (clusterStats.collection_types![thisCollectionType] || 0) + 1;
-
-      const theseEphemeralIds: string[] = [];
       const pipelines = logstashStats.pipelines || [];
 
       pipelines.forEach((pipeline) => {
@@ -162,10 +160,10 @@ export function processStatsResults(
 
         const ephemeralId = pipeline.ephemeral_id;
         if (ephemeralId !== undefined) {
-          theseEphemeralIds.push(ephemeralId);
+          allEphemeralIds[clusterUuid] = allEphemeralIds[clusterUuid] || [];
+          allEphemeralIds[clusterUuid].push(ephemeralId);
         }
       });
-      allEphemeralIds[clusterUuid] = theseEphemeralIds;
     }
   });
 }