[Telemetry] Fix Logstash telemetry collection for multi node clusters (…

…#96831) (#96963) Prior to this fix, each Logstash node was overwriting the collected list of ephemeral ids used to collect pipeline details. This meant that pipeline details were only being collected for the last Logstash node retrieved for each cluster. Co-authored-by: Rob Bavey <[email protected]>
elastic · Apr 13, 2021 · c803068 · c803068
1 parent 0596937
commit c803068
Show file tree

Hide file tree

Showing 2 changed files with 142 additions and 4 deletions.
diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.test.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.test.ts
@@ -194,6 +194,117 @@ describe('Get Logstash Stats', () => {
       });
     });
 
+    it('should retrieve all ephemeral ids from all hits for the same cluster', () => {
+      const results = {
+        hits: {
+          hits: [
+            {
+              _source: {
+                type: 'logstash_stats',
+                cluster_uuid: 'FlV4ckTxQ0a78hmBkzzc9A',
+                logstash_stats: {
+                  logstash: {
+                    uuid: '0000000-0000-0000-0000-000000000000',
+                  },
+                  pipelines: [
+                    {
+                      id: 'main',
+                      ephemeral_id: 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa',
+                      queue: {
+                        type: 'memory',
+                      },
+                    },
+                  ],
+                },
+              },
+            },
+            {
+              _source: {
+                type: 'logstash_stats',
+                cluster_uuid: 'FlV4ckTxQ0a78hmBkzzc9A',
+                logstash_stats: {
+                  logstash: {
+                    uuid: '11111111-1111-1111-1111-111111111111',
+                  },
+                  pipelines: [
+                    {
+                      id: 'main',
+                      ephemeral_id: 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb',
+                      queue: {
+                        type: 'memory',
+                      },
+                    },
+                  ],
+                },
+              },
+            },
+            {
+              _source: {
+                type: 'logstash_stats',
+                cluster_uuid: '3',
+                logstash_stats: {
+                  logstash: {
+                    uuid: '22222222-2222-2222-2222-222222222222',
+                  },
+                  pipelines: [
+                    {
+                      id: 'main',
+                      ephemeral_id: 'cccccccc-cccc-cccc-cccc-cccccccccccc',
+                      queue: {
+                        type: 'memory',
+                      },
+                    },
+                  ],
+                },
+              },
+            },
+          ],
+        },
+      };
+
+      const options = getBaseOptions();
+      processStatsResults(results as any, options);
+
+      expect(options.allEphemeralIds).toStrictEqual({
+        FlV4ckTxQ0a78hmBkzzc9A: [
+          'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa',
+          'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb',
+        ],
+        '3': ['cccccccc-cccc-cccc-cccc-cccccccccccc'],
+      });
+
+      expect(options.clusters).toStrictEqual({
+        FlV4ckTxQ0a78hmBkzzc9A: {
+          count: 2,
+          cluster_stats: {
+            plugins: [],
+            collection_types: {
+              internal_collection: 2,
+            },
+            pipelines: {},
+            queues: {
+              memory: 2,
+            },
+          },
+          versions: [],
+        },
+        '3': {
+          count: 1,
+          cluster_stats: {
+            plugins: [],
+            collection_types: {
+              internal_collection: 1,
+            },
+            pipelines: {},
+            queues: {
+              memory: 1,
+            },
+          },
+          versions: [],
+        },
+      });
+    });
+
     it('should summarize stats from hits across multiple result objects', () => {
       const options = getBaseOptions();
 
@@ -208,6 +319,35 @@ describe('Get Logstash Stats', () => {
         });
       });
 
+      expect(options.allEphemeralIds).toStrictEqual({
+        '1n1p': ['cf37c6fa-2f1a-41e2-9a89-36b420a8b9a5'],
+        '1nmp': [
+          '47a70feb-3cb5-4618-8670-2c0bada61acd',
+          '5a65d966-0330-4bd7-82f2-ee81040c13cf',
+          '8d33fe25-a2c0-4c54-9ecf-d218cb8dbfe4',
+          'f4167a94-20a8-43e7-828e-4cf38d906187',
+        ],
+        mnmp: [
+          '2fcd4161-e08f-4eea-818b-703ea3ec6389',
+          'c6785d63-6e5f-42c2-839d-5edf139b7c19',
+          'bc6ef6f2-ecce-4328-96a2-002de41a144d',
+          '72058ad1-68a1-45f6-a8e8-10621ffc7288',
+          '18593052-c021-4158-860d-d8122981a0ac',
+          '4207025c-9b00-4bea-a36c-6fbf2d3c215e',
+          '0ec4702d-b5e5-4c60-91e9-6fa6a836f0d1',
+          '41258219-b129-4fad-a629-f244826281f8',
+          'e73bc63d-561a-4acd-a0c4-d5f70c4603df',
+          'ddf882b7-be26-4a93-8144-0aeb35122651',
+          '602936f5-98a3-4f8c-9471-cf389a519f4b',
+          '8b300988-62cc-4bc6-9ee0-9194f3f78e27',
+          '6ab60531-fb6f-478c-9063-82f2b0af2bed',
+          '802a5994-a03c-44b8-a650-47c0f71c2e48',
+          '6070b400-5c10-4c5e-b5c5-a5bd9be6d321',
+          '3193df5f-2a34-4fe3-816e-6b05999aa5ce',
+          '994e68cd-d607-40e6-a54c-02a51caa17e0',
+        ],
+      });
+
       expect(options.clusters).toStrictEqual({
         '1n1p': {
           count: 1,

diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.ts
@@ -147,8 +147,6 @@ export function processStatsResults(
       }
       clusterStats.collection_types![thisCollectionType] =
         (clusterStats.collection_types![thisCollectionType] || 0) + 1;
-
-      const theseEphemeralIds: string[] = [];
       const pipelines = logstashStats.pipelines || [];
 
       pipelines.forEach((pipeline) => {
@@ -162,10 +160,10 @@ export function processStatsResults(
 
         const ephemeralId = pipeline.ephemeral_id;
         if (ephemeralId !== undefined) {
-          theseEphemeralIds.push(ephemeralId);
+          allEphemeralIds[clusterUuid] = allEphemeralIds[clusterUuid] || [];
+          allEphemeralIds[clusterUuid].push(ephemeralId);
         }
       });
-      allEphemeralIds[clusterUuid] = theseEphemeralIds;
     }
   });
 }