[Telemetry] Fix Logstash telemetry collection for multi node clusters (…

…elastic#96831) (elastic#96964) Prior to this fix, each Logstash node was overwriting the collected list of ephemeral ids used to collect pipeline details. This meant that pipeline details were only being collected for the last Logstash node retrieved for each cluster. Co-authored-by: Rob Bavey <[email protected]>
peteharverson · Apr 13, 2021 · 0083f09 · 0083f09
1 parent 971b612
commit 0083f09
Show file tree

Hide file tree

Showing 2 changed files with 142 additions and 4 deletions.
diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.test.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.test.ts
@@ -194,6 +194,117 @@ describe('Get Logstash Stats', () => {
       });
     });
 
+    it('should retrieve all ephemeral ids from all hits for the same cluster', () => {
+      const results = {
+        hits: {
+          hits: [
+            {
+              _source: {
+                type: 'logstash_stats',
+                cluster_uuid: 'FlV4ckTxQ0a78hmBkzzc9A',
+                logstash_stats: {
+                  logstash: {
+                    uuid: '0000000-0000-0000-0000-000000000000',
+                  },
+                  pipelines: [
+                    {
+                      id: 'main',
+                      ephemeral_id: 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa',
+                      queue: {
+                        type: 'memory',
+                      },
+                    },
+                  ],
+                },
+              },
+            },
+            {
+              _source: {
+                type: 'logstash_stats',
+                cluster_uuid: 'FlV4ckTxQ0a78hmBkzzc9A',
+                logstash_stats: {
+                  logstash: {
+                    uuid: '11111111-1111-1111-1111-111111111111',
+                  },
+                  pipelines: [
+                    {
+                      id: 'main',
+                      ephemeral_id: 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb',
+                      queue: {
+                        type: 'memory',
+                      },
+                    },
+                  ],
+                },
+              },
+            },
+            {
+              _source: {
+                type: 'logstash_stats',
+                cluster_uuid: '3',
+                logstash_stats: {
+                  logstash: {
+                    uuid: '22222222-2222-2222-2222-222222222222',
+                  },
+                  pipelines: [
+                    {
+                      id: 'main',
+                      ephemeral_id: 'cccccccc-cccc-cccc-cccc-cccccccccccc',
+                      queue: {
+                        type: 'memory',
+                      },
+                    },
+                  ],
+                },
+              },
+            },
+          ],
+        },
+      };
+
+      const options = getBaseOptions();
+      processStatsResults(results as any, options);
+
+      expect(options.allEphemeralIds).toStrictEqual({
+        FlV4ckTxQ0a78hmBkzzc9A: [
+          'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa',
+          'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb',
+        ],
+        '3': ['cccccccc-cccc-cccc-cccc-cccccccccccc'],
+      });
+
+      expect(options.clusters).toStrictEqual({
+        FlV4ckTxQ0a78hmBkzzc9A: {
+          count: 2,
+          cluster_stats: {
+            plugins: [],
+            collection_types: {
+              internal_collection: 2,
+            },
+            pipelines: {},
+            queues: {
+              memory: 2,
+            },
+          },
+          versions: [],
+        },
+        '3': {
+          count: 1,
+          cluster_stats: {
+            plugins: [],
+            collection_types: {
+              internal_collection: 1,
+            },
+            pipelines: {},
+            queues: {
+              memory: 1,
+            },
+          },
+          versions: [],
+        },
+      });
+    });
+
     it('should summarize stats from hits across multiple result objects', () => {
       const options = getBaseOptions();
 
@@ -208,6 +319,35 @@ describe('Get Logstash Stats', () => {
         });
       });
 
+      expect(options.allEphemeralIds).toStrictEqual({
+        '1n1p': ['cf37c6fa-2f1a-41e2-9a89-36b420a8b9a5'],
+        '1nmp': [
+          '47a70feb-3cb5-4618-8670-2c0bada61acd',
+          '5a65d966-0330-4bd7-82f2-ee81040c13cf',
+          '8d33fe25-a2c0-4c54-9ecf-d218cb8dbfe4',
+          'f4167a94-20a8-43e7-828e-4cf38d906187',
+        ],
+        mnmp: [
+          '2fcd4161-e08f-4eea-818b-703ea3ec6389',
+          'c6785d63-6e5f-42c2-839d-5edf139b7c19',
+          'bc6ef6f2-ecce-4328-96a2-002de41a144d',
+          '72058ad1-68a1-45f6-a8e8-10621ffc7288',
+          '18593052-c021-4158-860d-d8122981a0ac',
+          '4207025c-9b00-4bea-a36c-6fbf2d3c215e',
+          '0ec4702d-b5e5-4c60-91e9-6fa6a836f0d1',
+          '41258219-b129-4fad-a629-f244826281f8',
+          'e73bc63d-561a-4acd-a0c4-d5f70c4603df',
+          'ddf882b7-be26-4a93-8144-0aeb35122651',
+          '602936f5-98a3-4f8c-9471-cf389a519f4b',
+          '8b300988-62cc-4bc6-9ee0-9194f3f78e27',
+          '6ab60531-fb6f-478c-9063-82f2b0af2bed',
+          '802a5994-a03c-44b8-a650-47c0f71c2e48',
+          '6070b400-5c10-4c5e-b5c5-a5bd9be6d321',
+          '3193df5f-2a34-4fe3-816e-6b05999aa5ce',
+          '994e68cd-d607-40e6-a54c-02a51caa17e0',
+        ],
+      });
+
       expect(options.clusters).toStrictEqual({
         '1n1p': {
           count: 1,

diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_logstash_stats.ts
@@ -147,8 +147,6 @@ export function processStatsResults(
       }
       clusterStats.collection_types![thisCollectionType] =
         (clusterStats.collection_types![thisCollectionType] || 0) + 1;
-
-      const theseEphemeralIds: string[] = [];
       const pipelines = logstashStats.pipelines || [];
 
       pipelines.forEach((pipeline) => {
@@ -162,10 +160,10 @@ export function processStatsResults(
 
         const ephemeralId = pipeline.ephemeral_id;
         if (ephemeralId !== undefined) {
-          theseEphemeralIds.push(ephemeralId);
+          allEphemeralIds[clusterUuid] = allEphemeralIds[clusterUuid] || [];
+          allEphemeralIds[clusterUuid].push(ephemeralId);
         }
       });
-      allEphemeralIds[clusterUuid] = theseEphemeralIds;
     }
   });
 }