[8.x] [eem] merging nulls fix (#204183) (#204212)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[eem] merging nulls fix
(#204183)](#204183)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Kevin
Lacabane","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-12-13T13:39:04Z","message":"[eem]
merging nulls fix (#204183)\n\nCloses
https://github.com/elastic/kibana/issues/203982\r\n\r\nIgnore nulls when
merging metadata fields.\r\n\r\nThe flakiness comes from the random
arrival time of the queries. since\r\nwe execute two queries in
parallel, if the first one has the null value\r\nfor a field the merging
logic will ignore it and test succeeds, if the\r\nnull value arrives
second it
fails.","sha":"d89153dc55bdeab635a69a7eab48ddae208d79d4","branchLabelMapping":{"^v9.0.0$":"main","^v8.18.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:prev-minor","Team:obs-entities"],"title":"[eem]
merging nulls
fix","number":204183,"url":"https://github.com/elastic/kibana/pull/204183","mergeCommit":{"message":"[eem]
merging nulls fix (#204183)\n\nCloses
https://github.com/elastic/kibana/issues/203982\r\n\r\nIgnore nulls when
merging metadata fields.\r\n\r\nThe flakiness comes from the random
arrival time of the queries. since\r\nwe execute two queries in
parallel, if the first one has the null value\r\nfor a field the merging
logic will ignore it and test succeeds, if the\r\nnull value arrives
second it
fails.","sha":"d89153dc55bdeab635a69a7eab48ddae208d79d4"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/204183","number":204183,"mergeCommit":{"message":"[eem]
merging nulls fix (#204183)\n\nCloses
https://github.com/elastic/kibana/issues/203982\r\n\r\nIgnore nulls when
merging metadata fields.\r\n\r\nThe flakiness comes from the random
arrival time of the queries. since\r\nwe execute two queries in
parallel, if the first one has the null value\r\nfor a field the merging
logic will ignore it and test succeeds, if the\r\nnull value arrives
second it fails.","sha":"d89153dc55bdeab635a69a7eab48ddae208d79d4"}}]}]
BACKPORT-->

Co-authored-by: Kevin Lacabane <[email protected]>

x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/entity_instances.test.ts

@@ -28,7 +28,7 @@ describe('getEntityInstancesQuery', () => {
 
     expect(query).toEqual(
       'FROM logs-*, metrics-* | ' +
-        'STATS host.name = VALUES(host.name::keyword), entity.last_seen_timestamp = MAX(custom_timestamp_field), service.id = MAX(service.id::keyword) BY service.name::keyword | ' +
+        'STATS host.name = TOP(host.name::keyword, 10, "ASC"), entity.last_seen_timestamp = MAX(custom_timestamp_field), service.id = MAX(service.id::keyword) BY service.name::keyword | ' +
         'RENAME `service.name::keyword` AS service.name | ' +
         'EVAL entity.type = "service", entity.id = service.name, entity.display_name = COALESCE(service.id, entity.id) | ' +
         'SORT entity.id DESC | ' +

x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/entity_instances.ts

@@ -46,7 +46,7 @@ const dslFilter = ({
 const statsCommand = ({ source }: { source: EntitySourceDefinition }) => {
   const aggs = source.metadata_fields
     .filter((field) => !source.identity_fields.some((idField) => idField === field))
-    .map((field) => `${field} = VALUES(${asKeyword(field)})`);
+    .map((field) => `${field} = TOP(${asKeyword(field)}, 10, "ASC")`);
 
   if (source.timestamp_field) {
     aggs.push(`entity.last_seen_timestamp = MAX(${source.timestamp_field})`);

x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/utils.test.ts

@@ -277,5 +277,48 @@ describe('mergeEntitiesList', () => {
         service_name: 'foo',
       });
     });
+
+    it('ignores null values when merging', () => {
+      const entities = [
+        {
+          'entity.id': 'foo',
+          'entity.type': 'service',
+          'entity.display_name': 'foo',
+          'service.name': 'foo',
+          'host.name': null,
+        },
+        {
+          'entity.id': 'foo',
+          'entity.type': 'service',
+          'entity.display_name': 'foo',
+          'service.name': 'foo',
+          'host.name': 'host-2',
+        },
+        {
+          'entity.id': 'foo',
+          'entity.type': 'service',
+          'entity.display_name': 'foo',
+          'service.name': 'foo',
+          'host.name': null,
+        },
+      ];
+
+      const mergedEntities = mergeEntitiesList({
+        entities,
+        metadataFields: ['host.name'],
+        sources: [
+          { identity_fields: ['service.name'], metadata_fields: [] as string[] },
+          { identity_fields: ['service.name'], metadata_fields: [] as string[] },
+        ] as EntitySourceDefinition[],
+      });
+      expect(mergedEntities.length).toEqual(1);
+      expect(mergedEntities[0]).toEqual({
+        'entity.id': 'foo',
+        'entity.type': 'service',
+        'entity.display_name': 'foo',
+        'service.name': 'foo',
+        'host.name': 'host-2',
+      });
+    });
   });
 });

x-pack/platform/plugins/shared/entity_manager/server/lib/v2/queries/utils.ts

@@ -33,8 +33,8 @@ function mergeEntities(
     merged['entity.last_seen_timestamp'] = latestTimestamp;
   }
 
-  for (const [key, value] of Object.entries(entity2).filter(([_key]) =>
-    mergeableFields.includes(_key)
+  for (const [key, value] of Object.entries(entity2).filter(
+    ([_key]) => mergeableFields.includes(_key) && entity2[_key]
   )) {
     if (merged[key]) {
       // we want to keep identity fields as single-value properties.

x-pack/test/api_integration/apis/entity_manager/search.ts

@@ -20,8 +20,7 @@ export default function ({ getService }: FtrProviderContext) {
   const esClient = getService('es');
   const supertest = getService('supertest');
 
-  // Failing: See https://github.com/elastic/kibana/issues/203982
-  describe.skip('_search API', () => {
+  describe('_search API', () => {
     let cleanup: Function[] = [];
 
     before(() => clearEntityDefinitions(esClient));