[SecuritySolution] Service Entity Store (elastic#202344)

## Summary


### Service Definition:

https://github.com/elastic/kibana/pull/202344/files#diff-42c7dd345e0500c97f85824904a70a11162827ea8f8df6982082a9047ca04ff1


### Acceptance Criteria
- [x] Upon installation of the entity store, the Service entity
definition should be created by default
- [x] The Service definition will be installed in the exact same way as
the User and Host definitions
- [x] The unique identifier for service entities will be `service.name`
- [x] The fields captured for service entities should match the field
mapping spreadsheet (see Implementation Notes below)


### Stored Entity
```json
{
          "@timestamp": "2024-12-02T10:43:13.856Z",
          "event": {
            "ingested": "2024-12-02T10:51:28.987428Z"
          },
          "entity": {
            "name": "test123 name",
            "id": "test123 name",
            "source": "logs-blito",
            "type": "service"
          },
          "service": {
            "node": {
              "roles": [
                "test123 node roles"
              ],
              "name": [
                "test123 node name"
              ]
            },
            "environment": [
              "test123 environment"
            ],
            "address": [
              "test123 address"
            ],
            "name": "test123 name",
            "id": [
              "test123 id"
            ],
            "state": [
              "test123 state"
            ],
            "ephemeral_id": [
              "test123 ephemeral_id"
            ],
            "type": [
              "test123 type"
            ],
            "version": [
              "test123 version"
            ]
          }
}
```

### How to test it?

* Start Kibana
<details>
  <summary>Create mappings</summary>
  
```
PUT /logs-test
{
  "mappings": {
    "properties": {      
      "service.name": {
        "type": "keyword"
      },
      "service.address": {
        "type": "keyword"
      },
      "service.environment": {
        "type": "keyword"
      },
      "service.ephemeral_id": {
        "type": "keyword"
      },
      "service.id": {
        "type": "keyword"
      },
      "service.node.name": {
        "type": "keyword"
      },
      "service.node.roles": {
        "type": "keyword"
      },
      "service.state": {
        "type": "keyword"
      },
      "service.type": {
        "type": "keyword"
      },
      "service.version": {
        "type": "keyword"
      },
      "@timestamp": {
        "type": "date"
      }
    }
  }
}
```` 
</details>


<details>
  <summary>Create document</summary>
  
```
PUT /logs-test
POST logs-test/_doc
{
  "service": {
    "name": "test123 name",
    "address": "test123 address",
    "environment": "test123 environment",
    "ephemeral_id": "test123 ephemeral_id",
    "id": "test123 id",
    "node.roles": "test123 node roles",
    "node.name": "test123 node name",    
    "state": "test123 state",
    "type": "test123 type",
    "version": "test123 version"
  },
  "@timestamp": "2024-12-02T10:43:13.856Z"
}

```` 
</details>

* Init the entity store
* Wait...
* Query the service index `GET
.entities.v1.latest.security_service_default/_search`


### Open Questions
* Can we merge this PR without first updating all other features that
will use service entities?
* If we merge it, the service engine will be installed together with
other entities, but it won't provide any functionality
* Do we need an experimental flag?

---------

Co-authored-by: kibanamachine <[email protected]>

oas_docs/output/kibana.serverless.yaml

@@ -47102,6 +47102,7 @@ components:
       enum:
         - user
         - host
+        - service
       type: string
     Security_Entity_Analytics_API_HostEntity:
       type: object

oas_docs/output/kibana.yaml

@@ -54783,6 +54783,7 @@ components:
       enum:
         - user
         - host
+        - service
       type: string
     Security_Entity_Analytics_API_HostEntity:
       type: object

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts

@@ -17,7 +17,7 @@
 import { z } from '@kbn/zod';
 
 export type EntityType = z.infer<typeof EntityType>;
-export const EntityType = z.enum(['user', 'host']);
+export const EntityType = z.enum(['user', 'host', 'service']);
 export type EntityTypeEnum = typeof EntityType.enum;
 export const EntityTypeEnum = EntityType.enum;
 

x-pack/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml

@@ -11,6 +11,7 @@ components:
       enum:
         - user
         - host
+        - service
 
     EngineDescriptor:
       type: object

x-pack/plugins/security_solution/common/experimental_features.ts

@@ -243,6 +243,11 @@ export const allowedExperimentalValues = Object.freeze({
    */
   entityStoreDisabled: false,
 
+  /**
+   * Enables the Service Entity Store. The Entity Store feature will install the service engine by default.
+   */
+  serviceEntityStoreEnabled: true,
+
   /**
    * Enables the siem migrations feature
    */

x-pack/plugins/security_solution/docs/openapi/ess/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml

@@ -987,6 +987,7 @@ components:
       enum:
         - user
         - host
+        - service
       type: string
     HostEntity:
       type: object

x-pack/plugins/security_solution/docs/openapi/serverless/security_solution_entity_analytics_api_2023_10_31.bundled.schema.yaml

@@ -987,6 +987,7 @@ components:
       enum:
         - user
         - host
+        - service
       type: string
     HostEntity:
       type: object

x-pack/plugins/security_solution/public/entity_analytics/components/entity_store/hooks/use_entity_store.ts

@@ -12,10 +12,11 @@ import type { IHttpFetchError } from '@kbn/core-http-browser';
 import type { GetEntityStoreStatusResponse } from '../../../../../common/api/entity_analytics/entity_store/status.gen';
 import type { InitEntityStoreResponse } from '../../../../../common/api/entity_analytics/entity_store/enable.gen';
 import { useKibana } from '../../../../common/lib/kibana/kibana_react';
-import type {
-  DeleteEntityEngineResponse,
-  InitEntityEngineResponse,
-  StopEntityEngineResponse,
+import type { EntityType } from '../../../../../common/api/entity_analytics';
+import {
+  type DeleteEntityEngineResponse,
+  type InitEntityEngineResponse,
+  type StopEntityEngineResponse,
 } from '../../../../../common/api/entity_analytics';
 import { useEntityStoreRoutes } from '../../../api/entity_store';
 import { EntityEventTypes } from '../../../../common/lib/telemetry';
@@ -68,13 +69,16 @@ export const useEnableEntityStoreMutation = (options?: UseMutationOptions<{}>) =
 };
 
 export const INIT_ENTITY_ENGINE_STATUS_KEY = ['POST', 'INIT_ENTITY_ENGINE'];
+/**
+ * @deprecated
+ * It will be deleted on a follow-up PR
+ */
 export const useInitEntityEngineMutation = (options?: UseMutationOptions<{}>) => {
   const queryClient = useQueryClient();
 
   const { initEntityEngine } = useEntityStoreRoutes();
   return useMutation<InitEntityEngineResponse[]>(
     () => Promise.all([initEntityEngine('user'), initEntityEngine('host')]),
-
     {
       mutationKey: INIT_ENTITY_ENGINE_STATUS_KEY,
       onSuccess: () => queryClient.refetchQueries({ queryKey: ENTITY_STORE_STATUS }),
@@ -84,7 +88,7 @@ export const useInitEntityEngineMutation = (options?: UseMutationOptions<{}>) =>
 };
 
 export const STOP_ENTITY_ENGINE_STATUS_KEY = ['POST', 'STOP_ENTITY_ENGINE'];
-export const useStopEntityEngineMutation = (options?: UseMutationOptions<{}>) => {
+export const useStopEntityEngineMutation = (entityTypes: EntityType[]) => {
   const { telemetry } = useKibana().services;
   const queryClient = useQueryClient();
 
@@ -95,23 +99,28 @@ export const useStopEntityEngineMutation = (options?: UseMutationOptions<{}>) =>
         timestamp: new Date().toISOString(),
         action: 'stop',
       });
-      return Promise.all([stopEntityEngine('user'), stopEntityEngine('host')]);
+      return Promise.all(entityTypes.map((entityType) => stopEntityEngine(entityType)));
     },
     {
       mutationKey: STOP_ENTITY_ENGINE_STATUS_KEY,
       onSuccess: () => queryClient.refetchQueries({ queryKey: ENTITY_STORE_STATUS }),
-      ...options,
     }
   );
 };
 
 export const DELETE_ENTITY_ENGINE_STATUS_KEY = ['POST', 'STOP_ENTITY_ENGINE'];
-export const useDeleteEntityEngineMutation = ({ onSuccess }: { onSuccess?: () => void }) => {
+export const useDeleteEntityEngineMutation = ({
+  onSuccess,
+  entityTypes,
+}: {
+  onSuccess?: () => void;
+  entityTypes: EntityType[];
+}) => {
   const queryClient = useQueryClient();
   const { deleteEntityEngine } = useEntityStoreRoutes();
 
   return useMutation<DeleteEntityEngineResponse[]>(
-    () => Promise.all([deleteEntityEngine('user', true), deleteEntityEngine('host', true)]),
+    () => Promise.all(entityTypes.map((entityType) => deleteEntityEngine(entityType, true))),
     {
       mutationKey: DELETE_ENTITY_ENGINE_STATUS_KEY,
       onSuccess: () => {

x-pack/plugins/security_solution/public/entity_analytics/pages/entity_store_management_page.tsx

@@ -33,7 +33,7 @@ import React, { useCallback, useEffect, useState } from 'react';
 import { FormattedMessage } from '@kbn/i18n-react';
 
 import type { SecurityAppError } from '@kbn/securitysolution-t-grid';
-import type { StoreStatus } from '../../../common/api/entity_analytics';
+import { EntityType, EntityTypeEnum, type StoreStatus } from '../../../common/api/entity_analytics';
 import { useIsExperimentalFeatureEnabled } from '../../common/hooks/use_experimental_features';
 import { ASSET_CRITICALITY_INDEX_PATTERN } from '../../../common/entity_analytics/asset_criticality';
 import { useKibana } from '../../common/lib/kibana';
@@ -73,13 +73,20 @@ export const EntityStoreManagementPage = () => {
   const hasAssetCriticalityWritePermissions = assetCriticalityPrivileges?.has_write_permissions;
   const [selectedTabId, setSelectedTabId] = useState(TabId.Import);
   const entityStoreStatus = useEntityStoreStatus({});
+  const isServiceEntityStoreEnabled = useIsExperimentalFeatureEnabled('serviceEntityStoreEnabled');
+  const allEntityTypes = Object.values(EntityType.Values);
+
+  const entityTypes = isServiceEntityStoreEnabled
+    ? allEntityTypes
+    : allEntityTypes.filter((value) => value !== EntityTypeEnum.service);
 
   const enableStoreMutation = useEnableEntityStoreMutation();
-  const stopEntityEngineMutation = useStopEntityEngineMutation();
+  const stopEntityEngineMutation = useStopEntityEngineMutation(entityTypes);
   const deleteEntityEngineMutation = useDeleteEntityEngineMutation({
     onSuccess: () => {
       closeClearModal();
     },
+    entityTypes,
   });
 
   const [isClearModalVisible, setIsClearModalVisible] = useState(false);

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.test.ts

@@ -16,6 +16,7 @@ import type { EntityType } from '../../../../common/api/entity_analytics/entity_
 import type { DataViewsService } from '@kbn/data-views-plugin/common';
 import type { AppClient } from '../../..';
 import type { EntityStoreConfig } from './types';
+import { mockGlobalState } from '../../../../public/common/mock';
 
 describe('EntityStoreDataClient', () => {
   const mockSavedObjectClient = savedObjectsClientMock.create();
@@ -31,6 +32,7 @@ describe('EntityStoreDataClient', () => {
     dataViewsService: {} as DataViewsService,
     appClient: {} as AppClient,
     config: {} as EntityStoreConfig,
+    experimentalFeatures: mockGlobalState.app.enableExperimental,
   });
 
   const defaultSearchParams = {

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/entity_store_data_client.ts

@@ -23,6 +23,7 @@ import moment from 'moment';
 import type { EntityDefinitionWithState } from '@kbn/entityManager-plugin/server/lib/entities/types';
 import type { EntityDefinition } from '@kbn/entities-schema';
 import type { estypes } from '@elastic/elasticsearch';
+import type { ExperimentalFeatures } from '../../../../common';
 import type {
   GetEntityStoreStatusRequestQuery,
   GetEntityStoreStatusResponse,
@@ -32,7 +33,10 @@ import type {
   InitEntityStoreResponse,
 } from '../../../../common/api/entity_analytics/entity_store/enable.gen';
 import type { AppClient } from '../../..';
-import { EngineComponentResourceEnum, EntityType } from '../../../../common/api/entity_analytics';
+import {
+  EngineComponentResourceEnum,
+  EntityTypeEnum,
+} from '../../../../common/api/entity_analytics';
 import type {
   Entity,
   EngineDataviewUpdateResult,
@@ -42,6 +46,7 @@ import type {
   ListEntityEnginesResponse,
   EngineComponentStatus,
   EngineComponentResource,
+  EntityType,
 } from '../../../../common/api/entity_analytics';
 import { EngineDescriptorClient } from './saved_object/engine_descriptor';
 import { ENGINE_STATUS, ENTITY_STORE_STATUS, MAX_SEARCH_RESPONSE_SIZE } from './constants';
@@ -108,6 +113,7 @@ interface EntityStoreClientOpts {
   dataViewsService: DataViewsService;
   appClient: AppClient;
   config: EntityStoreConfig;
+  experimentalFeatures: ExperimentalFeatures;
   telemetry?: AnalyticsServiceSetup;
 }
 
@@ -204,7 +210,13 @@ export class EntityStoreDataClient {
     // Immediately defer the initialization to the next tick. This way we don't block on the init preflight checks
     const run = <T>(fn: () => Promise<T>) =>
       new Promise<T>((resolve) => setTimeout(() => fn().then(resolve), 0));
-    const promises = Object.values(EntityType.Values).map((entity) =>
+
+    const { experimentalFeatures } = this.options;
+    const enginesTypes = experimentalFeatures.serviceEntityStoreEnabled
+      ? [EntityTypeEnum.host, EntityTypeEnum.user, EntityTypeEnum.service]
+      : [EntityTypeEnum.host, EntityTypeEnum.user];
+
+    const promises = enginesTypes.map((entity) =>
       run(() =>
         this.init(entity, { indexPattern, filter, fieldHistoryLength }, { pipelineDebugMode })
       )

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/index.ts

@@ -7,4 +7,5 @@
 
 export * from './host';
 export * from './user';
+export * from './service';
 export { getCommonUnitedFieldDefinitions } from './common';

x-pack/plugins/security_solution/server/lib/entity_analytics/entity_store/united_entity_definitions/entity_types/service.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { collectValuesWithLength, newestValue } from '../definition_utils';
+import type { UnitedDefinitionBuilder } from '../types';
+
+export const SERVICE_DEFINITION_VERSION = '1.0.0';
+export const getServiceUnitedDefinition: UnitedDefinitionBuilder = (fieldHistoryLength: number) => {
+  const collect = collectValuesWithLength(fieldHistoryLength);
+  return {
+    entityType: 'service',
+    version: SERVICE_DEFINITION_VERSION,
+    fields: [
+      collect({ field: 'service.address' }),
+      collect({ field: 'service.environment' }),
+      collect({ field: 'service.ephemeral_id' }),
+      collect({ field: 'service.id' }),
+      collect({ field: 'service.node.name' }),
+      collect({ field: 'service.node.roles' }),
+      newestValue({ field: 'service.state' }),
+      collect({ field: 'service.type' }),
+      newestValue({ field: 'service.version' }),
+    ],
+  };
+};