[EEM] Add create and read APIs for v2 type and source definitions (el…

…astic#201470) ## Summary This PR adds: * Some basic features and privileges for the EEM app * A function that sets up an index with a template for the new definitions * 4 API endpoints to read and create entity types and sources * `POST /internal/entities/v2/definitions/types` * `GET /internal/entities/v2/definitions/types` * `POST /internal/entities/v2/definitions/sources` * `GET /internal/entities/v2/definitions/sources` * Some v2 shuffling around of code --------- Co-authored-by: kibanamachine <[email protected]> Co-authored-by: Maxim Palenov <[email protected]>
SoniaSanzV · Dec 9, 2024 · 7cfdd8c · 7cfdd8c
1 parent 169b0c2
commit 7cfdd8c
Show file tree

Hide file tree

Showing 32 changed files with 893 additions and 336 deletions.
diff --git a/x-pack/plugins/entity_manager/kibana.jsonc b/x-pack/plugins/entity_manager/kibana.jsonc
@@ -13,7 +13,8 @@
     "requiredPlugins": [
       "security",
       "encryptedSavedObjects",
-      "licensing"
+      "licensing",
+      "features"
     ],
     "requiredBundles": []
   }

diff --git a/x-pack/plugins/entity_manager/server/index.ts b/x-pack/plugins/entity_manager/server/index.ts
@@ -18,6 +18,14 @@ export type {
 };
 export { config };
 
+export {
+  CREATE_ENTITY_TYPE_DEFINITION_PRIVILEGE,
+  CREATE_ENTITY_SOURCE_DEFINITION_PRIVILEGE,
+  READ_ENTITY_TYPE_DEFINITION_PRIVILEGE,
+  READ_ENTITY_SOURCE_DEFINITION_PRIVILEGE,
+  READ_ENTITIES_PRIVILEGE,
+} from './lib/v2/constants';
+
 export const plugin = async (context: PluginInitializerContext<EntityManagerConfig>) => {
   const { EntityManagerServerPlugin } = await import('./plugin');
   return new EntityManagerServerPlugin(context);

diff --git a/x-pack/plugins/entity_manager/server/lib/client/index.ts b/x-pack/plugins/entity_manager/server/lib/client/index.ts
diff --git a/x-pack/plugins/entity_manager/server/lib/entity_client.ts b/x-pack/plugins/entity_manager/server/lib/entity_client.ts
@@ -5,10 +5,9 @@
  * 2.0.
  */
 
-import { without } from 'lodash';
-import { EntityV2, EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
+import { EntityDefinition, EntityDefinitionUpdate } from '@kbn/entities-schema';
 import { SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
-import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import { IScopedClusterClient } from '@kbn/core-elasticsearch-server';
 import { Logger } from '@kbn/logging';
 import {
   installEntityDefinition,
@@ -19,40 +18,24 @@ import { startTransforms } from './entities/start_transforms';
 import { findEntityDefinitionById, findEntityDefinitions } from './entities/find_entity_definition';
 import { uninstallEntityDefinition } from './entities/uninstall_entity_definition';
 import { EntityDefinitionNotFound } from './entities/errors/entity_not_found';
-
 import { stopTransforms } from './entities/stop_transforms';
 import { deleteIndices } from './entities/delete_index';
 import { EntityDefinitionWithState } from './entities/types';
 import { EntityDefinitionUpdateConflict } from './entities/errors/entity_definition_update_conflict';
-import { EntitySource, SortBy, getEntityInstancesQuery } from './queries';
-import { mergeEntitiesList, runESQLQuery } from './queries/utils';
-import { UnknownEntityType } from './entities/errors/unknown_entity_type';
-
-interface SearchCommon {
-  start: string;
-  end: string;
-  sort?: SortBy;
-  metadataFields?: string[];
-  filters?: string[];
-  limit?: number;
-}
-
-export type SearchByType = SearchCommon & {
-  type: string;
-};
-
-export type SearchBySources = SearchCommon & {
-  sources: EntitySource[];
-};
+import { EntityClient as EntityClient_v2 } from './v2/entity_client';
 
 export class EntityClient {
+  public v2: EntityClient_v2;
+
   constructor(
     private options: {
-      esClient: ElasticsearchClient;
+      clusterClient: IScopedClusterClient;
       soClient: SavedObjectsClientContract;
       logger: Logger;
     }
-  ) {}
+  ) {
+    this.v2 = new EntityClient_v2(options);
+  }
 
   async createEntityDefinition({
     definition,
@@ -66,13 +49,17 @@ export class EntityClient {
     );
     const installedDefinition = await installEntityDefinition({
       definition,
-      esClient: this.options.esClient,
+      esClient: this.options.clusterClient.asCurrentUser,
       soClient: this.options.soClient,
       logger: this.options.logger,
     });
 
     if (!installOnly) {
-      await startTransforms(this.options.esClient, installedDefinition, this.options.logger);
+      await startTransforms(
+        this.options.clusterClient.asCurrentUser,
+        installedDefinition,
+        this.options.logger
+      );
     }
 
     return installedDefinition;
@@ -88,7 +75,7 @@ export class EntityClient {
     const definition = await findEntityDefinitionById({
       id,
       soClient: this.options.soClient,
-      esClient: this.options.esClient,
+      esClient: this.options.clusterClient.asCurrentUser,
       includeState: true,
     });
 
@@ -115,20 +102,24 @@ export class EntityClient {
       definition,
       definitionUpdate,
       soClient: this.options.soClient,
-      esClient: this.options.esClient,
+      esClient: this.options.clusterClient.asCurrentUser,
       logger: this.options.logger,
     });
 
     if (shouldRestartTransforms) {
-      await startTransforms(this.options.esClient, updatedDefinition, this.options.logger);
+      await startTransforms(
+        this.options.clusterClient.asCurrentUser,
+        updatedDefinition,
+        this.options.logger
+      );
     }
     return updatedDefinition;
   }
 
   async deleteEntityDefinition({ id, deleteData = false }: { id: string; deleteData?: boolean }) {
     const definition = await findEntityDefinitionById({
       id,
-      esClient: this.options.esClient,
+      esClient: this.options.clusterClient.asCurrentUser,
       soClient: this.options.soClient,
     });
 
@@ -141,13 +132,17 @@ export class EntityClient {
     );
     await uninstallEntityDefinition({
       definition,
-      esClient: this.options.esClient,
+      esClient: this.options.clusterClient.asCurrentUser,
       soClient: this.options.soClient,
       logger: this.options.logger,
     });
 
     if (deleteData) {
-      await deleteIndices(this.options.esClient, definition, this.options.logger);
+      await deleteIndices(
+        this.options.clusterClient.asCurrentUser,
+        definition,
+        this.options.logger
+      );
     }
   }
 
@@ -167,7 +162,7 @@ export class EntityClient {
     builtIn?: boolean;
   }) {
     const definitions = await findEntityDefinitions({
-      esClient: this.options.esClient,
+      esClient: this.options.clusterClient.asCurrentUser,
       soClient: this.options.soClient,
       page,
       perPage,
@@ -182,124 +177,19 @@ export class EntityClient {
 
   async startEntityDefinition(definition: EntityDefinition) {
     this.options.logger.info(`Starting transforms for definition [${definition.id}]`);
-    return startTransforms(this.options.esClient, definition, this.options.logger);
+    return startTransforms(
+      this.options.clusterClient.asCurrentUser,
+      definition,
+      this.options.logger
+    );
   }
 
   async stopEntityDefinition(definition: EntityDefinition) {
     this.options.logger.info(`Stopping transforms for definition [${definition.id}]`);
-    return stopTransforms(this.options.esClient, definition, this.options.logger);
-  }
-
-  async getEntitySources({ type }: { type: string }) {
-    const result = await this.options.esClient.search<EntitySource>({
-      index: 'kibana_entity_definitions',
-      query: {
-        bool: {
-          must: {
-            term: { entity_type: type },
-          },
-        },
-      },
-    });
-
-    return result.hits.hits.map((hit) => hit._source) as EntitySource[];
-  }
-
-  async searchEntities({
-    type,
-    start,
-    end,
-    sort,
-    metadataFields = [],
-    filters = [],
-    limit = 10,
-  }: SearchByType) {
-    const sources = await this.getEntitySources({ type });
-    if (sources.length === 0) {
-      throw new UnknownEntityType(`No sources found for entity type [${type}]`);
-    }
-
-    return this.searchEntitiesBySources({
-      sources,
-      start,
-      end,
-      metadataFields,
-      filters,
-      sort,
-      limit,
-    });
-  }
-
-  async searchEntitiesBySources({
-    sources,
-    start,
-    end,
-    sort,
-    metadataFields = [],
-    filters = [],
-    limit = 10,
-  }: SearchBySources) {
-    const entities = await Promise.all(
-      sources.map(async (source) => {
-        const mandatoryFields = [
-          ...source.identity_fields,
-          ...(source.timestamp_field ? [source.timestamp_field] : []),
-          ...(source.display_name ? [source.display_name] : []),
-        ];
-        const metaFields = [...metadataFields, ...source.metadata_fields];
-
-        // operations on an unmapped field result in a failing query so we verify
-        // field capabilities beforehand
-        const { fields } = await this.options.esClient.fieldCaps({
-          index: source.index_patterns,
-          fields: [...mandatoryFields, ...metaFields],
-        });
-
-        const sourceHasMandatoryFields = mandatoryFields.every((field) => !!fields[field]);
-        if (!sourceHasMandatoryFields) {
-          // we can't build entities without id fields so we ignore the source.
-          // TODO filters should likely behave similarly. we should also throw
-          const missingFields = mandatoryFields.filter((field) => !fields[field]);
-          this.options.logger.info(
-            `Ignoring source for type [${source.type}] with index_patterns [${
-              source.index_patterns
-            }] because some mandatory fields [${missingFields.join(', ')}] are not mapped`
-          );
-          return [];
-        }
-
-        // but metadata field not being available is fine
-        const availableMetadataFields = metaFields.filter((field) => fields[field]);
-        if (availableMetadataFields.length < metaFields.length) {
-          this.options.logger.info(
-            `Ignoring unmapped fields [${without(metaFields, ...availableMetadataFields).join(
-              ', '
-            )}]`
-          );
-        }
-
-        const query = getEntityInstancesQuery({
-          source: {
-            ...source,
-            metadata_fields: availableMetadataFields,
-            filters: [...source.filters, ...filters],
-          },
-          start,
-          end,
-          sort,
-          limit,
-        });
-        this.options.logger.debug(`Entity query: ${query}`);
-
-        const rawEntities = await runESQLQuery<EntityV2>({
-          query,
-          esClient: this.options.esClient,
-        });
-
-        return rawEntities;
-      })
-    ).then((results) => results.flat());
-
-    return mergeEntitiesList(sources, entities).slice(0, limit);
+    return stopTransforms(
+      this.options.clusterClient.asCurrentUser,
+      definition,
+      this.options.logger
+    );
   }
 }
diff --git a/x-pack/plugins/entity_manager/server/lib/v2/constants.ts b/x-pack/plugins/entity_manager/server/lib/v2/constants.ts
@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+// Definitions index
+
+export const DEFINITIONS_ALIAS = '.kibana-entities-definitions';
+export const TEMPLATE_VERSION = 1;
+
+// Privileges
+
+export const CREATE_ENTITY_TYPE_DEFINITION_PRIVILEGE = 'create_entity_type_definition';
+export const CREATE_ENTITY_SOURCE_DEFINITION_PRIVILEGE = 'create_entity_source_definition';
+export const READ_ENTITY_TYPE_DEFINITION_PRIVILEGE = 'read_entity_type_definition';
+export const READ_ENTITY_SOURCE_DEFINITION_PRIVILEGE = 'read_entity_source_definition';
+export const READ_ENTITIES_PRIVILEGE = 'read_entities';