Merge remote-tracking branch 'upstream/renovate/main-@elasticcharts' …

…into renovate/main-@elasticcharts

.github/CODEOWNERS

@@ -395,6 +395,7 @@ x-pack/plugins/encrypted_saved_objects @elastic/kibana-security
 x-pack/plugins/enterprise_search @elastic/search-kibana
 x-pack/plugins/observability_solution/entities_data_access @elastic/obs-entities
 x-pack/packages/kbn-entities-schema @elastic/obs-entities
+x-pack/test/api_integration/apis/entity_manager/fixture_plugin @elastic/obs-entities
 x-pack/plugins/observability_solution/entity_manager @elastic/obs-entities
 examples/error_boundary @elastic/appex-sharedux
 packages/kbn-es @elastic/kibana-operations

package.json

@@ -457,6 +457,7 @@
     "@kbn/enterprise-search-plugin": "link:x-pack/plugins/enterprise_search",
     "@kbn/entities-data-access-plugin": "link:x-pack/plugins/observability_solution/entities_data_access",
     "@kbn/entities-schema": "link:x-pack/packages/kbn-entities-schema",
+    "@kbn/entity-manager-fixture-plugin": "link:x-pack/test/api_integration/apis/entity_manager/fixture_plugin",
     "@kbn/entityManager-plugin": "link:x-pack/plugins/observability_solution/entity_manager",
     "@kbn/error-boundary-example-plugin": "link:examples/error_boundary",
     "@kbn/es-errors": "link:packages/kbn-es-errors",

src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts

@@ -90,7 +90,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "endpoint:unified-user-artifact-manifest": "71c7fcb52c658b21ea2800a6b6a76972ae1c776e",
         "endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b",
         "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d",
-        "entity-definition": "331a2ba0ee9f24936ef049683549c8af7e46f03a",
+        "entity-definition": "61be3e95966045122b55e181bb39658b1dc9bbe9",
         "entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88",
         "epm-packages": "8042d4a1522f6c4e6f5486e791b3ffe3a22f88fd",
         "epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1",

tsconfig.base.json

@@ -784,6 +784,8 @@
       "@kbn/entities-data-access-plugin/*": ["x-pack/plugins/observability_solution/entities_data_access/*"],
       "@kbn/entities-schema": ["x-pack/packages/kbn-entities-schema"],
       "@kbn/entities-schema/*": ["x-pack/packages/kbn-entities-schema/*"],
+      "@kbn/entity-manager-fixture-plugin": ["x-pack/test/api_integration/apis/entity_manager/fixture_plugin"],
+      "@kbn/entity-manager-fixture-plugin/*": ["x-pack/test/api_integration/apis/entity_manager/fixture_plugin/*"],
       "@kbn/entityManager-plugin": ["x-pack/plugins/observability_solution/entity_manager"],
       "@kbn/entityManager-plugin/*": ["x-pack/plugins/observability_solution/entity_manager/*"],
       "@kbn/error-boundary-example-plugin": ["examples/error_boundary"],

x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts

@@ -48,6 +48,15 @@ export const entityDefinitionSchema = z.object({
       ),
     })
   ),
+  installStatus: z.optional(
+    z.union([
+      z.literal('installing'),
+      z.literal('upgrading'),
+      z.literal('installed'),
+      z.literal('failed'),
+    ])
+  ),
+  installStartedAt: z.optional(z.string()),
 });
 
 export type EntityDefinition = z.infer<typeof entityDefinitionSchema>;

x-pack/plugins/observability_solution/entity_manager/common/errors.ts

@@ -10,3 +10,4 @@ export const ERROR_API_KEY_NOT_VALID = 'api_key_not_valid';
 export const ERROR_API_KEY_SERVICE_DISABLED = 'api_key_service_disabled';
 export const ERROR_PARTIAL_BUILTIN_INSTALLATION = 'partial_builtin_installation';
 export const ERROR_DEFINITION_STOPPED = 'error_definition_stopped';
+export const ERROR_BUILTIN_UPGRADE_REQUIRED = 'builtin_upgrade_required';

x-pack/plugins/observability_solution/entity_manager/kibana.jsonc

@@ -6,7 +6,7 @@
   "plugin": {
     "id": "entityManager",
     "configPath": ["xpack", "entityManager"],
-    "requiredPlugins": ["security", "encryptedSavedObjects"],
+    "requiredPlugins": ["security", "encryptedSavedObjects", "licensing"],
     "browser": true,
     "server": true,
     "requiredBundles": []

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/delete_entity_definition.ts

@@ -5,27 +5,22 @@
  * 2.0.
  */
 
-import { Logger, SavedObjectsClientContract } from '@kbn/core/server';
+import { SavedObjectsClientContract, SavedObjectsErrorHelpers } from '@kbn/core/server';
 import { EntityDefinition } from '@kbn/entities-schema';
 import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects';
 import { EntityDefinitionNotFound } from './errors/entity_not_found';
 
 export async function deleteEntityDefinition(
   soClient: SavedObjectsClientContract,
-  definition: EntityDefinition,
-  logger: Logger
+  definition: EntityDefinition
 ) {
-  const response = await soClient.find<EntityDefinition>({
-    type: SO_ENTITY_DEFINITION_TYPE,
-    page: 1,
-    perPage: 1,
-    filter: `${SO_ENTITY_DEFINITION_TYPE}.attributes.id:(${definition.id})`,
-  });
+  try {
+    await soClient.delete(SO_ENTITY_DEFINITION_TYPE, definition.id);
+  } catch (err) {
+    if (SavedObjectsErrorHelpers.isNotFoundError(err)) {
+      throw new EntityDefinitionNotFound(`Entity definition with [${definition.id}] not found.`);
+    }
 
-  if (response.total === 0) {
-    logger.error(`Unable to delete entity definition [${definition.id}] because it doesn't exist.`);
-    throw new EntityDefinitionNotFound(`Entity definition with [${definition.id}] not found.`);
+    throw err;
   }
-
-  await soClient.delete(SO_ENTITY_DEFINITION_TYPE, response.saved_objects[0].id);
 }

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/find_entity_definition.ts

@@ -11,12 +11,16 @@ import { EntityDefinition } from '@kbn/entities-schema';
 import { SO_ENTITY_DEFINITION_TYPE } from '../../saved_objects';
 import {
   generateHistoryTransformId,
+  generateHistoryBackfillTransformId,
   generateHistoryIngestPipelineId,
+  generateHistoryIndexTemplateId,
   generateLatestTransformId,
   generateLatestIngestPipelineId,
+  generateLatestIndexTemplateId,
 } from './helpers/generate_component_id';
 import { BUILT_IN_ID_PREFIX } from './built_in';
 import { EntityDefinitionWithState } from './types';
+import { isBackfillEnabled } from './helpers/is_backfill_enabled';
 
 export async function findEntityDefinitions({
   soClient,
@@ -60,27 +64,40 @@ async function getEntityDefinitionState(
 ) {
   const historyIngestPipelineId = generateHistoryIngestPipelineId(definition);
   const latestIngestPipelineId = generateLatestIngestPipelineId(definition);
-  const [ingestPipelines, transforms] = await Promise.all([
-    esClient.ingest.getPipeline({
-      id: `${historyIngestPipelineId},${latestIngestPipelineId}`,
+  const transformIds = [
+    generateHistoryTransformId(definition),
+    generateLatestTransformId(definition),
+    ...(isBackfillEnabled(definition) ? [generateHistoryBackfillTransformId(definition)] : []),
+  ];
+  const [ingestPipelines, indexTemplatesInstalled, transforms] = await Promise.all([
+    esClient.ingest.getPipeline(
+      {
+        id: `${historyIngestPipelineId},${latestIngestPipelineId}`,
+      },
+      { ignore: [404] }
+    ),
+    esClient.indices.existsIndexTemplate({
+      name: `${
+        (generateLatestIndexTemplateId(definition), generateHistoryIndexTemplateId(definition))
+      }`,
     }),
     esClient.transform.getTransformStats({
-      transform_id: [generateHistoryTransformId(definition), generateLatestTransformId(definition)],
+      transform_id: transformIds,
     }),
   ]);
 
   const ingestPipelinesInstalled = !!(
     ingestPipelines[historyIngestPipelineId] && ingestPipelines[latestIngestPipelineId]
   );
-  const transformsInstalled = transforms.count === 2;
+  const transformsInstalled = transforms.count === transformIds.length;
   const transformsRunning =
     transformsInstalled &&
     transforms.transforms.every(
       (transform) => transform.state === 'started' || transform.state === 'indexing'
     );
 
   return {
-    installed: ingestPipelinesInstalled && transformsInstalled,
+    installed: ingestPipelinesInstalled && transformsInstalled && indexTemplatesInstalled,
     running: transformsRunning,
   };
 }

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/builtin_entity_definition.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { entityDefinitionSchema } from '@kbn/entities-schema';
+export const builtInEntityDefinition = entityDefinitionSchema.parse({
+  id: 'builtin_mock_entity_definition',
+  version: '1.0.0',
+  name: 'Mock builtin definition',
+  type: 'service',
+  indexPatterns: ['kbn-data-forge-fake_stack.*'],
+  managed: true,
+  history: {
+    timestampField: '@timestamp',
+    interval: '1m',
+  },
+  identityFields: ['log.logger', { field: 'event.category', optional: true }],
+  displayNameTemplate: '{{log.logger}}{{#event.category}}:{{.}}{{/event.category}}',
+  metadata: ['tags', 'host.name', 'host.os.name', { source: '_index', destination: 'sourceIndex' }],
+  metrics: [],
+});

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts

@@ -8,7 +8,7 @@
 import { entityDefinitionSchema } from '@kbn/entities-schema';
 export const rawEntityDefinition = {
   id: 'admin-console-services',
-  version: '999.999.999',
+  version: '1.0.0',
   name: 'Services for Admin Console',
   type: 'service',
   indexPatterns: ['kbn-data-forge-fake_stack.*'],

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/index.ts

@@ -7,3 +7,4 @@
 
 export { entityDefinition } from './entity_definition';
 export { entityDefinitionWithBackfill } from './entity_definition_with_backfill';
+export { builtInEntityDefinition } from './builtin_entity_definition';

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/generate_component_id.ts

@@ -6,6 +6,7 @@
  */
 
 import {
+  ENTITY_BASE_PREFIX,
   ENTITY_HISTORY,
   ENTITY_LATEST,
   ENTITY_SCHEMA_VERSION_V1,
@@ -38,6 +39,10 @@ export function generateHistoryIndexName(definition: EntityDefinition) {
   });
 }
 
+export function generateHistoryIndexTemplateId(definition: EntityDefinition) {
+  return `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_HISTORY}_${definition.id}_index_template` as const;
+}
+
 // Latest
 function generateLatestId(definition: EntityDefinition) {
   return `${ENTITY_LATEST_PREFIX_V1}-${definition.id}` as const;
@@ -53,3 +58,6 @@ export function generateLatestIndexName(definition: EntityDefinition) {
     definitionId: definition.id,
   });
 }
+
+export const generateLatestIndexTemplateId = (definition: EntityDefinition) =>
+  `${ENTITY_BASE_PREFIX}_${ENTITY_SCHEMA_VERSION_V1}_${ENTITY_LATEST}_${definition.id}_index_template` as const;