[EEM] Add versioning for entity definitions (#187692)

This PR adds a `version` field to the `EntityDefinition` type, making it required in the API calls. It must be a SemVer string. The version is added to the ingest pipelines and transforms as part of their metadata. The version is included in the output documents alongside the schema version. --------- Co-authored-by: kibanamachine <[email protected]>
elastic · Jul 9, 2024 · 47178a7 · 47178a7
1 parent b2e82f6
commit 47178a7
Show file tree

Hide file tree

Showing 19 changed files with 111 additions and 2 deletions.
diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json
@@ -301,7 +301,8 @@
     "metrics",
     "name",
     "staticFields",
-    "type"
+    "type",
+    "version"
   ],
   "entity-discovery-api-key": [
     "apiKey"

diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json
@@ -1028,6 +1028,9 @@
       },
       "type": {
         "type": "keyword"
+      },
+      "version": {
+        "type": "keyword"
       }
     }
   },

diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
@@ -88,7 +88,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "endpoint:unified-user-artifact-manifest": "71c7fcb52c658b21ea2800a6b6a76972ae1c776e",
         "endpoint:user-artifact-manifest": "1c3533161811a58772e30cdc77bac4631da3ef2b",
         "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d",
-        "entity-definition": "33fe0194bd896f0bfe479d55f6de20f8ba1d7713",
+        "entity-definition": "331a2ba0ee9f24936ef049683549c8af7e46f03a",
         "entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88",
         "epm-packages": "8042d4a1522f6c4e6f5486e791b3ffe3a22f88fd",
         "epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1",

diff --git a/x-pack/packages/kbn-entities-schema/src/schema/common.ts b/x-pack/packages/kbn-entities-schema/src/schema/common.ts
@@ -96,3 +96,9 @@ export const identityFieldsSchema = z
     optional: z.boolean(),
   })
   .or(z.string().transform((value) => ({ field: value, optional: false })));
+
+const semVerRegex = new RegExp(/^[0-9]{1,}\.[0-9]{1,}\.[0-9]{1,}$/);
+export const semVerSchema = z.string().refine((maybeSemVer) => semVerRegex.test(maybeSemVer), {
+  message:
+    'The string does use the Semantic Versioning (Semver) format of {major}.{minor}.{patch} (e.g., 1.0.0), ensure each part contains only digits.',
+});
diff --git a/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts b/x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts
@@ -13,10 +13,12 @@ import {
   filterSchema,
   durationSchema,
   identityFieldsSchema,
+  semVerSchema,
 } from './common';
 
 export const entityDefinitionSchema = z.object({
   id: z.string().regex(/^[\w-]+$/),
+  version: semVerSchema,
   name: z.string(),
   description: z.optional(z.string()),
   type: z.string(),

diff --git a/...ck/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts b/...ck/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts
@@ -10,6 +10,7 @@ import { BUILT_IN_ID_PREFIX } from './constants';
 
 export const builtInServicesEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({
   id: `${BUILT_IN_ID_PREFIX}services`,
+  version: '0.1.0',
   name: 'Services from logs',
   type: 'service',
   managed: true,

diff --git a/...ability_solution/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts b/...ability_solution/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts
@@ -28,6 +28,9 @@ export async function createAndInstallHistoryIngestPipeline(
         esClient.ingest.putPipeline({
           id: historyId,
           processors: historyProcessors,
+          _meta: {
+            definitionVersion: definition.version,
+          },
         }),
       { logger }
     );
@@ -51,6 +54,9 @@ export async function createAndInstallLatestIngestPipeline(
         esClient.ingest.putPipeline({
           id: latestId,
           processors: latestProcessors,
+          _meta: {
+            definitionVersion: definition.version,
+          },
         }),
       { logger }
     );

diff --git a/...ability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts b/...ability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts
@@ -8,6 +8,7 @@
 import { entityDefinitionSchema } from '@kbn/entities-schema';
 export const entityDefinition = entityDefinitionSchema.parse({
   id: 'admin-console-services',
+  version: '999.999.999',
   name: 'Services for Admin Console',
   type: 'service',
   indexPatterns: ['kbn-data-forge-fake_stack.*'],

diff --git a/...erver/lib/entities/ingest_pipeline/__snapshots__/generate_history_processors.test.ts.snap b/...erver/lib/entities/ingest_pipeline/__snapshots__/generate_history_processors.test.ts.snap
diff --git a/...server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap b/...server/lib/entities/ingest_pipeline/__snapshots__/generate_latest_processors.test.ts.snap
diff --git a/...olution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts b/...olution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts
@@ -6,6 +6,7 @@
  */
 
 import { EntityDefinition } from '@kbn/entities-schema';
+import { ENTITY_SCHEMA_VERSION_V1 } from '../../../../common/constants_entities';
 import { generateHistoryIndexName } from '../helpers/generate_component_id';
 
 function createIdTemplate(definition: EntityDefinition) {
@@ -62,6 +63,18 @@ export function generateHistoryProcessors(definition: EntityDefinition) {
         value: definition.id,
       },
     },
+    {
+      set: {
+        field: 'entity.definitionVersion',
+        value: definition.version,
+      },
+    },
+    {
+      set: {
+        field: 'entity.schemaVersion',
+        value: ENTITY_SCHEMA_VERSION_V1,
+      },
+    },
     {
       set: {
         field: 'entity.displayName',

diff --git a/...solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts b/...solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts
@@ -6,6 +6,7 @@
  */
 
 import { EntityDefinition } from '@kbn/entities-schema';
+import { ENTITY_SCHEMA_VERSION_V1 } from '../../../../common/constants_entities';
 import { generateLatestIndexName } from '../helpers/generate_component_id';
 
 function mapDestinationToPainless(destination: string) {
@@ -56,6 +57,18 @@ export function generateLatestProcessors(definition: EntityDefinition) {
         value: definition.id,
       },
     },
+    {
+      set: {
+        field: 'entity.definitionVersion',
+        value: definition.version,
+      },
+    },
+    {
+      set: {
+        field: 'entity.schemaVersion',
+        value: ENTITY_SCHEMA_VERSION_V1,
+      },
+    },
     ...(definition.staticFields != null
       ? Object.keys(definition.staticFields).map((field) => ({
           set: { field, value: definition.staticFields![field] },

diff --git a/...servability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts b/...servability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts
@@ -38,10 +38,16 @@ const assertHasCreatedDefinition = (
   expect(esClient.ingest.putPipeline).toBeCalledWith({
     id: generateHistoryIngestPipelineId(builtInServicesEntityDefinition),
     processors: expect.anything(),
+    _meta: {
+      definitionVersion: '0.1.0',
+    },
   });
   expect(esClient.ingest.putPipeline).toBeCalledWith({
     id: generateLatestIngestPipelineId(builtInServicesEntityDefinition),
     processors: expect.anything(),
+    _meta: {
+      definitionVersion: '0.1.0',
+    },
   });
 
   expect(esClient.transform.putTransform).toBeCalledTimes(2);

diff --git a/...nager/server/lib/entities/transform/__snapshots__/generate_history_transform.test.ts.snap b/...nager/server/lib/entities/transform/__snapshots__/generate_history_transform.test.ts.snap
diff --git a/...anager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap b/...anager/server/lib/entities/transform/__snapshots__/generate_latest_transform.test.ts.snap
diff --git a/...ility_solution/entity_manager/server/lib/entities/transform/generate_history_transform.ts b/...ility_solution/entity_manager/server/lib/entities/transform/generate_history_transform.ts
@@ -34,6 +34,9 @@ export function generateHistoryTransform(
 
   return {
     transform_id: generateHistoryTransformId(definition),
+    _meta: {
+      definitionVersion: definition.version,
+    },
     defer_validation: true,
     source: {
       index: definition.indexPatterns,

diff --git a/...bility_solution/entity_manager/server/lib/entities/transform/generate_latest_transform.ts b/...bility_solution/entity_manager/server/lib/entities/transform/generate_latest_transform.ts
@@ -25,6 +25,9 @@ export function generateLatestTransform(
 ): TransformPutTransformRequest {
   return {
     transform_id: generateLatestTransformId(definition),
+    _meta: {
+      definitionVersion: definition.version,
+    },
     defer_validation: true,
     source: {
       index: `${generateHistoryIndexName(definition)}.*`,

diff --git a/...k/plugins/observability_solution/entity_manager/server/saved_objects/entity_definition.ts b/...k/plugins/observability_solution/entity_manager/server/saved_objects/entity_definition.ts
@@ -18,6 +18,7 @@ export const entityDefinition: SavedObjectsType = {
     dynamic: false,
     properties: {
       id: { type: 'keyword' },
+      version: { type: 'keyword' },
       name: { type: 'text' },
       description: { type: 'text' },
       type: { type: 'keyword' },
@@ -37,4 +38,16 @@ export const entityDefinition: SavedObjectsType = {
       return `EntityDefinition: [${savedObject.attributes.name}]`;
     },
   },
+  modelVersions: {
+    '1': {
+      changes: [
+        {
+          type: 'mappings_addition',
+          addedMappings: {
+            version: { type: 'keyword' },
+          },
+        },
+      ],
+    },
+  },
 };
diff --git a/x-pack/plugins/observability_solution/entity_manager/server/templates/components/entity.ts b/x-pack/plugins/observability_solution/entity_manager/server/templates/components/entity.ts
@@ -41,6 +41,14 @@ export const entitiesEntityComponentTemplateConfig: ClusterPutComponentTemplateR
               ignore_above: 1024,
               type: 'keyword',
             },
+            definitionVersion: {
+              ignore_above: 1024,
+              type: 'keyword',
+            },
+            schemaVersion: {
+              ignore_above: 1024,
+              type: 'keyword',
+            },
             lastSeenTimestamp: {
               type: 'date',
             },