[EEM] Add versioning for entity definitions

x-pack/packages/kbn-entities-schema/src/schema/common.ts

@@ -96,3 +96,8 @@ export const identityFieldsSchema = z
     optional: z.boolean(),
   })
   .or(z.string().transform((value) => ({ field: value, optional: false })));
+
+const semVerRegex = new RegExp(/^[0-9]{1,}\.[0-9]{1,}\.[0-9]{1,}$/);
+export const semVerSchema = z.string().refine((maybeSemVer) => semVerRegex.test(maybeSemVer), {
+  message: 'The string does not match the semver pattern of 1.1.1',
+});

x-pack/packages/kbn-entities-schema/src/schema/entity_definition.ts

@@ -13,10 +13,12 @@ import {
   filterSchema,
   durationSchema,
   identityFieldsSchema,
+  semVerSchema,
 } from './common';
 
 export const entityDefinitionSchema = z.object({
   id: z.string().regex(/^[\w-]+$/),
+  version: semVerSchema,
   name: z.string(),
   description: z.optional(z.string()),
   type: z.string(),

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/built_in/services.ts

@@ -10,6 +10,7 @@ import { BUILT_IN_ID_PREFIX } from './constants';
 
 export const builtInServicesEntityDefinition: EntityDefinition = entityDefinitionSchema.parse({
   id: `${BUILT_IN_ID_PREFIX}services`,
+  version: '1.0.0',
   name: 'Services from logs',
   type: 'service',
   managed: true,

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/create_and_install_ingest_pipeline.ts

@@ -26,6 +26,9 @@ export async function createAndInstallHistoryIngestPipeline(
         esClient.ingest.putPipeline({
           id: historyId,
           processors: historyProcessors,
+          _meta: {
+            definitionVersion: definition.version,
+          },
         }),
       { logger }
     );
@@ -49,6 +52,9 @@ export async function createAndInstallLatestIngestPipeline(
         esClient.ingest.putPipeline({
           id: latestId,
           processors: latestProcessors,
+          _meta: {
+            definitionVersion: definition.version,
+          },
         }),
       { logger }
     );

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/helpers/fixtures/entity_definition.ts

@@ -8,6 +8,7 @@
 import { entityDefinitionSchema } from '@kbn/entities-schema';
 export const entityDefinition = entityDefinitionSchema.parse({
   id: 'admin-console-services',
+  version: '999.999.999',
   name: 'Services for Admin Console',
   type: 'service',
   indexPatterns: ['kbn-data-forge-fake_stack.*'],

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_history_processors.ts

@@ -63,6 +63,18 @@ export function generateHistoryProcessors(definition: EntityDefinition) {
         value: definition.id,
       },
     },
+    {
+      set: {
+        field: 'entity.definitionVersion',
+        value: definition.version,
+      },
+    },
+    {
+      set: {
+        field: 'entity.schemaVersion',
+        value: 'v1',
+      },
+    },
     {
       set: {
         field: 'entity.displayName',

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/ingest_pipeline/generate_latest_processors.ts

@@ -57,6 +57,18 @@ export function generateLatestProcessors(definition: EntityDefinition) {
         value: definition.id,
       },
     },
+    {
+      set: {
+        field: 'entity.definitionVersion',
+        value: definition.version,
+      },
+    },
+    {
+      set: {
+        field: 'entity.schemaVersion',
+        value: 'v1',
+      },
+    },
     ...(definition.staticFields != null
       ? Object.keys(definition.staticFields).map((field) => ({
           set: { field, value: definition.staticFields![field] },

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/install_entity_definition.test.ts

@@ -36,10 +36,16 @@ const assertHasCreatedDefinition = (
   expect(esClient.ingest.putPipeline).toBeCalledWith({
     id: generateHistoryIngestPipelineId(builtInServicesEntityDefinition),
     processors: expect.anything(),
+    _meta: {
+      definitionVersion: '1.0.0',
+    },
   });
   expect(esClient.ingest.putPipeline).toBeCalledWith({
     id: generateLatestIngestPipelineId(builtInServicesEntityDefinition),
     processors: expect.anything(),
+    _meta: {
+      definitionVersion: '1.0.0',
+    },
   });
 
   expect(esClient.transform.putTransform).toBeCalledTimes(2);

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_history_transform.ts

@@ -32,6 +32,9 @@ export function generateHistoryTransform(
 
   return {
     transform_id: generateHistoryTransformId(definition),
+    _meta: {
+      definitionVersion: definition.version,
+    },
     defer_validation: true,
     source: {
       index: definition.indexPatterns,

x-pack/plugins/observability_solution/entity_manager/server/lib/entities/transform/generate_latest_transform.ts

@@ -22,6 +22,9 @@ export function generateLatestTransform(
 ): TransformPutTransformRequest {
   return {
     transform_id: generateLatestTransformId(definition),
+    _meta: {
+      definitionVersion: definition.version,
+    },
     defer_validation: true,
     source: {
       index: `${generateHistoryIndexName(definition)}.*`,

x-pack/plugins/observability_solution/entity_manager/server/saved_objects/entity_definition.ts

@@ -18,6 +18,7 @@ export const entityDefinition: SavedObjectsType = {
     dynamic: false,
     properties: {
       id: { type: 'keyword' },
+      version: { type: 'keyword' },
       name: { type: 'text' },
       description: { type: 'text' },
       type: { type: 'keyword' },

x-pack/plugins/observability_solution/entity_manager/server/templates/components/entity.ts

@@ -38,6 +38,14 @@ export const entitiesEntityComponentTemplateConfig: ClusterPutComponentTemplateR
               ignore_above: 1024,
               type: 'keyword',
             },
+            definitionVersion: {
+              ignore_above: 1024,
+              type: 'keyword',
+            },
+            schemaVersion: {
+              ignore_above: 1024,
+              type: 'keyword',
+            },
             lastSeenTimestamp: {
               type: 'date',
             },