[Ingest] Add additional attributes to the Datasources Saved Object (#…

…66127) (#66313)

* add additional properties to ingest-datasources SO
* Adjusted Types and test generators
* Added datasources migrations to SO
* Add `user` object to calls to datasource.create()

x-pack/plugins/endpoint/common/generate_data.ts

@@ -504,10 +504,15 @@ export class EndpointDocGenerator {
    * Generates an Ingest `datasource` that includes the Endpoint Policy data
    */
   public generatePolicyDatasource(): PolicyData {
+    const created = new Date(Date.now() - 8.64e7).toISOString(); // 24h ago
     return {
       id: this.seededUUIDv4(),
       name: 'Endpoint Policy',
       description: 'Policy to protect the worlds data',
+      created_at: created,
+      created_by: 'elastic',
+      updated_at: new Date().toISOString(),
+      updated_by: 'elastic',
       config_id: this.seededUUIDv4(),
       enabled: true,
       output_id: '',

x-pack/plugins/endpoint/public/applications/endpoint/store/policy_details/index.test.ts

@@ -28,6 +28,10 @@ describe('policy details: ', () => {
           id: '',
           name: '',
           description: '',
+          created_at: '',
+          created_by: '',
+          updated_at: '',
+          updated_by: '',
           config_id: '',
           enabled: true,
           output_id: '',

x-pack/plugins/endpoint/public/applications/endpoint/store/policy_details/middleware.ts

@@ -5,13 +5,18 @@
  */
 
 import { ImmutableMiddlewareFactory, PolicyDetailsState, UpdatePolicyResponse } from '../../types';
-import { policyIdFromParams, isOnPolicyDetailsPage, policyDetails } from './selectors';
+import {
+  policyIdFromParams,
+  isOnPolicyDetailsPage,
+  policyDetails,
+  policyDetailsForUpdate,
+} from './selectors';
 import {
   sendGetDatasource,
   sendGetFleetAgentStatusForConfig,
   sendPutDatasource,
 } from '../policy_list/services/ingest';
-import { PolicyData } from '../../../../../common/types';
+import { NewPolicyData, PolicyData } from '../../../../../common/types';
 import { factory as policyConfigFactory } from '../../../../../common/models/policy_config';
 
 export const policyDetailsMiddlewareFactory: ImmutableMiddlewareFactory<PolicyDetailsState> = coreStart => {
@@ -71,7 +76,8 @@ export const policyDetailsMiddlewareFactory: ImmutableMiddlewareFactory<PolicyDe
         });
       }
     } else if (action.type === 'userClickedPolicyDetailsSaveButton') {
-      const { id, revision, ...updatedPolicyItem } = policyDetails(state) as PolicyData;
+      const { id } = policyDetails(state) as PolicyData;
+      const updatedPolicyItem = policyDetailsForUpdate(state) as NewPolicyData;
 
       let apiResponse: UpdatePolicyResponse;
       try {

x-pack/plugins/endpoint/public/applications/endpoint/store/policy_details/selectors.ts

@@ -6,12 +6,29 @@
 
 import { createSelector } from 'reselect';
 import { PolicyDetailsState } from '../../types';
-import { Immutable, PolicyConfig, UIPolicyConfig } from '../../../../../common/types';
+import {
+  Immutable,
+  NewPolicyData,
+  PolicyConfig,
+  UIPolicyConfig,
+} from '../../../../../common/types';
 import { factory as policyConfigFactory } from '../../../../../common/models/policy_config';
 
 /** Returns the policy details */
 export const policyDetails = (state: Immutable<PolicyDetailsState>) => state.policyItem;
 
+/**
+ * Return only the policy structure accepted for update/create
+ */
+export const policyDetailsForUpdate: (
+  state: Immutable<PolicyDetailsState>
+) => Immutable<NewPolicyData> | undefined = createSelector(policyDetails, policy => {
+  if (policy) {
+    const { id, revision, created_by, created_at, updated_by, updated_at, ...newPolicy } = policy;
+    return newPolicy;
+  }
+});
+
 /** Returns a boolean of whether the user is on the policy details page or not */
 export const isOnPolicyDetailsPage = (state: Immutable<PolicyDetailsState>) => {
   if (state.location) {

x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts

@@ -11,6 +11,10 @@ describe('Ingest Manager - storedDatasourceToAgentDatasource', () => {
     id: 'some-uuid',
     name: 'mock-datasource',
     description: '',
+    created_at: '',
+    created_by: '',
+    updated_at: '',
+    updated_by: '',
     config_id: '',
     enabled: true,
     output_id: '',

x-pack/plugins/ingest_manager/common/types/models/datasource.ts

@@ -58,4 +58,8 @@ export interface Datasource extends Omit<NewDatasource, 'inputs'> {
   id: string;
   inputs: DatasourceInput[];
   revision: number;
+  updated_at: string;
+  updated_by: string;
+  created_at: string;
+  created_by: string;
 }

x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_config/edit_datasource_page/index.tsx

@@ -78,7 +78,16 @@ export const EditDatasourcePage: React.FunctionComponent = () => {
           setAgentConfig(agentConfigData.item);
         }
         if (datasourceData?.item) {
-          const { id, revision, inputs, ...restOfDatasource } = datasourceData.item;
+          const {
+            id,
+            revision,
+            inputs,
+            created_by,
+            created_at,
+            updated_by,
+            updated_at,
+            ...restOfDatasource
+          } = datasourceData.item;
           // Remove `agent_stream` from all stream info, we assign this after saving
           const newDatasource = {
             ...restOfDatasource,

x-pack/plugins/ingest_manager/server/routes/agent_config/handlers.ts

@@ -100,14 +100,14 @@ export const createAgentConfigHandler: RequestHandler<
   TypeOf<typeof CreateAgentConfigRequestSchema.body>
 > = async (context, request, response) => {
   const soClient = context.core.savedObjects.client;
-  const user = await appContextService.getSecurity()?.authc.getCurrentUser(request);
+  const user = (await appContextService.getSecurity()?.authc.getCurrentUser(request)) || undefined;
   const withSysMonitoring = request.query.sys_monitoring ?? false;
   try {
     // eslint-disable-next-line prefer-const
     let [agentConfig, newSysDatasource] = await Promise.all<AgentConfig, NewDatasource | undefined>(
       [
         agentConfigService.create(soClient, request.body, {
-          user: user || undefined,
+          user,
         }),
         // If needed, retrieve System package information and build a new Datasource for the system package
         // NOTE: we ignore failures in attempting to create datasource, since config might have been created
@@ -123,7 +123,7 @@ export const createAgentConfigHandler: RequestHandler<
     // Create the system monitoring datasource and add it to config.
     if (withSysMonitoring && newSysDatasource !== undefined && agentConfig !== undefined) {
       newSysDatasource.config_id = agentConfig.id;
-      const sysDatasource = await datasourceService.create(soClient, newSysDatasource);
+      const sysDatasource = await datasourceService.create(soClient, newSysDatasource, { user });
 
       if (sysDatasource) {
         agentConfig = await agentConfigService.assignDatasources(soClient, agentConfig.id, [

x-pack/plugins/ingest_manager/server/saved_objects.ts → x-pack/plugins/ingest_manager/server/saved_objects/index.ts

@@ -5,7 +5,7 @@
  */
 
 import { SavedObjectsServiceSetup, SavedObjectsType } from 'kibana/server';
-import { EncryptedSavedObjectsPluginSetup } from '../../encrypted_saved_objects/server';
+import { EncryptedSavedObjectsPluginSetup } from '../../../encrypted_saved_objects/server';
 import {
   OUTPUT_SAVED_OBJECT_TYPE,
   AGENT_CONFIG_SAVED_OBJECT_TYPE,
@@ -16,7 +16,8 @@ import {
   AGENT_ACTION_SAVED_OBJECT_TYPE,
   ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE,
   GLOBAL_SETTINGS_SAVED_OBJET_TYPE,
-} from './constants';
+} from '../constants';
+import { migrateDatasourcesToV790 } from './migrations/datasources_v790';
 
 /*
  * Saved object types and mappings
@@ -218,8 +219,15 @@ const savedObjectTypes: { [key: string]: SavedObjectsType } = {
           },
         },
         revision: { type: 'integer' },
+        updated_at: { type: 'date' },
+        updated_by: { type: 'keyword' },
+        created_at: { type: 'date' },
+        created_by: { type: 'keyword' },
       },
     },
+    migrations: {
+      '7.9.0': migrateDatasourcesToV790,
+    },
   },
   [PACKAGES_SAVED_OBJECT_TYPE]: {
     name: PACKAGES_SAVED_OBJECT_TYPE,

x-pack/plugins/ingest_manager/server/saved_objects/migrations/datasources_v790.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SavedObjectMigrationFn } from 'kibana/server';
+import { cloneDeep } from 'lodash';
+import { Datasource } from '../../types/models';
+
+type Pre790Datasource = Exclude<
+  Datasource,
+  'created_at' | 'created_by' | 'updated_at' | 'updated_by'
+>;
+
+export const migrateDatasourcesToV790: SavedObjectMigrationFn<
+  Pre790Datasource,
+  Datasource
+> = doc => {
+  const updatedDatasource = cloneDeep(doc);
+  const defDate = new Date().toISOString();
+
+  updatedDatasource.attributes.created_by = 'system';
+  updatedDatasource.attributes.created_at = updatedDatasource?.updated_at ?? defDate;
+  updatedDatasource.attributes.updated_by = 'system';
+  updatedDatasource.attributes.updated_at = updatedDatasource?.updated_at ?? defDate;
+
+  return updatedDatasource;
+};

x-pack/plugins/ingest_manager/server/services/datasource.ts

@@ -31,11 +31,16 @@ class DatasourceService {
     datasource: NewDatasource,
     options?: { id?: string; user?: AuthenticatedUser }
   ): Promise<Datasource> {
+    const isoDate = new Date().toISOString();
     const newSo = await soClient.create<Omit<Datasource, 'id'>>(
       SAVED_OBJECT_TYPE,
       {
         ...datasource,
         revision: 1,
+        created_at: isoDate,
+        created_by: options?.user?.username ?? 'system',
+        updated_at: isoDate,
+        updated_by: options?.user?.username ?? 'system',
       },
       options
     );
@@ -134,6 +139,8 @@ class DatasourceService {
     await soClient.update<Datasource>(SAVED_OBJECT_TYPE, id, {
       ...datasource,
       revision: oldDatasource.revision + 1,
+      updated_at: new Date().toISOString(),
+      updated_by: options?.user?.username ?? 'system',
     });
 
     // Bump revision of associated agent config