From 7ce1a9a89052008b8dd30f2dc38f81b80e7e644a Mon Sep 17 00:00:00 2001 From: Candace Park <56409205+parkiino@users.noreply.github.com> Date: Mon, 3 Apr 2023 07:51:36 -0400 Subject: [PATCH 01/10] [Security Solution][Admin][Policy] Add License to Endpoint policy (#153060) ## Summary - [x] Adds license information to endpoint policy for telemetry purposes - [x] Add migration for license info - [x] Adds unit tests # Screenshots See meta object with license information ![image](https://user-images.githubusercontent.com/56409205/224435361-c073f07a-b120-4d2b-863b-889b5251692e.png) Real endpoint Policy ![image](https://user-images.githubusercontent.com/56409205/228891124-4ffe2eb9-ffc3-4cd2-9cd2-711ab76fb30f.png) --- .../group2/check_registered_types.test.ts | 2 +- .../fleet/server/saved_objects/index.ts | 6 +- .../migrations/security_solution/index.ts | 1 + .../security_solution/to_v8_8_0.test.ts | 129 ++++++++++++++++++ .../migrations/security_solution/to_v8_8_0.ts | 31 +++++ .../common/endpoint/models/policy_config.ts | 5 +- .../models/policy_config_helpers.test.ts | 1 + .../endpoint/models/policy_config_helpers.ts | 45 +++--- .../common/endpoint/types/index.ts | 3 + .../common/license/license.ts | 6 + .../policy/store/policy_details/index.test.ts | 1 + .../endpoint/lib/policy/license_watch.ts | 1 + .../fleet_integration.test.ts | 4 +- .../handlers/create_default_policy.test.ts | 2 + .../handlers/create_default_policy.ts | 3 + 15 files changed, 214 insertions(+), 26 deletions(-) create mode 100644 x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v8_8_0.test.ts create mode 100644 x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v8_8_0.ts diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts index 896cd26df093c..010d101fbe9d7 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts @@ -105,7 +105,7 @@ describe('checking migration metadata changes on all registered SO types', () => "ingest-agent-policies": "e5bb18f8c1d1106139e82fccb93fce01b21fde9b", "ingest-download-sources": "95a15b6589ef46e75aca8f7e534c493f99cc3ccd", "ingest-outputs": "f5adeb3f6abc732a6067137e170578dbf1f58c62", - "ingest-package-policies": "98a5f5defe00d606bfaa64f80bd745ff1465df18", + "ingest-package-policies": "6dc1c9b80a8dc95fbc9c6d9b73dfc56a098eb440", "ingest_manager_settings": "fb75bff08a8de3435b23664b1191f9244a255701", "inventory-view": "6d47ef0b38166ecbd1c2fc7394599a4500db1ae4", "kql-telemetry": "23ed96ff02cd69cbfaa22f313cae3a54c434db51", diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 2b1732142686b..8f0db94d1d31c 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -54,7 +54,10 @@ import { migrateInstallationToV860, migratePackagePolicyToV860, } from './migrations/to_v8_6_0'; -import { migratePackagePolicyToV870 } from './migrations/security_solution'; +import { + migratePackagePolicyToV870, + migratePackagePolicyToV880, +} from './migrations/security_solution'; /* * Saved object types and mappings @@ -212,6 +215,7 @@ const getSavedObjectTypes = (): { [key: string]: SavedObjectsType } => ({ '8.5.0': migratePackagePolicyToV850, '8.6.0': migratePackagePolicyToV860, '8.7.0': migratePackagePolicyToV870, + '8.8.0': migratePackagePolicyToV880, }, }, [PACKAGES_SAVED_OBJECT_TYPE]: { diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/index.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/index.ts index f8c64a09fc4c8..4dd924e832a22 100644 --- a/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/index.ts @@ -17,3 +17,4 @@ export { migratePackagePolicyToV840 } from './to_v8_4_0'; export { migratePackagePolicyToV850 } from './to_v8_5_0'; export { migratePackagePolicyToV860 } from './to_v8_6_0'; export { migratePackagePolicyToV870 } from './to_v8_7_0'; +export { migratePackagePolicyToV880 } from './to_v8_8_0'; diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v8_8_0.test.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v8_8_0.test.ts new file mode 100644 index 0000000000000..e360b655f2a61 --- /dev/null +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v8_8_0.test.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectMigrationContext, SavedObjectUnsanitizedDoc } from '@kbn/core/server'; + +import type { PackagePolicy } from '../../../../common'; + +import { migratePackagePolicyToV880 as migration } from './to_v8_8_0'; + +describe('8.8.0 Endpoint Package Policy migration', () => { + const policyDoc = ({ meta = {} }) => { + return { + id: 'mock-saved-object-id', + attributes: { + name: 'Some Policy Name', + package: { + name: 'endpoint', + title: '', + version: '', + }, + id: 'endpoint', + policy_id: '', + enabled: true, + namespace: '', + revision: 0, + updated_at: '', + updated_by: '', + created_at: '', + created_by: '', + inputs: [ + { + type: 'endpoint', + enabled: true, + streams: [], + config: { + policy: { + value: { + meta: { ...meta }, + windows: {}, + mac: {}, + linux: {}, + }, + }, + }, + }, + ], + }, + type: ' nested', + }; + }; + + it('adds license to policy, defaulted to empty string', () => { + const initialDoc = policyDoc({}); + + const migratedDoc = policyDoc({ + meta: { license: '' }, + }); + + expect(migration(initialDoc, {} as SavedObjectMigrationContext)).toEqual(migratedDoc); + }); + + it('does not modify non-endpoint package policies', () => { + const doc: SavedObjectUnsanitizedDoc = { + id: 'mock-saved-object-id', + attributes: { + name: 'Some Policy Name', + package: { + name: 'notEndpoint', + title: '', + version: '', + }, + id: 'notEndpoint', + policy_id: '', + enabled: true, + namespace: '', + revision: 0, + updated_at: '', + updated_by: '', + created_at: '', + created_by: '', + inputs: [ + { + type: 'notEndpoint', + enabled: true, + streams: [], + config: {}, + }, + ], + }, + type: ' nested', + }; + + expect( + migration(doc, {} as SavedObjectMigrationContext) as SavedObjectUnsanitizedDoc + ).toEqual({ + attributes: { + name: 'Some Policy Name', + package: { + name: 'notEndpoint', + title: '', + version: '', + }, + id: 'notEndpoint', + policy_id: '', + enabled: true, + namespace: '', + revision: 0, + updated_at: '', + updated_by: '', + created_at: '', + created_by: '', + inputs: [ + { + type: 'notEndpoint', + enabled: true, + streams: [], + config: {}, + }, + ], + }, + type: ' nested', + id: 'mock-saved-object-id', + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v8_8_0.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v8_8_0.ts new file mode 100644 index 0000000000000..24b769bcdec9a --- /dev/null +++ b/x-pack/plugins/fleet/server/saved_objects/migrations/security_solution/to_v8_8_0.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SavedObjectMigrationFn, SavedObjectUnsanitizedDoc } from '@kbn/core/server'; + +import type { PackagePolicy } from '../../../../common'; + +export const migratePackagePolicyToV880: SavedObjectMigrationFn = ( + packagePolicyDoc +) => { + if (packagePolicyDoc.attributes.package?.name !== 'endpoint') { + return packagePolicyDoc; + } + + const updatedPackagePolicyDoc: SavedObjectUnsanitizedDoc = packagePolicyDoc; + + const input = updatedPackagePolicyDoc.attributes.inputs[0]; + + if (input && input.config) { + const policy = input.config.policy.value; + + // For the migration, we add an empty string, and the license watcher will correct it if needed. + policy.meta = { license: '' }; + } + + return updatedPackagePolicyDoc; +}; diff --git a/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts b/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts index bf1ac9097a103..b0997a6b92787 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/policy_config.ts @@ -11,8 +11,11 @@ import { ProtectionModes } from '../types'; /** * Return a new default `PolicyConfig` for platinum and above licenses */ -export const policyFactory = (): PolicyConfig => { +export const policyFactory = (license = ''): PolicyConfig => { return { + meta: { + license, + }, windows: { events: { credential_access: true, diff --git a/x-pack/plugins/security_solution/common/endpoint/models/policy_config_helpers.test.ts b/x-pack/plugins/security_solution/common/endpoint/models/policy_config_helpers.test.ts index a5c126f587b53..b35bbc190fe43 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/policy_config_helpers.test.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/policy_config_helpers.test.ts @@ -119,6 +119,7 @@ describe('Policy Config helpers', () => { // This constant makes sure that if the type `PolicyConfig` is ever modified, // the logic for disabling protections is also modified due to type check. export const eventsOnlyPolicy: PolicyConfig = { + meta: { license: '' }, windows: { events: { credential_access: true, diff --git a/x-pack/plugins/security_solution/common/endpoint/models/policy_config_helpers.ts b/x-pack/plugins/security_solution/common/endpoint/models/policy_config_helpers.ts index 27a15e0be5401..4bdca10547bc2 100644 --- a/x-pack/plugins/security_solution/common/endpoint/models/policy_config_helpers.ts +++ b/x-pack/plugins/security_solution/common/endpoint/models/policy_config_helpers.ts @@ -31,29 +31,29 @@ export const disableProtections = (policy: PolicyConfig): PolicyConfig => { }; const disableCommonProtections = (policy: PolicyConfig) => { - let policyOutput = policy; - - for (const key in policyOutput) { - if (Object.prototype.hasOwnProperty.call(policyOutput, key)) { - const os = key as keyof PolicyConfig; - - policyOutput = { - ...policyOutput, - [os]: { - ...policyOutput[os], - ...getDisabledCommonProtectionsForOS(policyOutput, os), - popup: { - ...policyOutput[os].popup, - ...getDisabledCommonPopupsForOS(policyOutput, os), - }, - }, - }; + return Object.keys(policy).reduce((acc, item) => { + const os = item as keyof PolicyConfig; + if (os === 'meta') { + return acc; } - } - return policyOutput; + return { + ...acc, + [os]: { + ...policy[os], + ...getDisabledCommonProtectionsForOS(policy, os), + popup: { + ...policy[os].popup, + ...getDisabledCommonPopupsForOS(policy, os), + }, + }, + }; + }, policy); }; -const getDisabledCommonProtectionsForOS = (policy: PolicyConfig, os: keyof PolicyConfig) => ({ +const getDisabledCommonProtectionsForOS = ( + policy: PolicyConfig, + os: keyof Omit +) => ({ behavior_protection: { ...policy[os].behavior_protection, mode: ProtectionModes.off, @@ -69,7 +69,10 @@ const getDisabledCommonProtectionsForOS = (policy: PolicyConfig, os: keyof Polic }, }); -const getDisabledCommonPopupsForOS = (policy: PolicyConfig, os: keyof PolicyConfig) => ({ +const getDisabledCommonPopupsForOS = ( + policy: PolicyConfig, + os: keyof Omit +) => ({ behavior_protection: { ...policy[os].popup.behavior_protection, enabled: false, diff --git a/x-pack/plugins/security_solution/common/endpoint/types/index.ts b/x-pack/plugins/security_solution/common/endpoint/types/index.ts index 790645ad2e70f..d9d0677ed8f17 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types/index.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types/index.ts @@ -920,6 +920,9 @@ type KbnConfigSchemaNonOptionalProps> = Pi * Endpoint Policy configuration */ export interface PolicyConfig { + meta: { + license: string; + }; windows: { advanced?: { [key: string]: unknown; diff --git a/x-pack/plugins/security_solution/common/license/license.ts b/x-pack/plugins/security_solution/common/license/license.ts index c82fc21d80ceb..0054835d6d779 100644 --- a/x-pack/plugins/security_solution/common/license/license.ts +++ b/x-pack/plugins/security_solution/common/license/license.ts @@ -38,6 +38,12 @@ export class LicenseService { return this.observable; } + public getLicenseType() { + return this.licenseInformation && this.licenseInformation.type + ? this.licenseInformation.type + : ''; + } + public isAtLeast(level: LicenseType): boolean { return isAtLeast(this.licenseInformation, level); } diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts index f96a6c06cb8cf..f4035498f81df 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts @@ -269,6 +269,7 @@ describe('policy details: ', () => { }, policy: { value: { + meta: { license: '' }, windows: { events: { credential_access: true, diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/policy/license_watch.ts b/x-pack/plugins/security_solution/server/endpoint/lib/policy/license_watch.ts index 195c8509e60d5..28a6eb273bcf3 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/policy/license_watch.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/policy/license_watch.ts @@ -101,6 +101,7 @@ export class PolicyWatcher { for (const policy of response.items as PolicyData[]) { const updatePolicy = getPolicyDataForUpdate(policy); const policyConfig = updatePolicy.inputs[0].config.policy.value; + updatePolicy.inputs[0].config.policy.value.meta.license = license.type || ''; try { if (!isEndpointPolicyValidForLicense(policyConfig, license)) { diff --git a/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.test.ts b/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.test.ts index f17c9595a6786..0ffdc09844c9a 100644 --- a/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.test.ts +++ b/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.test.ts @@ -92,13 +92,13 @@ describe('ingest_integration tests ', () => { const soClient = savedObjectsClientMock.create(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - const createNewEndpointPolicyInput = (manifest: ManifestSchema) => ({ + const createNewEndpointPolicyInput = (manifest: ManifestSchema, license = 'platinum') => ({ type: 'endpoint', enabled: true, streams: [], config: { integration_config: {}, - policy: { value: disableProtections(policyFactory()) }, + policy: { value: disableProtections(policyFactory(license)) }, artifact_manifest: { value: manifest }, }, }); diff --git a/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_default_policy.test.ts b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_default_policy.test.ts index a385a37e78b61..c0b713b1bfa17 100644 --- a/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_default_policy.test.ts +++ b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_default_policy.test.ts @@ -169,6 +169,8 @@ describe('Create Default Policy tests ', () => { const config = createEndpointConfig({ preset: 'EDRComplete' }); const policy = createDefaultPolicyCallback(config); const defaultPolicy = policyFactory(); + // update defaultPolicy w/ platinum license + defaultPolicy.meta.license = 'platinum'; expect(policy).toMatchObject(defaultPolicy); }); }); diff --git a/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_default_policy.ts b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_default_policy.ts index 233abc25d0357..17b1972b743d6 100644 --- a/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_default_policy.ts +++ b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_default_policy.ts @@ -29,6 +29,9 @@ export const createDefaultPolicy = ( ): PolicyConfig => { const factoryPolicy = policyConfigFactory(); + // Add license information after policy creation + factoryPolicy.meta.license = licenseService.getLicenseType(); + const defaultPolicyPerType = config?.type === 'cloud' ? getCloudPolicyConfig(factoryPolicy) From b80f4fd74cd3505227384dda123fa13c413fb306 Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Mon, 3 Apr 2023 13:05:33 +0100 Subject: [PATCH 02/10] [Fleet] Show pipelines and mappings editor for input packages (#154077) ## Summary Closes #153105 Show the pipelines and mappings editor for input packages. This can be seen when editing an input package integration policy. Previously this was hidden for input packages and any integration package which allowed datastream to be configured. I have tweaked this logic so that we check if the matching index template for a stream exists, if it does then show the editor. Screenshot 2023-03-30 at 17 02 26 This allows us to handle an edge case (which will become more common as input packages are rolled out) when integration package which has since been upgraded to an input package, for example: - user installs the custom logs integration package and sets dataset to helloworld - the `logs-helloworld` index template is not created as integration packages only create index templates on install (custom logs creates `logs-log.log`) - there is therefore no ingest pipeline `@custom` or `@package`component template to customize, we do not show the pipeline editor. - The user upgrades to an input package version of custom logs (coming in v2.0.0) there still isn't an index template because we do not create them on upgrade, so we still don't want to show the pipelines and mappings editor - Just checking for input package in this scenario would mean we showed the editor incorrectly I have opted to show the editors even if the stream is owned by another package e.g custom logs data is being sent to an nginx stream. ### Automated tests I have added cypress tests for checking that the mappings pipelines editors are shown for input packages. ### Manual test cases Here is version 2.0.0 of logs which is an input package: [log-2.0.0.zip](https://github.com/elastic/kibana/files/11113579/log-2.0.0.zip) it can be added to a deployment using curl: ``` curl -XPOST -H 'content-type: application/zip' -H 'kbn-xsrf: true' http://localhost:5601/mark/api/fleet/epm/packages -u elastic:changeme --data-binary @/path/to/log-2.0.0.zip ``` - Create a policy for an integration package which does not allow dataset to be configured (e.g Apache) - pipelines & editor should be visible when editing package policy - Create a policy for an integration package which does allow dataset to be configured (e.g custom logs 1.6.1) - pipelines & mappings editor should not be shown if dataset doesn't match the one installed by the package `logs.log` - Upgrade custom logs to the input package version (2.0.0) - pipelines and mappings editor still should not be shown --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- src/dev/precommit_hook/casing_check_config.js | 1 + .../cypress/e2e/package_policy_real.cy.ts | 98 ++++++++++++++++++ .../cypress/packages/input_package-1.0.0.zip | Bin 0 -> 2620 bytes x-pack/plugins/fleet/cypress/plugins/index.ts | 47 +++++++++ .../fleet/cypress/screens/integrations.ts | 8 ++ .../components/datastream_hooks.tsx | 32 +++++- .../components/datastream_mappings.tsx | 12 ++- .../components/datastream_pipelines.tsx | 7 +- .../steps/components/dataset_combo.tsx | 1 + .../steps/components/multi_text_input.tsx | 5 +- .../package_policy_input_stream.tsx | 34 ++++-- .../package_policy_input_var_field.tsx | 9 +- 12 files changed, 239 insertions(+), 15 deletions(-) create mode 100644 x-pack/plugins/fleet/cypress/e2e/package_policy_real.cy.ts create mode 100644 x-pack/plugins/fleet/cypress/packages/input_package-1.0.0.zip diff --git a/src/dev/precommit_hook/casing_check_config.js b/src/dev/precommit_hook/casing_check_config.js index 8adbd0a3d672e..5d34a004d8a99 100644 --- a/src/dev/precommit_hook/casing_check_config.js +++ b/src/dev/precommit_hook/casing_check_config.js @@ -29,6 +29,7 @@ export const IGNORE_FILE_GLOBS = [ 'x-pack/plugins/canvas/server/templates/assets/*.{png,jpg,svg}', 'x-pack/plugins/cases/docs/**/*', 'x-pack/plugins/monitoring/public/lib/jquery_flot/**/*', + 'x-pack/plugins/fleet/cypress/packages/*.zip', '**/.*', '**/__mocks__/**/*', 'x-pack/docs/**/*', diff --git a/x-pack/plugins/fleet/cypress/e2e/package_policy_real.cy.ts b/x-pack/plugins/fleet/cypress/e2e/package_policy_real.cy.ts new file mode 100644 index 0000000000000..00d9c4547966f --- /dev/null +++ b/x-pack/plugins/fleet/cypress/e2e/package_policy_real.cy.ts @@ -0,0 +1,98 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + ADD_INTEGRATION_POLICY_BTN, + CREATE_PACKAGE_POLICY_SAVE_BTN, + INTEGRATION_NAME_LINK, + POLICY_EDITOR, +} from '../screens/integrations'; +import { EXISTING_HOSTS_TAB } from '../screens/fleet'; +import { CONFIRM_MODAL } from '../screens/navigation'; + +const TEST_PACKAGE = 'input_package-1.0.0'; +const agentPolicyId = 'test-input-package-policy'; +const agentPolicyName = 'Test input package policy'; +const inputPackagePolicyName = 'input-package-policy'; + +function editPackagePolicyandShowAdvanced() { + cy.visit(`/app/integrations/detail/${TEST_PACKAGE}/policies`); + + cy.getBySel(INTEGRATION_NAME_LINK).contains(inputPackagePolicyName).click(); + + cy.get('button').contains('Change defaults').click(); + cy.get('[data-test-subj^="advancedStreamOptionsToggle"]').click(); +} +describe('Input package create and edit package policy', () => { + before(() => { + cy.task('installTestPackage', TEST_PACKAGE); + + cy.request({ + method: 'POST', + url: `/api/fleet/agent_policies`, + body: { + id: agentPolicyId, + name: agentPolicyName, + description: 'desc', + namespace: 'default', + monitoring_enabled: [], + }, + headers: { 'kbn-xsrf': 'cypress' }, + }); + }); + after(() => { + // delete agent policy + cy.request({ + method: 'POST', + url: `/api/fleet/agent_policies/delete`, + headers: { 'kbn-xsrf': 'cypress' }, + body: JSON.stringify({ + agentPolicyId, + }), + }); + cy.task('uninstallTestPackage', TEST_PACKAGE); + }); + it('should successfully create a package policy', () => { + cy.visit(`/app/integrations/detail/${TEST_PACKAGE}/overview`); + cy.getBySel(ADD_INTEGRATION_POLICY_BTN).click(); + + cy.getBySel(POLICY_EDITOR.POLICY_NAME_INPUT).click().clear().type(inputPackagePolicyName); + cy.getBySel('multiTextInput-paths') + .find('[data-test-subj="multiTextInputRow-0"]') + .click() + .type('/var/log/test.log'); + + cy.getBySel('multiTextInput-tags') + .find('[data-test-subj="multiTextInputRow-0"]') + .click() + .type('tag1'); + + cy.getBySel(POLICY_EDITOR.DATASET_SELECT).click().type('testdataset'); + + cy.getBySel(EXISTING_HOSTS_TAB).click(); + + cy.getBySel(POLICY_EDITOR.AGENT_POLICY_SELECT).click().get(`#${agentPolicyId}`).click(); + cy.wait(500); // wait for policy id to be set + cy.getBySel(CREATE_PACKAGE_POLICY_SAVE_BTN).click(); + + cy.getBySel(CONFIRM_MODAL.CANCEL_BUTTON).click(); + }); + + it('should show pipelines editor with link to pipeline', () => { + editPackagePolicyandShowAdvanced(); + cy.getBySel(POLICY_EDITOR.INSPECT_PIPELINES_BTN).click(); + cy.getBySel(CONFIRM_MODAL.CONFIRM_BUTTON).click(); + cy.get('body').should('not.contain', 'Pipeline not found'); + cy.get('body').should('contain', '"managed_by": "fleet"'); + }); + it('should show mappings editor with link to create custom template', () => { + editPackagePolicyandShowAdvanced(); + cy.getBySel(POLICY_EDITOR.EDIT_MAPPINGS_BTN).click(); + cy.getBySel(CONFIRM_MODAL.CONFIRM_BUTTON).click(); + cy.get('body').should('contain', 'logs-testdataset@custom'); + }); +}); diff --git a/x-pack/plugins/fleet/cypress/packages/input_package-1.0.0.zip b/x-pack/plugins/fleet/cypress/packages/input_package-1.0.0.zip new file mode 100644 index 0000000000000000000000000000000000000000..639072cab5ea7f6a49be7819e955f007be4eb433 GIT binary patch literal 2620 zcmWIWW@h1H0D*wbbzxuzln`N%VaUuYC@qOENKDR7Oi$G{)HBdC&<_paWMJ<2r<&3U z#HAJ742&#a85tN@L_lT$O#{&!VAEon8FgBKysbd2h{LqxjKsY3)SUcuy~^AixY1ph zMw|9H@--Xqu)N=sSan@5dkb= zS(pDVbgzEjUY8-=uuNg%Re8Y|AJUeZhFY&R%urHUIeQ`7EaxY|%x|J^Hb0dR*4HWf zI49-Xj5DjMo+oX}l&;q5ek^lEXy@x?mruFhEGj#HeDzO8koT|$mJ|-}0fQ~CgkX?i zH&X$xnc%1)Xfh%)^gK@;F9t@$Z6MYq!ep@dph(fnNGiq`F`+#NK@sEpzQ%RQ+{Wtq zv@W*=6U{=_?ri;My>ZLhHzL0+%MwqU^1d^F=)R2U_ALjKNxM^93+D^A9h$!`Wl!Vd zFIqB%OO;t(UUaYceVb!x_I<6WCo6Ze{5&PHmiwQmnAyJYYVQ3(J%_^#YvL!pY>u|Z z6AzL&e3p`*Tud;q5Yg~EXz@K>U_gPe5rlf-lfdPTf^eKK{9s3+*o!M+1%4BZ@zYB`_TD{;aruG3hbYjT~Xe3Bdfyrh2{B*4Dp6HZD%8U=PlV4o>IC@bTQ{eB~hVo3|0$M za|P}P7@4gVJ{~jY*`$m~$_J(t+4%{dJFMHVvc&Y7l|}okM2^@wpU*vha&rEO;Mc+3 ztgdoXURQ3b$%u)2zx+qs(H6@(?pH6CMn+di*8kA)J#$+-{<7t^q#oIMTRQH~?XB1` zaV6jJP0Bstzl6@8x-)s-?k#`y_XPFjU0(Z9F-h_c<6~Ko*(vOAQkyw0={|`1X|JJo z`Plqu%|^WkPj{X>bdkINO3#|q?fMMM&(>^-6VqK)$ofJYcjlMIkwVfkQ*%-XCp$zQ z4fyAID;${Ql7LtRzoGE#i7(-a_8jDGFyJ}z&!>?407GmuV}Jr<=oZhbYy!7~>(p=W z67-5>S~TtS@_PP;{Yw|0-D0^?fBByN(05Hsb4|timoXYYoy1Z-RZz}&*H zq|pzD$!PTnK5MXhKnleJxq?flWx8OiusLr%?H~IEVl63f(R_s5*|6QZWCb( zJV8Oqdwf=64>(B_k8Neb9&pftm^kB6i(zCRaRAFpZ1Dm%9;r~qXAdIeP)lKCdkS&b zgIXrzvjux#Nuvao7Atn&zzYz379hNVT6`dTqaMEn@VLjF)L7X-d6^3cKLLZp1k3{f DAZZ3i literal 0 HcmV?d00001 diff --git a/x-pack/plugins/fleet/cypress/plugins/index.ts b/x-pack/plugins/fleet/cypress/plugins/index.ts index 17825ba12a2bb..ee01dd20c470c 100644 --- a/x-pack/plugins/fleet/cypress/plugins/index.ts +++ b/x-pack/plugins/fleet/cypress/plugins/index.ts @@ -5,12 +5,42 @@ * 2.0. */ +import { promisify } from 'util'; + +import fs from 'fs'; + +import fetch from 'node-fetch'; import { createEsClientForTesting } from '@kbn/test'; const plugin: Cypress.PluginConfig = (on, config) => { const client = createEsClientForTesting({ esUrl: config.env.ELASTICSEARCH_URL, }); + + async function kibanaFetch(opts: { + method: string; + path: string; + body?: any; + contentType?: string; + }) { + const { method, path, body, contentType } = opts; + const Authorization = `Basic ${Buffer.from( + `elastic:${config.env.ELASTICSEARCH_PASSWORD}` + ).toString('base64')}`; + + const url = `${config.env.KIBANA_URL}${path}`; + const res = await fetch(url, { + method, + headers: { + 'kbn-xsrf': 'cypress', + 'Content-Type': contentType || 'application/json', + Authorization, + }, + ...(body ? { body } : {}), + }); + + return res.json(); + } on('task', { async insertDoc({ index, doc, id }: { index: string; doc: any; id: string }) { return client.create({ id, document: doc, index, refresh: 'wait_for' }); @@ -37,6 +67,23 @@ const plugin: Cypress.PluginConfig = (on, config) => { conflicts: 'proceed', }); }, + async installTestPackage(packageName: string) { + const zipPath = require.resolve('../packages/' + packageName + '.zip'); + const zipContent = await promisify(fs.readFile)(zipPath, 'base64'); + return kibanaFetch({ + method: 'POST', + path: '/api/fleet/epm/packages', + body: Buffer.from(zipContent, 'base64'), + contentType: 'application/zip', + }); + }, + + async uninstallTestPackage(packageName: string) { + return kibanaFetch({ + method: 'DELETE', + path: `/api/fleet/epm/packages/${packageName}`, + }); + }, }); }; diff --git a/x-pack/plugins/fleet/cypress/screens/integrations.ts b/x-pack/plugins/fleet/cypress/screens/integrations.ts index 63b13d8ff7fd3..2faa0ade447a6 100644 --- a/x-pack/plugins/fleet/cypress/screens/integrations.ts +++ b/x-pack/plugins/fleet/cypress/screens/integrations.ts @@ -35,6 +35,14 @@ export const SETTINGS = { UNINSTALL_ASSETS_BTN: 'uninstallAssetsButton', }; +export const POLICY_EDITOR = { + POLICY_NAME_INPUT: 'packagePolicyNameInput', + DATASET_SELECT: 'datasetComboBox', + AGENT_POLICY_SELECT: 'agentPolicySelect', + INSPECT_PIPELINES_BTN: 'datastreamInspectPipelineBtn', + EDIT_MAPPINGS_BTN: 'datastreamEditMappingsBtn', +}; + export const INTEGRATION_POLICIES_UPGRADE_CHECKBOX = 'epmDetails.upgradePoliciesCheckbox'; export const getIntegrationCard = (integration: string) => `integration-card:epr:${integration}`; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_hooks.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_hooks.tsx index d8e21fce7c306..de740b99d97ce 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_hooks.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_hooks.tsx @@ -6,8 +6,9 @@ */ import { useRouteMatch } from 'react-router-dom'; +import { useQuery } from '@tanstack/react-query'; -import { useLink } from '../../../../hooks'; +import { sendRequestForRq, useLink } from '../../../../hooks'; export function usePackagePolicyEditorPageUrl(dataStreamId?: string) { const { @@ -27,3 +28,32 @@ export function usePackagePolicyEditorPageUrl(dataStreamId?: string) { return `${baseUrl}${dataStreamId ? `?datastreamId=${encodeURIComponent(dataStreamId)}` : ''}`; } + +export function useIndexTemplateExists( + templateName: string, + enabled: boolean +): { + exists?: boolean; + isLoading: boolean; +} { + const { data, isLoading } = useQuery( + ['indexTemplateExists', templateName], + () => + sendRequestForRq({ + path: `/api/index_management/index_templates/${templateName}`, + method: 'get', + }), + { enabled: enabled || !!templateName } + ); + + if (isLoading) { + return { + isLoading: true, + }; + } + + return { + exists: !!data, + isLoading: false, + }; +} diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_mappings.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_mappings.tsx index 37963d4abe1ac..86d1f3404d4f9 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_mappings.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_mappings.tsx @@ -18,7 +18,11 @@ import { usePackagePolicyEditorPageUrl } from './datastream_hooks'; export interface PackagePolicyEditorDatastreamMappingsProps { packageInfo: PackageInfo; - packageInputStream: { id?: string; data_stream: { dataset: string; type: string } }; + packageInputStream: { + id?: string; + data_stream: { dataset: string; type: string }; + }; + customDataset?: string; } function useComponentTemplates(dataStream: { dataset: string; type: string }) { @@ -35,8 +39,10 @@ function useComponentTemplates(dataStream: { dataset: string; type: string }) { export const PackagePolicyEditorDatastreamMappings: React.FunctionComponent< PackagePolicyEditorDatastreamMappingsProps -> = ({ packageInputStream, packageInfo }) => { - const dataStream = packageInputStream.data_stream; +> = ({ packageInputStream, packageInfo, customDataset }) => { + const dataStream = customDataset + ? { ...packageInputStream.data_stream, dataset: customDataset } + : packageInputStream.data_stream; const pageUrl = usePackagePolicyEditorPageUrl(packageInputStream.id); const { application, docLinks } = useStartServices(); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_pipelines.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_pipelines.tsx index 75e95a7975f61..b2325dcc15213 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_pipelines.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/datastream_pipelines.tsx @@ -31,6 +31,7 @@ import { usePackagePolicyEditorPageUrl } from './datastream_hooks'; export interface PackagePolicyEditorDatastreamPipelinesProps { packageInfo: PackageInfo; packageInputStream: { id?: string; data_stream: { dataset: string; type: string } }; + customDataset?: string; } interface PipelineItem { @@ -92,8 +93,10 @@ function useDatastreamIngestPipelines( export const PackagePolicyEditorDatastreamPipelines: React.FunctionComponent< PackagePolicyEditorDatastreamPipelinesProps -> = ({ packageInputStream, packageInfo }) => { - const dataStream = packageInputStream.data_stream; +> = ({ packageInputStream, packageInfo, customDataset }) => { + const dataStream = customDataset + ? { ...packageInputStream.data_stream, dataset: customDataset } + : packageInputStream.data_stream; const { application, share, docLinks } = useStartServices(); const ingestPipelineLocator = share.url.locators.get('INGEST_PIPELINES_APP_LOCATOR'); diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/dataset_combo.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/dataset_combo.tsx index 680da0fd75012..31fcbee8d6821 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/dataset_combo.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/dataset_combo.tsx @@ -94,6 +94,7 @@ export const DatasetComboBox: React.FC<{ })} isClearable={false} isDisabled={isDisabled} + data-test-subj="datasetComboBox" /> {valueAsOption && valueAsOption.value.package !== pkgName && ( <> diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/multi_text_input.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/multi_text_input.tsx index 66ec097624c41..ebcf666d24710 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/multi_text_input.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/multi_text_input.tsx @@ -25,6 +25,7 @@ interface Props { errors?: Array<{ message: string; index?: number }>; isInvalid?: boolean; isDisabled?: boolean; + 'data-test-subj'?: string; } interface RowProps { @@ -69,6 +70,7 @@ const Row: FunctionComponent = ({ autoFocus={autoFocus} disabled={isDisabled} onBlur={onBlur} + data-test-subj={`multiTextInputRow-${index}`} /> {showDeleteButton && ( @@ -99,6 +101,7 @@ export const MultiTextInput: FunctionComponent = ({ isInvalid, isDisabled, errors, + 'data-test-subj': dataTestSubj, }) => { const [autoFocus, setAutoFocus] = useState(false); const [rows, setRows] = useState(() => defaultValue(value)); @@ -139,7 +142,7 @@ export const MultiTextInput: FunctionComponent = ({ return ( <> - + {rows.map((row, idx) => ( ( !!packagePolicyInputStream.id && packagePolicyInputStream.id === defaultDataStreamId; const isPackagePolicyEdit = !!packagePolicyId; - const isInputOnlyPackage = packageInfo.type === 'input'; - const hasDatasetVar = packageInputStream.vars?.some( - (varDef) => varDef.name === DATASET_VAR_NAME - ); - const showPipelinesAndMappings = !isInputOnlyPackage && !hasDatasetVar; + const customDatasetVar = packagePolicyInputStream.vars?.[DATASET_VAR_NAME]; + const customDatasetVarValue = customDatasetVar?.value?.dataset || customDatasetVar?.value; + + const { exists: indexTemplateExists, isLoading: isLoadingIndexTemplate } = + useIndexTemplateExists( + getRegistryDataStreamAssetBaseName({ + dataset: customDatasetVarValue, + type: packageInputStream.data_stream.type, + }), + isPackagePolicyEdit + ); + + // only show pipelines and mappings if the matching index template exists + // in the legacy case (e.g logs package pre 2.0.0) the index template will not exist + // because we allowed dataset to be customized but didnt create a matching index template + // for the new dataset. + const showPipelinesAndMappings = !isLoadingIndexTemplate && indexTemplateExists; + useEffect(() => { if (isDefaultDatastream && containerRef.current) { containerRef.current.scrollIntoView(); @@ -249,6 +267,7 @@ export const PackagePolicyInputStreamConfig = memo( iconType={isShowingAdvanced ? 'arrowDown' : 'arrowRight'} onClick={() => setIsShowingAdvanced(!isShowingAdvanced)} flush="left" + data-test-subj={`advancedStreamOptionsToggle-${packagePolicyInputStream.id}`} > ( ); })} - {/* Only show datastream pipelines and mappings on edit and not for input packages*/} {isPackagePolicyEdit && showPipelinesAndMappings && ( <> diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_var_field.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_var_field.tsx index 27457dedbc9b1..827d1fb2722c2 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_var_field.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agent_policy/create_package_policy_page/components/steps/components/package_policy_input_var_field.tsx @@ -63,7 +63,7 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{ const isInvalid = (isDirty || forceShowErrors) && !!varErrors; const errors = isInvalid ? varErrors : null; const fieldLabel = title || name; - + const fieldTestSelector = fieldLabel.replace(/\s/g, '-').toLowerCase(); const field = useMemo(() => { if (multi) { return ( @@ -72,6 +72,7 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{ onChange={onChange} onBlur={() => setIsDirty(true)} isDisabled={frozen} + data-test-subj={`multiTextInput-${fieldTestSelector}`} /> ); } @@ -96,6 +97,7 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{ onBlur={() => setIsDirty(true)} disabled={frozen} resize="vertical" + data-test-subj={`textAreaInput-${fieldTestSelector}`} /> ); case 'yaml': @@ -142,6 +144,7 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{ onChange={(e) => onChange(e.target.checked)} onBlur={() => setIsDirty(true)} disabled={frozen} + data-test-subj={`switch-${fieldTestSelector}`} /> ); case 'password': @@ -153,6 +156,7 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{ onChange={(e) => onChange(e.target.value)} onBlur={() => setIsDirty(true)} disabled={frozen} + data-test-subj={`passwordInput-${fieldTestSelector}`} /> ); case 'select': @@ -177,6 +181,7 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{ return onChange(newValue); }} onBlur={() => setIsDirty(true)} + data-test-subj={`select-${fieldTestSelector}`} /> ); default: @@ -187,6 +192,7 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{ onChange={(e) => onChange(e.target.value)} onBlur={() => setIsDirty(true)} disabled={frozen} + data-test-subj={`textInput-${fieldTestSelector}`} /> ); } @@ -204,6 +210,7 @@ export const PackagePolicyInputVarField: React.FunctionComponent<{ isInvalid, fieldLabel, options, + fieldTestSelector, ]); // Boolean cannot be optional by default set to false From ab30f3f12360f840ab36875115abee422346dea0 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 3 Apr 2023 08:28:15 -0400 Subject: [PATCH 03/10] [Fleet] Implement audit logging for core CRUD operations (#152118) ## Summary Adds custom [audit logging ](https://www.elastic.co/guide/en/kibana/current/xpack-security-audit-logging.html) for the majority of Fleet's persistence operations. ### Operations with custom audit logging - Agent Policy CRUD - Package Policy CRUD - EPM (packages) CRUD - Fleet Settings CRUD - Outputs CRUD - `.fleet-actions` reads/writes - `.fleet-policies` reads/writes - `.fleet-enrollment-api-keys` read/writes - `.fleet-agents` reads/writes ## Background Fleet uses an "internal" saved objects client and commonly performs persistence operations as `kibana_system` while using its own security/role implementation around Fleet/Integrations permissions. This means we can't simply enable Kibana's default audit logger and receive meaningful logs for Fleet usage. Instead, we need to implement our own custom audit logging around these operations. ## Implementation This PR adds audit logging helpers to our `appContextService` class, meaning they can be used anywhere throughout the Fleet codebase. In HTTP request contexts, the audit logger on the `AppContext` object will include request-level values. In non-HTTP contexts (e.g. Fleet setup running during Kibana boot), the logs will be "un-scoped" and won't include request info. For saved objects specifically, there are a few expectations around how audit logs are formatted. To capture this, I've added a `writeCustomSoAuditLog` helper that includes the expected field structure and message format for consistency across SO operations. ## Testing Add the following to your `kibana.dev.yml` ```yaml xpack.security.audit.enabled: true ``` Run Kibana and perform various Fleet operations. Note the `logs/audit.log` file that's created in your local Kibana directory. As you perform actions, grep the `audit.log` for your policy ID's, request paths, etc and note the data. You can also upload your `audit.log` file via Kibana's file upload and explore the data that way. ![image](https://user-images.githubusercontent.com/6766512/227549012-4c03659a-9692-458d-a791-7124df1ffcdf.png) --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- x-pack/plugins/fleet/server/plugin.ts | 69 +- .../server/routes/agent_policy/handlers.ts | 6 +- .../agent_policies/output_helpers.test.ts | 6 + .../server/services/agent_policy.test.ts | 311 ++++++-- .../fleet/server/services/agent_policy.ts | 67 +- .../server/services/agent_policy_create.ts | 1 + .../server/services/agents/actions.test.ts | 124 +++- .../fleet/server/services/agents/actions.ts | 78 +- .../fleet/server/services/agents/crud.test.ts | 56 +- .../fleet/server/services/agents/crud.ts | 15 + .../server/services/agents/reassign.test.ts | 214 +++--- .../agents/request_diagnostics.test.ts | 111 +-- .../server/services/agents/unenroll.test.ts | 680 +++++++++--------- .../services/agents/update_agent_tags.test.ts | 1 + .../api_keys/enrollment_api_key.test.ts | 102 +++ .../services/api_keys/enrollment_api_key.ts | 10 + .../fleet/server/services/app_context.ts | 4 + .../fleet/server/services/audit_logging.ts | 77 ++ .../server/services/download_source.test.ts | 6 + .../template/default_settings.test.ts | 6 + .../services/epm/packages/_install_package.ts | 8 + .../server/services/epm/packages/get.test.ts | 66 +- .../fleet/server/services/epm/packages/get.ts | 75 +- .../services/epm/packages/install.test.ts | 67 +- .../server/services/epm/packages/install.ts | 50 ++ .../services/epm/packages/remove.test.ts | 21 + .../server/services/epm/packages/remove.ts | 23 + .../services/epm/packages/update.test.ts | 79 ++ .../server/services/epm/packages/update.ts | 14 + x-pack/plugins/fleet/server/services/index.ts | 1 + .../services/managed_package_policies.test.ts | 1 + .../fleet/server/services/output.test.ts | 70 ++ .../plugins/fleet/server/services/output.ts | 76 +- .../experimental_datastream_features.test.ts | 7 + .../server/services/package_policy.test.ts | 402 +++++++++-- .../fleet/server/services/package_policy.ts | 97 ++- .../server/services/preconfiguration.test.ts | 2 + .../fleet_server_host.test.ts | 5 + .../fleet/server/services/request_store.ts | 16 + .../fleet/server/services/settings.test.ts | 157 +++- .../plugins/fleet/server/services/settings.ts | 19 + 41 files changed, 2524 insertions(+), 676 deletions(-) create mode 100644 x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.test.ts create mode 100644 x-pack/plugins/fleet/server/services/audit_logging.ts create mode 100644 x-pack/plugins/fleet/server/services/epm/packages/update.test.ts create mode 100644 x-pack/plugins/fleet/server/services/request_store.ts diff --git a/x-pack/plugins/fleet/server/plugin.ts b/x-pack/plugins/fleet/server/plugin.ts index 4e9a7f19b1818..269e220e58da7 100644 --- a/x-pack/plugins/fleet/server/plugin.ts +++ b/x-pack/plugins/fleet/server/plugin.ts @@ -35,7 +35,11 @@ import type { EncryptedSavedObjectsPluginStart, EncryptedSavedObjectsPluginSetup, } from '@kbn/encrypted-saved-objects-plugin/server'; -import type { SecurityPluginSetup, SecurityPluginStart } from '@kbn/security-plugin/server'; +import type { + AuditLogger, + SecurityPluginSetup, + SecurityPluginStart, +} from '@kbn/security-plugin/server'; import type { PluginSetupContract as FeaturesPluginSetup } from '@kbn/features-plugin/server'; import type { TaskManagerSetupContract, @@ -111,6 +115,7 @@ import type { PackagePolicyService } from './services/package_policy_service'; import { PackagePolicyServiceImpl } from './services/package_policy'; import { registerFleetUsageLogger, startFleetUsageLogger } from './services/fleet_usage_logger'; import { CheckDeletedFilesTask } from './tasks/check_deleted_files_task'; +import { getRequestStore } from './services/request_store'; export interface FleetSetupDeps { security: SecurityPluginSetup; @@ -154,6 +159,7 @@ export interface FleetAppContext { telemetryEventsSender: TelemetryEventsSender; bulkActionsResolver: BulkActionsResolver; messageSigningService: MessageSigningServiceInterface; + auditLogger?: AuditLogger; } export type FleetSetupContract = void; @@ -362,39 +368,42 @@ export class FleetPlugin .getSavedObjects() .getScopedClient(request, { excludedExtensions: [SECURITY_EXTENSION_ID] }); - return { - get agentClient() { - const agentService = plugin.setupAgentService(esClient.asInternalUser, soClient); + const requestStore = getRequestStore(); - return { - asCurrentUser: agentService.asScoped(request), - asInternalUser: agentService.asInternalUser, - }; - }, - get packagePolicyService() { - const service = plugin.setupPackagePolicyService(); + return requestStore.run(request, () => { + return { + get agentClient() { + const agentService = plugin.setupAgentService(esClient.asInternalUser, soClient); - return { - asCurrentUser: service.asScoped(request), - asInternalUser: service.asInternalUser, - }; - }, - authz, + return { + asCurrentUser: agentService.asScoped(request), + asInternalUser: agentService.asInternalUser, + }; + }, + get packagePolicyService() { + const service = plugin.setupPackagePolicyService(); - get internalSoClient() { - // Use a lazy getter to avoid constructing this client when not used by a request handler - return getInternalSoClient(); - }, - get spaceId() { - return deps.spaces?.spacesService?.getSpaceId(request) ?? DEFAULT_SPACE_ID; - }, + return { + asCurrentUser: service.asScoped(request), + asInternalUser: service.asInternalUser, + }; + }, + authz, + get internalSoClient() { + // Use a lazy getter to avoid constructing this client when not used by a request handler + return getInternalSoClient(); + }, + get spaceId() { + return deps.spaces?.spacesService?.getSpaceId(request) ?? DEFAULT_SPACE_ID; + }, - get limitedToPackages() { - if (routeAuthz && routeAuthz.granted) { - return routeAuthz.scopeDataToPackages; - } - }, - }; + get limitedToPackages() { + if (routeAuthz && routeAuthz.granted) { + return routeAuthz.scopeDataToPackages; + } + }, + }; + }); } ); diff --git a/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts b/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts index 83e2600d1beb2..a31ba8fa5cb70 100644 --- a/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts +++ b/x-pack/plugins/fleet/server/routes/agent_policy/handlers.ts @@ -267,11 +267,15 @@ export const deleteAgentPoliciesHandler: RequestHandler< const coreContext = await context.core; const soClient = coreContext.savedObjects.client; const esClient = coreContext.elasticsearch.client.asInternalUser; + const user = await appContextService.getSecurity()?.authc.getCurrentUser(request); try { const body: DeleteAgentPolicyResponse = await agentPolicyService.delete( soClient, esClient, - request.body.agentPolicyId + request.body.agentPolicyId, + { + user: user || undefined, + } ); return response.ok({ body, diff --git a/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts b/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts index 629640c574b09..29d48d5d595ef 100644 --- a/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policies/output_helpers.test.ts @@ -7,6 +7,8 @@ import { savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { securityMock } from '@kbn/security-plugin/server/mocks'; + import { appContextService } from '..'; import { outputService } from '../output'; @@ -16,6 +18,10 @@ jest.mock('../app_context'); jest.mock('../output'); const mockedAppContextService = appContextService as jest.Mocked; +mockedAppContextService.getSecuritySetup.mockImplementation(() => ({ + ...securityMock.createSetup(), +})); + const mockedOutputService = outputService as jest.Mocked; function mockHasLicence(res: boolean) { diff --git a/x-pack/plugins/fleet/server/services/agent_policy.test.ts b/x-pack/plugins/fleet/server/services/agent_policy.test.ts index 1d2241696704d..9646b6e774223 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.test.ts @@ -6,18 +6,16 @@ */ import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; - import { SavedObjectsErrorHelpers } from '@kbn/core/server'; +import { securityMock } from '@kbn/security-plugin/server/mocks'; import { PackagePolicyRestrictionRelatedError } from '../errors'; - import type { AgentPolicy, FullAgentPolicy, NewAgentPolicy, PreconfiguredAgentPolicy, } from '../types'; - import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../constants'; import { AGENT_POLICY_INDEX } from '../../common'; @@ -31,6 +29,8 @@ import { appContextService } from './app_context'; import { outputService } from './output'; import { downloadSourceService } from './download_source'; import { getFullAgentPolicy } from './agent_policies'; +import * as outputsHelpers from './agent_policies/outputs_helpers'; +import { auditLoggingService } from './audit_logging'; function getSavedObjectMock(agentPolicyAttributes: any) { const mock = savedObjectsClientMock.create(); @@ -70,9 +70,17 @@ jest.mock('./agent_policy_update'); jest.mock('./agents'); jest.mock('./package_policy'); jest.mock('./app_context'); +jest.mock('./audit_logging'); jest.mock('./agent_policies/full_agent_policy'); +jest.mock('./agent_policies/outputs_helpers'); const mockedAppContextService = appContextService as jest.Mocked; +mockedAppContextService.getSecuritySetup.mockImplementation(() => ({ + ...securityMock.createSetup(), +})); + +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; +const mockOutputsHelpers = outputsHelpers as jest.Mocked; const mockedOutputService = outputService as jest.Mocked; const mockedDownloadSourceService = downloadSourceService as jest.Mocked< typeof downloadSourceService @@ -83,12 +91,6 @@ const mockedGetFullAgentPolicy = getFullAgentPolicy as jest.Mock< ReturnType >; -function getAgentPolicyUpdateMock() { - return agentPolicyUpdateEventHandler as unknown as jest.Mock< - typeof agentPolicyUpdateEventHandler - >; -} - function getAgentPolicyCreateMock() { const soClient = savedObjectsClientMock.create(); soClient.create.mockImplementation(async (type, attributes) => { @@ -103,8 +105,8 @@ function getAgentPolicyCreateMock() { } describe('agent policy', () => { - beforeEach(() => { - getAgentPolicyUpdateMock().mockClear(); + afterEach(() => { + jest.resetAllMocks(); }); describe('create', () => { @@ -141,6 +143,148 @@ describe('agent policy', () => { const [, attributes] = soClient.create.mock.calls[0]; expect(attributes).toHaveProperty('is_managed', true); }); + + it('should call audit logger', async () => { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + const soClient = savedObjectsClientMock.create(); + + soClient.find.mockResolvedValueOnce({ + total: 0, + saved_objects: [], + per_page: 0, + page: 1, + }); + + soClient.create.mockResolvedValueOnce({ + id: 'test-agent-policy', + type: AGENT_POLICY_SAVED_OBJECT_TYPE, + attributes: {}, + references: [], + }); + + mockOutputsHelpers.validateOutputForPolicy.mockResolvedValueOnce(undefined); + + await agentPolicyService.create( + soClient, + esClient, + { + name: 'test', + namespace: 'default', + }, + { id: 'test-agent-policy' } + ); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'create', + id: 'test-agent-policy', + savedObjectType: AGENT_POLICY_SAVED_OBJECT_TYPE, + }); + }); + }); + + // TODO: Add more test coverage to `get` service method + describe('get', () => { + it('should call audit logger', async () => { + const soClient = savedObjectsClientMock.create(); + + soClient.get.mockResolvedValueOnce({ + id: 'test-agent-policy', + attributes: {}, + references: [], + type: AGENT_POLICY_SAVED_OBJECT_TYPE, + }); + + await agentPolicyService.get(soClient, 'test-agent-policy', false); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toBeCalledWith({ + action: 'get', + id: 'test-agent-policy', + savedObjectType: AGENT_POLICY_SAVED_OBJECT_TYPE, + }); + }); + }); + + describe('getByIDs', () => { + it('should call audit logger', async () => { + const soClient = savedObjectsClientMock.create(); + + soClient.bulkGet.mockResolvedValueOnce({ + saved_objects: [ + { + id: 'test-agent-policy-1', + attributes: {}, + references: [], + type: AGENT_POLICY_SAVED_OBJECT_TYPE, + }, + { + id: 'test-agent-policy-2', + attributes: {}, + references: [], + type: AGENT_POLICY_SAVED_OBJECT_TYPE, + }, + ], + }); + + await agentPolicyService.getByIDs(soClient, ['test-agent-policy-1', 'test-agent-policy-2']); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenNthCalledWith(1, { + action: 'get', + id: 'test-agent-policy-1', + savedObjectType: AGENT_POLICY_SAVED_OBJECT_TYPE, + }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenNthCalledWith(2, { + action: 'get', + id: 'test-agent-policy-2', + savedObjectType: AGENT_POLICY_SAVED_OBJECT_TYPE, + }); + }); + }); + + describe('list', () => { + it('should call audit logger', async () => { + const soClient = savedObjectsClientMock.create(); + + soClient.find.mockResolvedValueOnce({ + total: 2, + saved_objects: [ + { + id: 'test-agent-policy-1', + attributes: {}, + references: [], + type: AGENT_POLICY_SAVED_OBJECT_TYPE, + score: 0, + }, + { + id: 'test-agent-policy-2', + attributes: {}, + references: [], + type: AGENT_POLICY_SAVED_OBJECT_TYPE, + score: 0, + }, + ], + per_page: 0, + page: 1, + }); + + await agentPolicyService.list(soClient, { + page: 1, + perPage: 10, + kuery: '', + }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenNthCalledWith(1, { + action: 'find', + id: 'test-agent-policy-1', + savedObjectType: AGENT_POLICY_SAVED_OBJECT_TYPE, + }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenNthCalledWith(2, { + action: 'find', + id: 'test-agent-policy-2', + savedObjectType: AGENT_POLICY_SAVED_OBJECT_TYPE, + }); + }); }); describe('delete', () => { @@ -187,6 +331,16 @@ describe('agent policy', () => { ); } }); + + it('should call audit logger', async () => { + await agentPolicyService.delete(soClient, esClient, 'mocked'); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'delete', + id: 'mocked', + savedObjectType: AGENT_POLICY_SAVED_OBJECT_TYPE, + }); + }); }); describe('bumpRevision', () => { @@ -406,6 +560,30 @@ describe('agent policy', () => { calledWith = soClient.update.mock.calls[1]; expect(calledWith[2]).toHaveProperty('is_managed', true); }); + + it('should call audit logger', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + soClient.get.mockResolvedValue({ + attributes: {}, + references: [], + id: 'test-agent-policy', + type: AGENT_POLICY_SAVED_OBJECT_TYPE, + }); + + await agentPolicyService.update(soClient, esClient, 'test-agent-policy', { + name: 'Test Agent Policy', + namespace: 'default', + is_managed: false, + }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'update', + id: 'test-agent-policy', + savedObjectType: AGENT_POLICY_SAVED_OBJECT_TYPE, + }); + }); }); describe('deployPolicy', () => { @@ -528,50 +706,75 @@ describe('agent policy', () => { ); }); - describe('ensurePreconfiguredAgentPolicy', () => { - it('should use preconfigured id if provided for policy', async () => { - const soClient = savedObjectsClientMock.create(); - const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; - - const preconfiguredAgentPolicy: PreconfiguredAgentPolicy = { - id: 'my-unique-id', - name: 'My Preconfigured Policy', - package_policies: [ - { - name: 'my-package-policy', - id: 'my-package-policy-id', - package: { - name: 'test-package', - }, - }, - ], - }; + it('should call audit logger', async () => { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + const soClient = savedObjectsClientMock.create(); - soClient.find.mockResolvedValueOnce({ total: 0, saved_objects: [], page: 1, per_page: 10 }); - soClient.get.mockRejectedValueOnce(SavedObjectsErrorHelpers.createGenericNotFoundError()); + mockedAppContextService.getInternalUserESClient.mockReturnValue(esClient); + mockedOutputService.getDefaultDataOutputId.mockResolvedValueOnce('default-output'); - soClient.create.mockResolvedValueOnce({ - id: 'my-unique-id', - type: AGENT_POLICY_SAVED_OBJECT_TYPE, - attributes: {}, - references: [], - }); + soClient.bulkGet.mockResolvedValue({ + saved_objects: [ + { + attributes: {}, + references: [], + id: 'test-agent-policy', + type: AGENT_POLICY_SAVED_OBJECT_TYPE, + }, + ], + }); - await agentPolicyService.ensurePreconfiguredAgentPolicy( - soClient, - esClient, - preconfiguredAgentPolicy - ); + await agentPolicyService.deployPolicy(soClient, 'test-agent-policy'); - expect(soClient.create).toHaveBeenCalledWith( - AGENT_POLICY_SAVED_OBJECT_TYPE, - expect.anything(), - expect.objectContaining({ id: 'my-unique-id' }) - ); + expect(mockedAuditLoggingService.writeCustomAuditLog).toHaveBeenCalledWith({ + message: `User deploying policy [id=test-agent-policy]`, }); }); }); + describe('ensurePreconfiguredAgentPolicy', () => { + it('should use preconfigured id if provided for policy', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + const preconfiguredAgentPolicy: PreconfiguredAgentPolicy = { + id: 'my-unique-id', + name: 'My Preconfigured Policy', + package_policies: [ + { + name: 'my-package-policy', + id: 'my-package-policy-id', + package: { + name: 'test-package', + }, + }, + ], + }; + + soClient.find.mockResolvedValueOnce({ total: 0, saved_objects: [], page: 1, per_page: 10 }); + soClient.get.mockRejectedValueOnce(SavedObjectsErrorHelpers.createGenericNotFoundError()); + + soClient.create.mockResolvedValueOnce({ + id: 'my-unique-id', + type: AGENT_POLICY_SAVED_OBJECT_TYPE, + attributes: {}, + references: [], + }); + + await agentPolicyService.ensurePreconfiguredAgentPolicy( + soClient, + esClient, + preconfiguredAgentPolicy + ); + + expect(soClient.create).toHaveBeenCalledWith( + AGENT_POLICY_SAVED_OBJECT_TYPE, + expect.anything(), + expect.objectContaining({ id: 'my-unique-id' }) + ); + }); + }); + describe('getInactivityTimeouts', () => { const createPolicySO = (id: string, inactivityTimeout: number) => ({ id, @@ -626,4 +829,18 @@ describe('agent policy', () => { ]); }); }); + + describe('deleteFleetServerPoliciesForPolicyId', () => { + it('should call audit logger', async () => { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + esClient.deleteByQuery.mockResolvedValueOnce({} as any); + + await agentPolicyService.deleteFleetServerPoliciesForPolicyId(esClient, 'test-agent-policy'); + + expect(mockedAuditLoggingService.writeCustomAuditLog).toHaveBeenCalledWith({ + message: 'User deleting policy [id=test-agent-policy]', + }); + }); + }); }); diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index 2720701f0b29c..96b8fd55e3b84 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -15,6 +15,7 @@ import type { SavedObjectsClientContract, SavedObjectsBulkUpdateResponse, } from '@kbn/core/server'; +import { SavedObjectsUtils } from '@kbn/core/server'; import type { AuthenticatedUser } from '@kbn/security-plugin/server'; import type { BulkResponseItem } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; @@ -83,6 +84,7 @@ import { normalizeKuery, escapeSearchQueryPhrase } from './saved_object'; import { appContextService } from './app_context'; import { getFullAgentPolicy } from './agent_policies'; import { validateOutputForPolicy } from './agent_policies'; +import { auditLoggingService } from './audit_logging'; const SAVED_OBJECT_TYPE = AGENT_POLICY_SAVED_OBJECT_TYPE; @@ -106,6 +108,12 @@ class AgentPolicyService { user?: AuthenticatedUser, options: { bumpRevision: boolean } = { bumpRevision: true } ): Promise { + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id, + savedObjectType: AGENT_POLICY_SAVED_OBJECT_TYPE, + }); + const existingAgentPolicy = await this.get(soClient, id, true); if (!existingAgentPolicy) { @@ -201,8 +209,19 @@ class AgentPolicyService { soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, agentPolicy: NewAgentPolicy, - options?: { id?: string; user?: AuthenticatedUser } + options: { id?: string; user?: AuthenticatedUser } = {} ): Promise { + // Ensure an ID is provided, so we can include it in the audit logs below + if (!options.id) { + options.id = SavedObjectsUtils.generateId(); + } + + auditLoggingService.writeCustomSoAuditLog({ + action: 'create', + id: options.id, + savedObjectType: AGENT_POLICY_SAVED_OBJECT_TYPE, + }); + await this.requireUniqueName(soClient, agentPolicy); await validateOutputForPolicy(soClient, agentPolicy); @@ -270,6 +289,12 @@ class AgentPolicyService { (await packagePolicyService.findAllForAgentPolicy(soClient, id)) || []; } + auditLoggingService.writeCustomSoAuditLog({ + action: 'get', + id, + savedObjectType: AGENT_POLICY_SAVED_OBJECT_TYPE, + }); + return agentPolicy; } @@ -313,7 +338,19 @@ class AgentPolicyService { { concurrency: 50 } ); - return agentPolicies.filter((agentPolicy): agentPolicy is AgentPolicy => agentPolicy !== null); + const result = agentPolicies.filter( + (agentPolicy): agentPolicy is AgentPolicy => agentPolicy !== null + ); + + for (const agentPolicy of result) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'get', + id: agentPolicy.id, + savedObjectType: AGENT_POLICY_SAVED_OBJECT_TYPE, + }); + } + + return result; } public async list( @@ -386,6 +423,14 @@ class AgentPolicyService { { concurrency: 50 } ); + for (const agentPolicy of agentPolicies) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'find', + id: agentPolicy.id, + savedObjectType: AGENT_POLICY_SAVED_OBJECT_TYPE, + }); + } + return { items: agentPolicies, total: agentPoliciesSO.total, @@ -659,8 +704,14 @@ class AgentPolicyService { soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, id: string, - options?: { force?: boolean; removeFleetServerDocuments?: boolean } + options?: { force?: boolean; removeFleetServerDocuments?: boolean; user?: AuthenticatedUser } ): Promise { + auditLoggingService.writeCustomSoAuditLog({ + action: 'delete', + id, + savedObjectType: AGENT_POLICY_SAVED_OBJECT_TYPE, + }); + const agentPolicy = await this.get(soClient, id, false); if (!agentPolicy) { throw new Error('Agent policy not found'); @@ -736,6 +787,12 @@ class AgentPolicyService { return; } + for (const policyId of agentPolicyIds) { + auditLoggingService.writeCustomAuditLog({ + message: `User deploying policy [id=${policyId}]`, + }); + } + const policies = await agentPolicyService.getByIDs(soClient, agentPolicyIds); const policiesMap = keyBy(policies, 'id'); const fullPolicies = await Promise.all( @@ -835,6 +892,10 @@ class AgentPolicyService { esClient: ElasticsearchClient, agentPolicyId: string ) { + auditLoggingService.writeCustomAuditLog({ + message: `User deleting policy [id=${agentPolicyId}]`, + }); + await esClient.deleteByQuery({ index: AGENT_POLICY_INDEX, ignore_unavailable: true, diff --git a/x-pack/plugins/fleet/server/services/agent_policy_create.ts b/x-pack/plugins/fleet/server/services/agent_policy_create.ts index 5f3d6d14822c9..1f7aeb2bc985f 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy_create.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy_create.ts @@ -56,6 +56,7 @@ async function createPackagePolicy( // rollback agent policy on error await agentPolicyService.delete(soClient, esClient, agentPolicy.id, { force: true, + user: options.user, }); throw error; }); diff --git a/x-pack/plugins/fleet/server/services/agents/actions.test.ts b/x-pack/plugins/fleet/server/services/agents/actions.test.ts index 30a63e212370b..b6e72b8208b1b 100644 --- a/x-pack/plugins/fleet/server/services/agents/actions.test.ts +++ b/x-pack/plugins/fleet/server/services/agents/actions.test.ts @@ -7,15 +7,26 @@ import { elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import type { NewAgentAction } from '../../../common/types'; + import { createAppContextStartContractMock } from '../../mocks'; import { appContextService } from '../app_context'; +import { auditLoggingService } from '../audit_logging'; -import { cancelAgentAction, getAgentsByActionsIds } from './actions'; +import { + bulkCreateAgentActionResults, + bulkCreateAgentActions, + cancelAgentAction, + createAgentAction, + getAgentsByActionsIds, +} from './actions'; import { bulkUpdateAgents } from './crud'; jest.mock('./crud'); +jest.mock('../audit_logging'); -const mockedBulkUpdateAgents = bulkUpdateAgents as jest.Mock; +const mockedBulkUpdateAgents = bulkUpdateAgents as jest.MockedFunction; +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; describe('Agent actions', () => { beforeEach(async () => { @@ -24,7 +35,115 @@ describe('Agent actions', () => { afterEach(() => { appContextService.stop(); + mockedAuditLoggingService.writeCustomAuditLog.mockReset(); }); + + describe('getAgentActions', () => { + it('should call audit logger', async () => { + const esClientMock = elasticsearchServiceMock.createInternalClient(); + + esClientMock.search.mockResolvedValue({ + hits: { + hits: [ + { + _source: { + action_id: 'action1', + agents: ['agent1'], + expiration: new Date().toISOString(), + type: 'UPGRADE', + }, + }, + ], + }, + } as any); + + await getAgentsByActionsIds(esClientMock, ['action1']); + + expect(mockedAuditLoggingService.writeCustomAuditLog).toHaveBeenCalledWith({ + message: `User retrieved Fleet action [id=action1]`, + }); + }); + }); + + describe('createAgentAction', () => { + it('should call audit logger', async () => { + const esClient = elasticsearchServiceMock.createInternalClient(); + esClient.search.mockResolvedValue({ + hits: { + hits: [ + { + _source: { + type: 'UPGRADE', + action_id: 'action1', + agents: ['agent1', 'agent2'], + expiration: new Date().toISOString(), + }, + }, + ], + }, + } as any); + + await createAgentAction(esClient, { + id: 'action1', + type: 'UPGRADE', + agents: ['agent1'], + }); + + expect(mockedAuditLoggingService.writeCustomAuditLog).toHaveBeenCalledWith({ + message: expect.stringMatching(/User created Fleet action/), + }); + }); + }); + + describe('bulkCreateAgentAction', () => { + it('should call audit logger', async () => { + const esClient = elasticsearchServiceMock.createInternalClient(); + + const newActions: NewAgentAction[] = [ + { + id: 'action1', + type: 'UPGRADE', + agents: ['agent1'], + }, + { + id: 'action2', + type: 'UPGRADE', + agents: ['agent2'], + }, + { + id: 'action3', + type: 'UPGRADE', + agents: ['agent3'], + }, + ]; + + await bulkCreateAgentActions(esClient, newActions); + + for (const action of newActions) { + expect(mockedAuditLoggingService.writeCustomAuditLog).toHaveBeenCalledWith({ + message: `User created Fleet action [id=${action.id}]`, + }); + } + }); + }); + + describe('bulkCreateAgentActionResults', () => { + it('should call audit logger', async () => { + const mockEsClient = elasticsearchServiceMock.createInternalClient(); + + await bulkCreateAgentActionResults(mockEsClient, [ + { + actionId: 'action1', + agentId: 'agent1', + }, + ]); + + expect(mockedAuditLoggingService.writeCustomAuditLog).toHaveBeenCalledWith({ + message: `User created Fleet action result [id=action1]`, + }); + }); + }); + describe('cancelAgentAction', () => { it('throw if the target action is not found', async () => { const esClient = elasticsearchServiceMock.createInternalClient(); @@ -114,6 +233,7 @@ describe('Agent actions', () => { ); }); }); + describe('getAgentsByActionsIds', () => { const esClientMock = elasticsearchServiceMock.createElasticsearchClient(); diff --git a/x-pack/plugins/fleet/server/services/agents/actions.ts b/x-pack/plugins/fleet/server/services/agents/actions.ts index 79d0449b60ff5..bc56b7e80e8af 100644 --- a/x-pack/plugins/fleet/server/services/agents/actions.ts +++ b/x-pack/plugins/fleet/server/services/agents/actions.ts @@ -22,6 +22,8 @@ import { } from '../../../common/constants'; import { AgentActionNotFoundError } from '../../errors'; +import { auditLoggingService } from '../audit_logging'; + import { bulkUpdateAgents } from './crud'; const ONE_MONTH_IN_MS = 2592000000; @@ -57,6 +59,10 @@ export async function createAgentAction( refresh: 'wait_for', }); + auditLoggingService.writeCustomAuditLog({ + message: `User created Fleet action [id=${actionId}]`, + }); + return { id: actionId, ...newAgentAction, @@ -105,6 +111,12 @@ export async function bulkCreateAgentActions( }), }); + for (const action of actions) { + auditLoggingService.writeCustomAuditLog({ + message: `User created Fleet action [id=${action.id}]`, + }); + } + return actions; } @@ -164,6 +176,12 @@ export async function bulkCreateAgentActionResults( ]; }); + for (const result of results) { + auditLoggingService.writeCustomAuditLog({ + message: `User created Fleet action result [id=${result.actionId}]`, + }); + } + await esClient.bulk({ index: AGENT_ACTIONS_RESULTS_INDEX, body: bulkBody, @@ -192,10 +210,20 @@ export async function getAgentActions(esClient: ElasticsearchClient, actionId: s throw new AgentActionNotFoundError('Action not found'); } - return res.hits.hits.map((hit) => ({ - ...hit._source, - id: hit._id, - })) as FleetServerAgentAction[]; + const result: FleetServerAgentAction[] = []; + + for (const hit of res.hits.hits) { + auditLoggingService.writeCustomAuditLog({ + message: `User retrieved Fleet action [id=${hit._source?.action_id}]`, + }); + + result.push({ + ...hit._source, + id: hit._id, + }); + } + + return result; } export async function getUnenrollAgentActions( @@ -227,10 +255,20 @@ export async function getUnenrollAgentActions( size: SO_SEARCH_LIMIT, }); - return res.hits.hits.map((hit) => ({ - ...hit._source, - id: hit._id, - })); + const result: FleetServerAgentAction[] = []; + + for (const hit of res.hits.hits) { + auditLoggingService.writeCustomAuditLog({ + message: `User retrieved Fleet action [id=${hit._source?.action_id}]`, + }); + + result.push({ + ...hit._source, + id: hit._id, + }); + } + + return result; } export async function cancelAgentAction(esClient: ElasticsearchClient, actionId: string) { @@ -255,6 +293,12 @@ export async function cancelAgentAction(esClient: ElasticsearchClient, actionId: throw new AgentActionNotFoundError('Action not found'); } + for (const hit of res.hits.hits) { + auditLoggingService.writeCustomAuditLog({ + message: `User retrieved Fleet action [id=${hit._source?.action_id}]}]`, + }); + } + const upgradeActions: FleetServerAgentAction[] = res.hits.hits .map((hit) => hit._source as FleetServerAgentAction) .filter( @@ -368,10 +412,20 @@ async function getAgentActionsByIds(esClient: ElasticsearchClient, actionIds: st throw new AgentActionNotFoundError('Action not found'); } - return res.hits.hits.map((hit) => ({ - ...hit._source, - id: hit._id, - })) as FleetServerAgentAction[]; + const result: FleetServerAgentAction[] = []; + + for (const hit of res.hits.hits) { + auditLoggingService.writeCustomAuditLog({ + message: `User retrieved Fleet action [id=${hit._source?.action_id}]`, + }); + + result.push({ + ...hit._source, + id: hit._id, + }); + } + + return result; } export const getAgentsByActionsIds = async ( diff --git a/x-pack/plugins/fleet/server/services/agents/crud.test.ts b/x-pack/plugins/fleet/server/services/agents/crud.test.ts index 623ca8d0039bc..47bb8028fffcd 100644 --- a/x-pack/plugins/fleet/server/services/agents/crud.test.ts +++ b/x-pack/plugins/fleet/server/services/agents/crud.test.ts @@ -6,16 +6,28 @@ */ import { errors } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from '@kbn/core/server'; -import { savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { AGENTS_INDEX } from '../../constants'; import type { Agent } from '../../types'; -import { getAgentsByKuery, getAgentTags } from './crud'; +import { auditLoggingService } from '../audit_logging'; +import { + closePointInTime, + getAgentsByKuery, + getAgentTags, + openPointInTime, + updateAgent, +} from './crud'; + +jest.mock('../audit_logging'); jest.mock('../../../common/services/is_agent_upgradeable', () => ({ isAgentUpgradeable: jest.fn().mockImplementation((agent: Agent) => agent.id.includes('up')), })); +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; + describe('Agents CRUD test', () => { const soClientMock = savedObjectsClientMock.create(); let esClientMock: ElasticsearchClient; @@ -308,4 +320,44 @@ describe('Agents CRUD test', () => { expect(searchMock.mock.calls.at(-1)[0].sort).toEqual([{ policy_id: { order: 'desc' } }]); }); }); + + describe('update', () => { + it('should write to audit log', async () => { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + esClient.update.mockResolvedValueOnce({} as any); + + await updateAgent(esClient, 'test-agent-id', { tags: ['new-tag'] }); + + expect(mockedAuditLoggingService.writeCustomAuditLog).toHaveBeenCalledWith({ + message: 'User updated agent [id=test-agent-id]', + }); + }); + }); + + describe('openPointInTime', () => { + it('should call audit logger', async () => { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.openPointInTime.mockResolvedValueOnce({ id: 'test-pit' } as any); + + await openPointInTime(esClient, AGENTS_INDEX); + + expect(mockedAuditLoggingService.writeCustomAuditLog).toHaveBeenCalledWith({ + message: `User opened point in time query [index=${AGENTS_INDEX}] [pitId=test-pit]`, + }); + }); + }); + + describe('closePointInTime', () => { + it('should call audit logger', async () => { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + esClient.closePointInTime.mockResolvedValueOnce({} as any); + + await closePointInTime(esClient, 'test-pit'); + + expect(mockedAuditLoggingService.writeCustomAuditLog).toHaveBeenCalledWith({ + message: `User closing point in time query [pitId=test-pit]`, + }); + }); + }); }); diff --git a/x-pack/plugins/fleet/server/services/agents/crud.ts b/x-pack/plugins/fleet/server/services/agents/crud.ts index 379a656b7d519..fdce358049006 100644 --- a/x-pack/plugins/fleet/server/services/agents/crud.ts +++ b/x-pack/plugins/fleet/server/services/agents/crud.ts @@ -19,6 +19,8 @@ import { isAgentUpgradeable } from '../../../common/services'; import { AGENTS_INDEX } from '../../constants'; import { FleetError, isESClientError, AgentNotFoundError } from '../../errors'; +import { auditLoggingService } from '../audit_logging'; + import { searchHitToAgent, agentSOAttributesToFleetServerAgentDoc } from './helpers'; import { buildAgentStatusRuntimeField } from './build_status_runtime_field'; @@ -105,10 +107,19 @@ export async function openPointInTime( index, keep_alive: pitKeepAlive, }); + + auditLoggingService.writeCustomAuditLog({ + message: `User opened point in time query [index=${index}] [pitId=${pitRes.id}]`, + }); + return pitRes.id; } export async function closePointInTime(esClient: ElasticsearchClient, pitId: string) { + auditLoggingService.writeCustomAuditLog({ + message: `User closing point in time query [pitId=${pitId}]`, + }); + try { await esClient.closePointInTime({ id: pitId }); } catch (error) { @@ -465,6 +476,10 @@ export async function updateAgent( agentId: string, data: Partial ) { + auditLoggingService.writeCustomAuditLog({ + message: `User updated agent [id=${agentId}]`, + }); + await esClient.update({ id: agentId, index: AGENTS_INDEX, diff --git a/x-pack/plugins/fleet/server/services/agents/reassign.test.ts b/x-pack/plugins/fleet/server/services/agents/reassign.test.ts index 8ef9ef77b7d92..6261b3295496a 100644 --- a/x-pack/plugins/fleet/server/services/agents/reassign.test.ts +++ b/x-pack/plugins/fleet/server/services/agents/reassign.test.ts @@ -14,49 +14,7 @@ import { createAppContextStartContractMock } from '../../mocks'; import { reassignAgent, reassignAgents } from './reassign'; import { createClientMock } from './action.mock'; -describe('reassignAgent (singular)', () => { - it('can reassign from regular agent policy to regular', async () => { - const { soClient, esClient, agentInRegularDoc, regularAgentPolicySO } = createClientMock(); - await reassignAgent(soClient, esClient, agentInRegularDoc._id, regularAgentPolicySO.id); - - // calls ES update with correct values - expect(esClient.update).toBeCalledTimes(1); - const calledWith = esClient.update.mock.calls[0]; - expect(calledWith[0]?.id).toBe(agentInRegularDoc._id); - expect((calledWith[0] as estypes.UpdateRequest)?.body?.doc).toHaveProperty( - 'policy_id', - regularAgentPolicySO.id - ); - }); - - it('cannot reassign from regular agent policy to hosted', async () => { - const { soClient, esClient, agentInRegularDoc, hostedAgentPolicySO } = createClientMock(); - await expect( - reassignAgent(soClient, esClient, agentInRegularDoc._id, hostedAgentPolicySO.id) - ).rejects.toThrowError(HostedAgentPolicyRestrictionRelatedError); - - // does not call ES update - expect(esClient.update).toBeCalledTimes(0); - }); - - it('cannot reassign from hosted agent policy', async () => { - const { soClient, esClient, agentInHostedDoc, hostedAgentPolicySO, regularAgentPolicySO } = - createClientMock(); - await expect( - reassignAgent(soClient, esClient, agentInHostedDoc._id, regularAgentPolicySO.id) - ).rejects.toThrowError(HostedAgentPolicyRestrictionRelatedError); - // does not call ES update - expect(esClient.update).toBeCalledTimes(0); - - await expect( - reassignAgent(soClient, esClient, agentInHostedDoc._id, hostedAgentPolicySO.id) - ).rejects.toThrowError(HostedAgentPolicyRestrictionRelatedError); - // does not call ES update - expect(esClient.update).toBeCalledTimes(0); - }); -}); - -describe('reassignAgents (plural)', () => { +describe('reassignAgent', () => { beforeEach(async () => { appContextService.start(createAppContextStartContractMock()); }); @@ -64,68 +22,122 @@ describe('reassignAgents (plural)', () => { afterEach(() => { appContextService.stop(); }); - it('agents in hosted policies are not updated', async () => { - const { - soClient, - esClient, - agentInRegularDoc, - agentInHostedDoc, - agentInHostedDoc2, - regularAgentPolicySO2, - } = createClientMock(); - - esClient.search.mockResponse({ - hits: { - hits: [agentInRegularDoc, agentInHostedDoc, agentInHostedDoc2], - }, - } as any); - - const idsToReassign = [agentInRegularDoc._id, agentInHostedDoc._id, agentInHostedDoc2._id]; - await reassignAgents(soClient, esClient, { agentIds: idsToReassign }, regularAgentPolicySO2.id); - - // calls ES update with correct values - const calledWith = esClient.bulk.mock.calls[0][0]; - // only 1 are regular and bulk write two line per update - expect((calledWith as estypes.BulkRequest).body?.length).toBe(2); - // @ts-expect-error - expect(calledWith.body[0].update._id).toEqual(agentInRegularDoc._id); - - // hosted policy is updated in action results with error - const calledWithActionResults = esClient.bulk.mock.calls[1][0] as estypes.BulkRequest; - // bulk write two line per create - expect(calledWithActionResults.body?.length).toBe(4); - const expectedObject = expect.objectContaining({ - '@timestamp': expect.anything(), - action_id: expect.anything(), - agent_id: 'agent-in-hosted-policy', - error: - 'Cannot reassign an agent from hosted agent policy hosted-agent-policy in Fleet because the agent policy is managed by an external orchestration solution, such as Elastic Cloud, Kubernetes, etc. Please make changes using your orchestration solution.', + describe('reassignAgent (singular)', () => { + it('can reassign from regular agent policy to regular', async () => { + const { soClient, esClient, agentInRegularDoc, regularAgentPolicySO } = createClientMock(); + await reassignAgent(soClient, esClient, agentInRegularDoc._id, regularAgentPolicySO.id); + + // calls ES update with correct values + expect(esClient.update).toBeCalledTimes(1); + const calledWith = esClient.update.mock.calls[0]; + expect(calledWith[0]?.id).toBe(agentInRegularDoc._id); + expect((calledWith[0] as estypes.UpdateRequest)?.body?.doc).toHaveProperty( + 'policy_id', + regularAgentPolicySO.id + ); + }); + + it('cannot reassign from regular agent policy to hosted', async () => { + const { soClient, esClient, agentInRegularDoc, hostedAgentPolicySO } = createClientMock(); + await expect( + reassignAgent(soClient, esClient, agentInRegularDoc._id, hostedAgentPolicySO.id) + ).rejects.toThrowError(HostedAgentPolicyRestrictionRelatedError); + + // does not call ES update + expect(esClient.update).toBeCalledTimes(0); + }); + + it('cannot reassign from hosted agent policy', async () => { + const { soClient, esClient, agentInHostedDoc, hostedAgentPolicySO, regularAgentPolicySO } = + createClientMock(); + await expect( + reassignAgent(soClient, esClient, agentInHostedDoc._id, regularAgentPolicySO.id) + ).rejects.toThrowError(HostedAgentPolicyRestrictionRelatedError); + // does not call ES update + expect(esClient.update).toBeCalledTimes(0); + + await expect( + reassignAgent(soClient, esClient, agentInHostedDoc._id, hostedAgentPolicySO.id) + ).rejects.toThrowError(HostedAgentPolicyRestrictionRelatedError); + // does not call ES update + expect(esClient.update).toBeCalledTimes(0); }); - expect(calledWithActionResults.body?.[1] as any).toEqual(expectedObject); }); - it('should report errors from ES agent update call', async () => { - const { soClient, esClient, agentInRegularDoc, regularAgentPolicySO2 } = createClientMock(); - esClient.bulk.mockResponse({ - items: [ - { - update: { - _id: agentInRegularDoc._id, - error: new Error('version conflict'), - }, + describe('reassignAgents (plural)', () => { + it('agents in hosted policies are not updated', async () => { + const { + soClient, + esClient, + agentInRegularDoc, + agentInHostedDoc, + agentInHostedDoc2, + regularAgentPolicySO2, + } = createClientMock(); + + esClient.search.mockResponse({ + hits: { + hits: [agentInRegularDoc, agentInHostedDoc, agentInHostedDoc2], }, - ], - } as any); - const idsToReassign = [agentInRegularDoc._id]; - await reassignAgents(soClient, esClient, { agentIds: idsToReassign }, regularAgentPolicySO2.id); - - const calledWithActionResults = esClient.bulk.mock.calls[1][0] as estypes.BulkRequest; - const expectedObject = expect.objectContaining({ - '@timestamp': expect.anything(), - action_id: expect.anything(), - agent_id: agentInRegularDoc._id, - error: 'version conflict', + } as any); + + const idsToReassign = [agentInRegularDoc._id, agentInHostedDoc._id, agentInHostedDoc2._id]; + await reassignAgents( + soClient, + esClient, + { agentIds: idsToReassign }, + regularAgentPolicySO2.id + ); + + // calls ES update with correct values + const calledWith = esClient.bulk.mock.calls[0][0]; + // only 1 are regular and bulk write two line per update + expect((calledWith as estypes.BulkRequest).body?.length).toBe(2); + // @ts-expect-error + expect(calledWith.body[0].update._id).toEqual(agentInRegularDoc._id); + + // hosted policy is updated in action results with error + const calledWithActionResults = esClient.bulk.mock.calls[1][0] as estypes.BulkRequest; + // bulk write two line per create + expect(calledWithActionResults.body?.length).toBe(4); + const expectedObject = expect.objectContaining({ + '@timestamp': expect.anything(), + action_id: expect.anything(), + agent_id: 'agent-in-hosted-policy', + error: + 'Cannot reassign an agent from hosted agent policy hosted-agent-policy in Fleet because the agent policy is managed by an external orchestration solution, such as Elastic Cloud, Kubernetes, etc. Please make changes using your orchestration solution.', + }); + expect(calledWithActionResults.body?.[1] as any).toEqual(expectedObject); + }); + + it('should report errors from ES agent update call', async () => { + const { soClient, esClient, agentInRegularDoc, regularAgentPolicySO2 } = createClientMock(); + esClient.bulk.mockResponse({ + items: [ + { + update: { + _id: agentInRegularDoc._id, + error: new Error('version conflict'), + }, + }, + ], + } as any); + const idsToReassign = [agentInRegularDoc._id]; + await reassignAgents( + soClient, + esClient, + { agentIds: idsToReassign }, + regularAgentPolicySO2.id + ); + + const calledWithActionResults = esClient.bulk.mock.calls[1][0] as estypes.BulkRequest; + const expectedObject = expect.objectContaining({ + '@timestamp': expect.anything(), + action_id: expect.anything(), + agent_id: agentInRegularDoc._id, + error: 'version conflict', + }); + expect(calledWithActionResults.body?.[1] as any).toEqual(expectedObject); }); - expect(calledWithActionResults.body?.[1] as any).toEqual(expectedObject); }); }); diff --git a/x-pack/plugins/fleet/server/services/agents/request_diagnostics.test.ts b/x-pack/plugins/fleet/server/services/agents/request_diagnostics.test.ts index 3c459ce178a3a..b297e4cb7c584 100644 --- a/x-pack/plugins/fleet/server/services/agents/request_diagnostics.test.ts +++ b/x-pack/plugins/fleet/server/services/agents/request_diagnostics.test.ts @@ -12,24 +12,7 @@ import { createAppContextStartContractMock } from '../../mocks'; import { createClientMock } from './action.mock'; import { bulkRequestDiagnostics, requestDiagnostics } from './request_diagnostics'; -describe('requestDiagnostics (singular)', () => { - it('can request diagnostics for single agent', async () => { - const { esClient, agentInRegularDoc } = createClientMock(); - await requestDiagnostics(esClient, agentInRegularDoc._id); - - expect(esClient.create).toHaveBeenCalledWith( - expect.objectContaining({ - body: expect.objectContaining({ - agents: ['agent-in-regular-policy'], - type: 'REQUEST_DIAGNOSTICS', - }), - index: '.fleet-actions', - }) - ); - }); -}); - -describe('requestDiagnostics (plural)', () => { +describe('requestDiagnostics', () => { beforeEach(async () => { appContextService.start(createAppContextStartContractMock()); }); @@ -37,46 +20,66 @@ describe('requestDiagnostics (plural)', () => { afterEach(() => { appContextService.stop(); }); - it('can request diagnostics for multiple agents', async () => { - const { soClient, esClient, agentInRegularDocNewer, agentInRegularDocNewer2 } = - createClientMock(); - const idsToAction = [agentInRegularDocNewer._id, agentInRegularDocNewer2._id]; - await bulkRequestDiagnostics(esClient, soClient, { agentIds: idsToAction }); - expect(esClient.create).toHaveBeenCalledWith( - expect.objectContaining({ - body: expect.objectContaining({ - agents: ['agent-in-regular-policy-newer', 'agent-in-regular-policy-newer2'], - type: 'REQUEST_DIAGNOSTICS', - }), - index: '.fleet-actions', - }) - ); + describe('requestDiagnostics (singular)', () => { + it('can request diagnostics for single agent', async () => { + const { esClient, agentInRegularDoc } = createClientMock(); + await requestDiagnostics(esClient, agentInRegularDoc._id); + + expect(esClient.create).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.objectContaining({ + agents: ['agent-in-regular-policy'], + type: 'REQUEST_DIAGNOSTICS', + }), + index: '.fleet-actions', + }) + ); + }); }); - it('should report error when diagnostics for older agent', async () => { - const { soClient, esClient, agentInRegularDoc, agentInRegularDocNewer } = createClientMock(); - const idsToAction = [agentInRegularDocNewer._id, agentInRegularDoc._id]; - await bulkRequestDiagnostics(esClient, soClient, { agentIds: idsToAction }); + describe('requestDiagnostics (plural)', () => { + it('can request diagnostics for multiple agents', async () => { + const { soClient, esClient, agentInRegularDocNewer, agentInRegularDocNewer2 } = + createClientMock(); + const idsToAction = [agentInRegularDocNewer._id, agentInRegularDocNewer2._id]; + await bulkRequestDiagnostics(esClient, soClient, { agentIds: idsToAction }); + + expect(esClient.create).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.objectContaining({ + agents: ['agent-in-regular-policy-newer', 'agent-in-regular-policy-newer2'], + type: 'REQUEST_DIAGNOSTICS', + }), + index: '.fleet-actions', + }) + ); + }); + + it('should report error when diagnostics for older agent', async () => { + const { soClient, esClient, agentInRegularDoc, agentInRegularDocNewer } = createClientMock(); + const idsToAction = [agentInRegularDocNewer._id, agentInRegularDoc._id]; + await bulkRequestDiagnostics(esClient, soClient, { agentIds: idsToAction }); - expect(esClient.create).toHaveBeenCalledWith( - expect.objectContaining({ - body: expect.objectContaining({ - agents: ['agent-in-regular-policy-newer', 'agent-in-regular-policy'], - type: 'REQUEST_DIAGNOSTICS', - }), - index: '.fleet-actions', - }) - ); - const calledWithActionResults = esClient.bulk.mock.calls[0][0] as estypes.BulkRequest; - // bulk write two line per create - expect(calledWithActionResults.body?.length).toBe(2); - const expectedObject = expect.objectContaining({ - '@timestamp': expect.anything(), - action_id: expect.anything(), - agent_id: 'agent-in-regular-policy', - error: 'Agent agent-in-regular-policy does not support request diagnostics action.', + expect(esClient.create).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.objectContaining({ + agents: ['agent-in-regular-policy-newer', 'agent-in-regular-policy'], + type: 'REQUEST_DIAGNOSTICS', + }), + index: '.fleet-actions', + }) + ); + const calledWithActionResults = esClient.bulk.mock.calls[0][0] as estypes.BulkRequest; + // bulk write two line per create + expect(calledWithActionResults.body?.length).toBe(2); + const expectedObject = expect.objectContaining({ + '@timestamp': expect.anything(), + action_id: expect.anything(), + agent_id: 'agent-in-regular-policy', + error: 'Agent agent-in-regular-policy does not support request diagnostics action.', + }); + expect(calledWithActionResults.body?.[1] as any).toEqual(expectedObject); }); - expect(calledWithActionResults.body?.[1] as any).toEqual(expectedObject); }); }); diff --git a/x-pack/plugins/fleet/server/services/agents/unenroll.test.ts b/x-pack/plugins/fleet/server/services/agents/unenroll.test.ts index ddf7f0a2ab8f0..4231d8dfc078d 100644 --- a/x-pack/plugins/fleet/server/services/agents/unenroll.test.ts +++ b/x-pack/plugins/fleet/server/services/agents/unenroll.test.ts @@ -25,62 +25,7 @@ jest.mock('../api_keys'); const mockedInvalidateAPIKeys = invalidateAPIKeys as jest.MockedFunction; -describe('unenrollAgent (singular)', () => { - it('can unenroll from regular agent policy', async () => { - const { soClient, esClient, agentInRegularDoc } = createClientMock(); - await unenrollAgent(soClient, esClient, agentInRegularDoc._id); - - // calls ES update with correct values - expect(esClient.update).toBeCalledTimes(1); - const calledWith = esClient.update.mock.calls[0]; - expect(calledWith[0]?.id).toBe(agentInRegularDoc._id); - expect((calledWith[0] as estypes.UpdateRequest)?.body).toHaveProperty( - 'doc.unenrollment_started_at' - ); - }); - - it('cannot unenroll from hosted agent policy by default', async () => { - const { soClient, esClient, agentInHostedDoc } = createClientMock(); - await expect(unenrollAgent(soClient, esClient, agentInHostedDoc._id)).rejects.toThrowError( - HostedAgentPolicyRestrictionRelatedError - ); - // does not call ES update - expect(esClient.update).toBeCalledTimes(0); - }); - - it('cannot unenroll from hosted agent policy with revoke=true', async () => { - const { soClient, esClient, agentInHostedDoc } = createClientMock(); - await expect( - unenrollAgent(soClient, esClient, agentInHostedDoc._id, { revoke: true }) - ).rejects.toThrowError(HostedAgentPolicyRestrictionRelatedError); - // does not call ES update - expect(esClient.update).toBeCalledTimes(0); - }); - - it('can unenroll from hosted agent policy with force=true', async () => { - const { soClient, esClient, agentInHostedDoc } = createClientMock(); - await unenrollAgent(soClient, esClient, agentInHostedDoc._id, { force: true }); - // calls ES update with correct values - expect(esClient.update).toBeCalledTimes(1); - const calledWith = esClient.update.mock.calls[0]; - expect(calledWith[0]?.id).toBe(agentInHostedDoc._id); - expect((calledWith[0] as estypes.UpdateRequest)?.body).toHaveProperty( - 'doc.unenrollment_started_at' - ); - }); - - it('can unenroll from hosted agent policy with force=true and revoke=true', async () => { - const { soClient, esClient, agentInHostedDoc } = createClientMock(); - await unenrollAgent(soClient, esClient, agentInHostedDoc._id, { force: true, revoke: true }); - // calls ES update with correct values - expect(esClient.update).toBeCalledTimes(1); - const calledWith = esClient.update.mock.calls[0]; - expect(calledWith[0]?.id).toBe(agentInHostedDoc._id); - expect((calledWith[0] as estypes.UpdateRequest)?.body).toHaveProperty('doc.unenrolled_at'); - }); -}); - -describe('unenrollAgents (plural)', () => { +describe('unenroll', () => { beforeEach(async () => { appContextService.start(createAppContextStartContractMock()); }); @@ -88,320 +33,381 @@ describe('unenrollAgents (plural)', () => { afterEach(() => { appContextService.stop(); }); - it('can unenroll from an regular agent policy', async () => { - const { soClient, esClient, agentInRegularDoc, agentInRegularDoc2 } = createClientMock(); - const idsToUnenroll = [agentInRegularDoc._id, agentInRegularDoc2._id]; - await unenrollAgents(soClient, esClient, { agentIds: idsToUnenroll }); - - // calls ES update with correct values - const calledWith = esClient.bulk.mock.calls[0][0]; - const ids = (calledWith as estypes.BulkRequest)?.body - ?.filter((i: any) => i.update !== undefined) - .map((i: any) => i.update._id); - const docs = (calledWith as estypes.BulkRequest)?.body - ?.filter((i: any) => i.doc) - .map((i: any) => i.doc); - expect(ids).toEqual(idsToUnenroll); - for (const doc of docs!) { - expect(doc).toHaveProperty('unenrollment_started_at'); - } - }); - it('cannot unenroll from a hosted agent policy by default', async () => { - const { soClient, esClient, agentInHostedDoc, agentInRegularDoc, agentInRegularDoc2 } = - createClientMock(); - - const idsToUnenroll = [agentInRegularDoc._id, agentInHostedDoc._id, agentInRegularDoc2._id]; - await unenrollAgents(soClient, esClient, { agentIds: idsToUnenroll }); - - // calls ES update with correct values - const onlyRegular = [agentInRegularDoc._id, agentInRegularDoc2._id]; - const calledWith = esClient.bulk.mock.calls[0][0]; - const ids = (calledWith as estypes.BulkRequest)?.body - ?.filter((i: any) => i.update !== undefined) - .map((i: any) => i.update._id); - const docs = (calledWith as estypes.BulkRequest)?.body - ?.filter((i: any) => i.doc) - .map((i: any) => i.doc); - expect(ids).toEqual(onlyRegular); - for (const doc of docs!) { - expect(doc).toHaveProperty('unenrollment_started_at'); - } - - // hosted policy is updated in action results with error - const calledWithActionResults = esClient.bulk.mock.calls[1][0] as estypes.BulkRequest; - // bulk write two line per create - expect(calledWithActionResults.body?.length).toBe(2); - const expectedObject = expect.objectContaining({ - '@timestamp': expect.anything(), - action_id: expect.anything(), - agent_id: 'agent-in-hosted-policy', - error: - 'Cannot unenroll agent-in-hosted-policy from a hosted agent policy hosted-agent-policy in Fleet because the agent policy is managed by an external orchestration solution, such as Elastic Cloud, Kubernetes, etc. Please make changes using your orchestration solution.', - }); - expect(calledWithActionResults.body?.[1] as any).toEqual(expectedObject); - }); - it('force unenroll updates in progress unenroll actions', async () => { - const { soClient, esClient, agentInRegularDoc, agentInRegularDoc2 } = createClientMock(); - esClient.search.mockReset(); - - esClient.search.mockImplementation(async (request) => { - if (request?.index === AGENT_ACTIONS_INDEX) { - return { - hits: { - hits: [ - { - _source: { - agents: ['agent-in-regular-policy'], - action_id: 'other-action', - }, - }, - ], - }, - } as any; - } + describe('unenrollAgent (singular)', () => { + it('can unenroll from regular agent policy', async () => { + const { soClient, esClient, agentInRegularDoc } = createClientMock(); + await unenrollAgent(soClient, esClient, agentInRegularDoc._id); + + // calls ES update with correct values + expect(esClient.update).toBeCalledTimes(1); + const calledWith = esClient.update.mock.calls[0]; + expect(calledWith[0]?.id).toBe(agentInRegularDoc._id); + expect((calledWith[0] as estypes.UpdateRequest)?.body).toHaveProperty( + 'doc.unenrollment_started_at' + ); + }); - if (request?.index === AGENT_ACTIONS_RESULTS_INDEX) { - return { - hits: { - hits: [], - }, - }; - } + it('cannot unenroll from hosted agent policy by default', async () => { + const { soClient, esClient, agentInHostedDoc } = createClientMock(); + await expect(unenrollAgent(soClient, esClient, agentInHostedDoc._id)).rejects.toThrowError( + HostedAgentPolicyRestrictionRelatedError + ); + // does not call ES update + expect(esClient.update).toBeCalledTimes(0); + }); - return { hits: { hits: [agentInRegularDoc, agentInRegularDoc2] } }; + it('cannot unenroll from hosted agent policy with revoke=true', async () => { + const { soClient, esClient, agentInHostedDoc } = createClientMock(); + await expect( + unenrollAgent(soClient, esClient, agentInHostedDoc._id, { revoke: true }) + ).rejects.toThrowError(HostedAgentPolicyRestrictionRelatedError); + // does not call ES update + expect(esClient.update).toBeCalledTimes(0); }); - const idsToUnenroll = [agentInRegularDoc._id, agentInRegularDoc2._id]; - await unenrollAgents(soClient, esClient, { - agentIds: idsToUnenroll, - revoke: true, + it('can unenroll from hosted agent policy with force=true', async () => { + const { soClient, esClient, agentInHostedDoc } = createClientMock(); + await unenrollAgent(soClient, esClient, agentInHostedDoc._id, { force: true }); + // calls ES update with correct values + expect(esClient.update).toBeCalledTimes(1); + const calledWith = esClient.update.mock.calls[0]; + expect(calledWith[0]?.id).toBe(agentInHostedDoc._id); + expect((calledWith[0] as estypes.UpdateRequest)?.body).toHaveProperty( + 'doc.unenrollment_started_at' + ); }); - expect(esClient.bulk.mock.calls.length).toEqual(3); - const bulkBody = (esClient.bulk.mock.calls[2][0] as estypes.BulkRequest)?.body?.[1] as any; - expect(bulkBody.agent_id).toEqual(agentInRegularDoc._id); - expect(bulkBody.action_id).toEqual('other-action'); + it('can unenroll from hosted agent policy with force=true and revoke=true', async () => { + const { soClient, esClient, agentInHostedDoc } = createClientMock(); + await unenrollAgent(soClient, esClient, agentInHostedDoc._id, { force: true, revoke: true }); + // calls ES update with correct values + expect(esClient.update).toBeCalledTimes(1); + const calledWith = esClient.update.mock.calls[0]; + expect(calledWith[0]?.id).toBe(agentInHostedDoc._id); + expect((calledWith[0] as estypes.UpdateRequest)?.body).toHaveProperty('doc.unenrolled_at'); + }); }); - it('force unenroll should not update completed unenroll actions', async () => { - const { soClient, esClient, agentInRegularDoc, agentInRegularDoc2 } = createClientMock(); - esClient.search.mockReset(); - esClient.search.mockImplementation(async (request) => { - if (request?.index === AGENT_ACTIONS_INDEX) { - return { - hits: { - hits: [ - { - _source: { - agents: ['agent-in-regular-policy'], - action_id: 'other-action1', - }, - }, - ], - }, - } as any; + describe('unenrollAgents (plural)', () => { + it('can unenroll from an regular agent policy', async () => { + const { soClient, esClient, agentInRegularDoc, agentInRegularDoc2 } = createClientMock(); + const idsToUnenroll = [agentInRegularDoc._id, agentInRegularDoc2._id]; + await unenrollAgents(soClient, esClient, { agentIds: idsToUnenroll }); + + // calls ES update with correct values + const calledWith = esClient.bulk.mock.calls[0][0]; + const ids = (calledWith as estypes.BulkRequest)?.body + ?.filter((i: any) => i.update !== undefined) + .map((i: any) => i.update._id); + const docs = (calledWith as estypes.BulkRequest)?.body + ?.filter((i: any) => i.doc) + .map((i: any) => i.doc); + expect(ids).toEqual(idsToUnenroll); + for (const doc of docs!) { + expect(doc).toHaveProperty('unenrollment_started_at'); } - - if (request?.index === AGENT_ACTIONS_RESULTS_INDEX) { - return { - hits: { - hits: [ - { _source: { action_id: 'other-action1', agent_id: 'agent-in-regular-policy' } }, - ], - }, - }; + }); + it('cannot unenroll from a hosted agent policy by default', async () => { + const { soClient, esClient, agentInHostedDoc, agentInRegularDoc, agentInRegularDoc2 } = + createClientMock(); + + const idsToUnenroll = [agentInRegularDoc._id, agentInHostedDoc._id, agentInRegularDoc2._id]; + await unenrollAgents(soClient, esClient, { agentIds: idsToUnenroll }); + + // calls ES update with correct values + const onlyRegular = [agentInRegularDoc._id, agentInRegularDoc2._id]; + const calledWith = esClient.bulk.mock.calls[0][0]; + const ids = (calledWith as estypes.BulkRequest)?.body + ?.filter((i: any) => i.update !== undefined) + .map((i: any) => i.update._id); + const docs = (calledWith as estypes.BulkRequest)?.body + ?.filter((i: any) => i.doc) + .map((i: any) => i.doc); + expect(ids).toEqual(onlyRegular); + for (const doc of docs!) { + expect(doc).toHaveProperty('unenrollment_started_at'); } - return { hits: { hits: [agentInRegularDoc, agentInRegularDoc2] } }; + // hosted policy is updated in action results with error + const calledWithActionResults = esClient.bulk.mock.calls[1][0] as estypes.BulkRequest; + // bulk write two line per create + expect(calledWithActionResults.body?.length).toBe(2); + const expectedObject = expect.objectContaining({ + '@timestamp': expect.anything(), + action_id: expect.anything(), + agent_id: 'agent-in-hosted-policy', + error: + 'Cannot unenroll agent-in-hosted-policy from a hosted agent policy hosted-agent-policy in Fleet because the agent policy is managed by an external orchestration solution, such as Elastic Cloud, Kubernetes, etc. Please make changes using your orchestration solution.', + }); + expect(calledWithActionResults.body?.[1] as any).toEqual(expectedObject); }); - const idsToUnenroll = [agentInRegularDoc._id, agentInRegularDoc2._id]; - await unenrollAgents(soClient, esClient, { - agentIds: idsToUnenroll, - revoke: true, + it('force unenroll updates in progress unenroll actions', async () => { + const { soClient, esClient, agentInRegularDoc, agentInRegularDoc2 } = createClientMock(); + esClient.search.mockReset(); + + esClient.search.mockImplementation(async (request) => { + if (request?.index === AGENT_ACTIONS_INDEX) { + return { + hits: { + hits: [ + { + _source: { + agents: ['agent-in-regular-policy'], + action_id: 'other-action', + }, + }, + ], + }, + } as any; + } + + if (request?.index === AGENT_ACTIONS_RESULTS_INDEX) { + return { + hits: { + hits: [], + }, + }; + } + + return { hits: { hits: [agentInRegularDoc, agentInRegularDoc2] } }; + }); + + const idsToUnenroll = [agentInRegularDoc._id, agentInRegularDoc2._id]; + await unenrollAgents(soClient, esClient, { + agentIds: idsToUnenroll, + revoke: true, + }); + + expect(esClient.bulk.mock.calls.length).toEqual(3); + const bulkBody = (esClient.bulk.mock.calls[2][0] as estypes.BulkRequest)?.body?.[1] as any; + expect(bulkBody.agent_id).toEqual(agentInRegularDoc._id); + expect(bulkBody.action_id).toEqual('other-action'); }); - // agent and force unenroll results updated, no other action results - expect(esClient.bulk.mock.calls.length).toEqual(2); - }); + it('force unenroll should not update completed unenroll actions', async () => { + const { soClient, esClient, agentInRegularDoc, agentInRegularDoc2 } = createClientMock(); + esClient.search.mockReset(); + esClient.search.mockImplementation(async (request) => { + if (request?.index === AGENT_ACTIONS_INDEX) { + return { + hits: { + hits: [ + { + _source: { + agents: ['agent-in-regular-policy'], + action_id: 'other-action1', + }, + }, + ], + }, + } as any; + } + + if (request?.index === AGENT_ACTIONS_RESULTS_INDEX) { + return { + hits: { + hits: [ + { _source: { action_id: 'other-action1', agent_id: 'agent-in-regular-policy' } }, + ], + }, + }; + } + + return { hits: { hits: [agentInRegularDoc, agentInRegularDoc2] } }; + }); + + const idsToUnenroll = [agentInRegularDoc._id, agentInRegularDoc2._id]; + await unenrollAgents(soClient, esClient, { + agentIds: idsToUnenroll, + revoke: true, + }); + + // agent and force unenroll results updated, no other action results + expect(esClient.bulk.mock.calls.length).toEqual(2); + }); + + it('cannot unenroll from a hosted agent policy with revoke=true', async () => { + const { soClient, esClient, agentInHostedDoc, agentInRegularDoc, agentInRegularDoc2 } = + createClientMock(); + + const idsToUnenroll = [agentInRegularDoc._id, agentInHostedDoc._id, agentInRegularDoc2._id]; + + const unenrolledResponse = await unenrollAgents(soClient, esClient, { + agentIds: idsToUnenroll, + revoke: true, + }); + expect(unenrolledResponse.actionId).toBeDefined(); + + // calls ES update with correct values + const onlyRegular = [agentInRegularDoc._id, agentInRegularDoc2._id]; + const calledWith = esClient.bulk.mock.calls[0][0]; + const ids = (calledWith as estypes.BulkRequest)?.body + ?.filter((i: any) => i.update !== undefined) + .map((i: any) => i.update._id); + const docs = (calledWith as estypes.BulkRequest)?.body + ?.filter((i: any) => i.doc) + .map((i: any) => i.doc); + expect(ids).toEqual(onlyRegular); + for (const doc of docs!) { + expect(doc).toHaveProperty('unenrolled_at'); + } - it('cannot unenroll from a hosted agent policy with revoke=true', async () => { - const { soClient, esClient, agentInHostedDoc, agentInRegularDoc, agentInRegularDoc2 } = - createClientMock(); + const errorResults = esClient.bulk.mock.calls[2][0]; + const errorIds = (errorResults as estypes.BulkRequest)?.body + ?.filter((i: any) => i.agent_id) + .map((i: any) => i.agent_id); + expect(errorIds).toEqual([agentInHostedDoc._id]); - const idsToUnenroll = [agentInRegularDoc._id, agentInHostedDoc._id, agentInRegularDoc2._id]; + const actionResults = esClient.bulk.mock.calls[1][0]; + const resultIds = (actionResults as estypes.BulkRequest)?.body + ?.filter((i: any) => i.agent_id) + .map((i: any) => i.agent_id); + expect(resultIds).toEqual(onlyRegular); - const unenrolledResponse = await unenrollAgents(soClient, esClient, { - agentIds: idsToUnenroll, - revoke: true, + const action = esClient.create.mock.calls[0][0] as any; + expect(action.body.type).toEqual('FORCE_UNENROLL'); }); - expect(unenrolledResponse.actionId).toBeDefined(); - - // calls ES update with correct values - const onlyRegular = [agentInRegularDoc._id, agentInRegularDoc2._id]; - const calledWith = esClient.bulk.mock.calls[0][0]; - const ids = (calledWith as estypes.BulkRequest)?.body - ?.filter((i: any) => i.update !== undefined) - .map((i: any) => i.update._id); - const docs = (calledWith as estypes.BulkRequest)?.body - ?.filter((i: any) => i.doc) - .map((i: any) => i.doc); - expect(ids).toEqual(onlyRegular); - for (const doc of docs!) { - expect(doc).toHaveProperty('unenrolled_at'); - } - - const errorResults = esClient.bulk.mock.calls[2][0]; - const errorIds = (errorResults as estypes.BulkRequest)?.body - ?.filter((i: any) => i.agent_id) - .map((i: any) => i.agent_id); - expect(errorIds).toEqual([agentInHostedDoc._id]); - - const actionResults = esClient.bulk.mock.calls[1][0]; - const resultIds = (actionResults as estypes.BulkRequest)?.body - ?.filter((i: any) => i.agent_id) - .map((i: any) => i.agent_id); - expect(resultIds).toEqual(onlyRegular); - - const action = esClient.create.mock.calls[0][0] as any; - expect(action.body.type).toEqual('FORCE_UNENROLL'); - }); - it('can unenroll from hosted agent policy with force=true', async () => { - const { soClient, esClient, agentInHostedDoc, agentInRegularDoc, agentInRegularDoc2 } = - createClientMock(); - const idsToUnenroll = [agentInRegularDoc._id, agentInHostedDoc._id, agentInRegularDoc2._id]; - await unenrollAgents(soClient, esClient, { agentIds: idsToUnenroll, force: true }); - - // calls ES update with correct values - const calledWith = esClient.bulk.mock.calls[0][0]; - const ids = (calledWith as estypes.BulkRequest)?.body - ?.filter((i: any) => i.update !== undefined) - .map((i: any) => i.update._id); - const docs = (calledWith as estypes.BulkRequest)?.body - ?.filter((i: any) => i.doc) - .map((i: any) => i.doc); - expect(ids).toEqual(idsToUnenroll); - for (const doc of docs!) { - expect(doc).toHaveProperty('unenrollment_started_at'); - } - }); + it('can unenroll from hosted agent policy with force=true', async () => { + const { soClient, esClient, agentInHostedDoc, agentInRegularDoc, agentInRegularDoc2 } = + createClientMock(); + const idsToUnenroll = [agentInRegularDoc._id, agentInHostedDoc._id, agentInRegularDoc2._id]; + await unenrollAgents(soClient, esClient, { agentIds: idsToUnenroll, force: true }); + + // calls ES update with correct values + const calledWith = esClient.bulk.mock.calls[0][0]; + const ids = (calledWith as estypes.BulkRequest)?.body + ?.filter((i: any) => i.update !== undefined) + .map((i: any) => i.update._id); + const docs = (calledWith as estypes.BulkRequest)?.body + ?.filter((i: any) => i.doc) + .map((i: any) => i.doc); + expect(ids).toEqual(idsToUnenroll); + for (const doc of docs!) { + expect(doc).toHaveProperty('unenrollment_started_at'); + } + }); - it('can unenroll from hosted agent policy with force=true and revoke=true', async () => { - const { soClient, esClient, agentInHostedDoc, agentInRegularDoc, agentInRegularDoc2 } = - createClientMock(); + it('can unenroll from hosted agent policy with force=true and revoke=true', async () => { + const { soClient, esClient, agentInHostedDoc, agentInRegularDoc, agentInRegularDoc2 } = + createClientMock(); + + const idsToUnenroll = [agentInRegularDoc._id, agentInHostedDoc._id, agentInRegularDoc2._id]; + + const unenrolledResponse = await unenrollAgents(soClient, esClient, { + agentIds: idsToUnenroll, + revoke: true, + force: true, + }); + + expect(unenrolledResponse.actionId).toBeDefined(); + + // calls ES update with correct values + const calledWith = esClient.bulk.mock.calls[0][0]; + const ids = (calledWith as estypes.BulkRequest)?.body + ?.filter((i: any) => i.update !== undefined) + .map((i: any) => i.update._id); + const docs = (calledWith as estypes.BulkRequest)?.body + ?.filter((i: any) => i.doc) + .map((i: any) => i.doc); + expect(ids).toEqual(idsToUnenroll); + for (const doc of docs!) { + expect(doc).toHaveProperty('unenrolled_at'); + } - const idsToUnenroll = [agentInRegularDoc._id, agentInHostedDoc._id, agentInRegularDoc2._id]; + const actionResults = esClient.bulk.mock.calls[1][0]; + const resultIds = (actionResults as estypes.BulkRequest)?.body + ?.filter((i: any) => i.agent_id) + .map((i: any) => i.agent_id); + expect(resultIds).toEqual(idsToUnenroll); - const unenrolledResponse = await unenrollAgents(soClient, esClient, { - agentIds: idsToUnenroll, - revoke: true, - force: true, + const action = esClient.create.mock.calls[0][0] as any; + expect(action.body.type).toEqual('FORCE_UNENROLL'); }); - - expect(unenrolledResponse.actionId).toBeDefined(); - - // calls ES update with correct values - const calledWith = esClient.bulk.mock.calls[0][0]; - const ids = (calledWith as estypes.BulkRequest)?.body - ?.filter((i: any) => i.update !== undefined) - .map((i: any) => i.update._id); - const docs = (calledWith as estypes.BulkRequest)?.body - ?.filter((i: any) => i.doc) - .map((i: any) => i.doc); - expect(ids).toEqual(idsToUnenroll); - for (const doc of docs!) { - expect(doc).toHaveProperty('unenrolled_at'); - } - - const actionResults = esClient.bulk.mock.calls[1][0]; - const resultIds = (actionResults as estypes.BulkRequest)?.body - ?.filter((i: any) => i.agent_id) - .map((i: any) => i.agent_id); - expect(resultIds).toEqual(idsToUnenroll); - - const action = esClient.create.mock.calls[0][0] as any; - expect(action.body.type).toEqual('FORCE_UNENROLL'); }); -}); -describe('invalidateAPIKeysForAgents', () => { - beforeEach(() => { - mockedInvalidateAPIKeys.mockReset(); - }); - it('revoke all the agents API keys', async () => { - await invalidateAPIKeysForAgents([ - { - id: 'agent1', - default_api_key_id: 'defaultApiKey1', - access_api_key_id: 'accessApiKey1', - default_api_key_history: [ - { - id: 'defaultApiKeyHistory1', - }, - { - id: 'defaultApiKeyHistory2', - }, - ], - outputs: { - output1: { - api_key_id: 'outputApiKey1', - to_retire_api_key_ids: [{ id: 'outputApiKeyRetire1' }, { id: 'outputApiKeyRetire2' }], - }, - output2: { - api_key_id: 'outputApiKey2', - }, - output3: { - api_key_id: 'outputApiKey3', - to_retire_api_key_ids: [ - // Somes Fleet Server agents don't have an id here (probably a bug) - { retired_at: 'foo' }, - ], + describe('invalidateAPIKeysForAgents', () => { + beforeEach(() => { + mockedInvalidateAPIKeys.mockReset(); + }); + it('revoke all the agents API keys', async () => { + await invalidateAPIKeysForAgents([ + { + id: 'agent1', + default_api_key_id: 'defaultApiKey1', + access_api_key_id: 'accessApiKey1', + default_api_key_history: [ + { + id: 'defaultApiKeyHistory1', + }, + { + id: 'defaultApiKeyHistory2', + }, + ], + outputs: { + output1: { + api_key_id: 'outputApiKey1', + to_retire_api_key_ids: [{ id: 'outputApiKeyRetire1' }, { id: 'outputApiKeyRetire2' }], + }, + output2: { + api_key_id: 'outputApiKey2', + }, + output3: { + api_key_id: 'outputApiKey3', + to_retire_api_key_ids: [ + // Somes Fleet Server agents don't have an id here (probably a bug) + { retired_at: 'foo' }, + ], + }, }, - }, - } as any, - ]); - - expect(mockedInvalidateAPIKeys).toBeCalledTimes(1); - expect(mockedInvalidateAPIKeys).toBeCalledWith([ - 'accessApiKey1', - 'defaultApiKey1', - 'defaultApiKeyHistory1', - 'defaultApiKeyHistory2', - 'outputApiKey1', - 'outputApiKeyRetire1', - 'outputApiKeyRetire2', - 'outputApiKey2', - 'outputApiKey3', - ]); + } as any, + ]); + + expect(mockedInvalidateAPIKeys).toBeCalledTimes(1); + expect(mockedInvalidateAPIKeys).toBeCalledWith([ + 'accessApiKey1', + 'defaultApiKey1', + 'defaultApiKeyHistory1', + 'defaultApiKeyHistory2', + 'outputApiKey1', + 'outputApiKeyRetire1', + 'outputApiKeyRetire2', + 'outputApiKey2', + 'outputApiKey3', + ]); + }); }); -}); -describe('isAgentUnenrolled', () => { - it('should return true if revoke and unenrolled_at set', () => { - expect(isAgentUnenrolled({ unenrolled_at: '2022-09-21' } as Agent, true)).toBe(true); - }); + describe('isAgentUnenrolled', () => { + it('should return true if revoke and unenrolled_at set', () => { + expect(isAgentUnenrolled({ unenrolled_at: '2022-09-21' } as Agent, true)).toBe(true); + }); - it('should return false if revoke and unenrolled_at null', () => { - expect( - isAgentUnenrolled({ unenrolled_at: null, unenrollment_started_at: '2022-09-21' } as any, true) - ).toBe(false); - }); + it('should return false if revoke and unenrolled_at null', () => { + expect( + isAgentUnenrolled( + { unenrolled_at: null, unenrollment_started_at: '2022-09-21' } as any, + true + ) + ).toBe(false); + }); - it('should return true if unenrolled_at set', () => { - expect(isAgentUnenrolled({ unenrolled_at: '2022-09-21' } as Agent)).toBe(true); - }); + it('should return true if unenrolled_at set', () => { + expect(isAgentUnenrolled({ unenrolled_at: '2022-09-21' } as Agent)).toBe(true); + }); - it('should return true if unenrollment_started_at set and unenrolled_at null', () => { - expect( - isAgentUnenrolled({ unenrolled_at: null, unenrollment_started_at: '2022-09-21' } as any) - ).toBe(true); - }); + it('should return true if unenrollment_started_at set and unenrolled_at null', () => { + expect( + isAgentUnenrolled({ unenrolled_at: null, unenrollment_started_at: '2022-09-21' } as any) + ).toBe(true); + }); - it('should return false if unenrollment_started_at null and unenrolled_at null', () => { - expect(isAgentUnenrolled({ unenrolled_at: null, unenrollment_started_at: null } as any)).toBe( - false - ); + it('should return false if unenrollment_started_at null and unenrolled_at null', () => { + expect(isAgentUnenrolled({ unenrolled_at: null, unenrollment_started_at: null } as any)).toBe( + false + ); + }); }); }); diff --git a/x-pack/plugins/fleet/server/services/agents/update_agent_tags.test.ts b/x-pack/plugins/fleet/server/services/agents/update_agent_tags.test.ts index ab407705007db..2dc239e01d82a 100644 --- a/x-pack/plugins/fleet/server/services/agents/update_agent_tags.test.ts +++ b/x-pack/plugins/fleet/server/services/agents/update_agent_tags.test.ts @@ -24,6 +24,7 @@ jest.mock('../app_context', () => { }, }; }); +jest.mock('../audit_logging'); jest.mock('../agent_policy', () => { return { diff --git a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.test.ts b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.test.ts new file mode 100644 index 0000000000000..bc58941e4c295 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.test.ts @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; + +import { ENROLLMENT_API_KEYS_INDEX } from '../../constants'; + +import { agentPolicyService } from '../agent_policy'; +import { auditLoggingService } from '../audit_logging'; +import { appContextService } from '../app_context'; + +import { deleteEnrollmentApiKey, generateEnrollmentAPIKey } from './enrollment_api_key'; + +jest.mock('../audit_logging'); +jest.mock('../agent_policy'); +jest.mock('../app_context'); + +jest.mock('uuid', () => { + return { + v4: () => 'mock-uuid', + }; +}); + +const mockedAgentPolicyService = agentPolicyService as jest.Mocked; +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; +const mockedAppContextService = appContextService as jest.Mocked; + +describe('enrollment api keys', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + describe('generateEnrollmentAPIKey', () => { + it('should call audit logger', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + esClient.create.mockResolvedValue({ + _id: 'test-enrollment-api-key-id', + } as any); + + esClient.security.createApiKey.mockResolvedValue({ + api_key: 'test-api-key-value', + id: 'test-api-key-id', + } as any); + + mockedAgentPolicyService.get.mockResolvedValue({ + id: 'test-agent-policy', + } as any); + + await generateEnrollmentAPIKey(soClient, esClient, { + name: 'test-api-key', + expiration: '7d', + agentPolicyId: 'test-agent-policy', + forceRecreate: true, + }); + + expect(mockedAuditLoggingService.writeCustomAuditLog).toHaveBeenCalledWith({ + message: + 'User creating enrollment API key [name=test-api-key (mock-uuid)] [policy_id=test-agent-policy]', + }); + }); + }); + + describe('deleteEnrollmentApiKey', () => { + it('should call audit logger', async () => { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + esClient.update.mockResolvedValue({} as any); + + esClient.get.mockResolvedValue({ + _id: 'test-id', + _index: ENROLLMENT_API_KEYS_INDEX, + _source: { + active: true, + created_at: new Date().toISOString(), + api_key_id: 'test-enrollment-api-key-id', + }, + found: true, + }); + + mockedAppContextService.getSecurity.mockReturnValue({ + authc: { + apiKeys: { + invalidateAsInternalUser: jest.fn().mockResolvedValue({}), + }, + }, + } as any); + + await deleteEnrollmentApiKey(esClient, 'test-enrollment-api-key-id'); + + expect(auditLoggingService.writeCustomAuditLog).toHaveBeenCalledWith({ + message: + 'User deleting enrollment API key [id=test-id] [api_key_id=test-enrollment-api-key-id]', + }); + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts index dadb41a27aac1..ac7087c95296a 100644 --- a/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts +++ b/x-pack/plugins/fleet/server/services/api_keys/enrollment_api_key.ts @@ -21,6 +21,8 @@ import { ENROLLMENT_API_KEYS_INDEX } from '../../constants'; import { agentPolicyService } from '../agent_policy'; import { escapeSearchQueryPhrase } from '../saved_object'; +import { auditLoggingService } from '../audit_logging'; + import { invalidateAPIKeys } from './security'; const uuidRegex = @@ -106,6 +108,10 @@ export async function deleteEnrollmentApiKey( ) { const enrollmentApiKey = await getEnrollmentAPIKey(esClient, id); + auditLoggingService.writeCustomAuditLog({ + message: `User deleting enrollment API key [id=${enrollmentApiKey.id}] [api_key_id=${enrollmentApiKey.api_key_id}]`, + }); + await invalidateAPIKeys([enrollmentApiKey.api_key_id]); if (forceDelete) { @@ -208,6 +214,10 @@ export async function generateEnrollmentAPIKey( const name = providedKeyName ? `${providedKeyName} (${id})` : id; + auditLoggingService.writeCustomAuditLog({ + message: `User creating enrollment API key [name=${name}] [policy_id=${agentPolicyId}]`, + }); + const key = await esClient.security .createApiKey({ body: { diff --git a/x-pack/plugins/fleet/server/services/app_context.ts b/x-pack/plugins/fleet/server/services/app_context.ts index 71c09621c59a0..3ea1f4e464e4a 100644 --- a/x-pack/plugins/fleet/server/services/app_context.ts +++ b/x-pack/plugins/fleet/server/services/app_context.ts @@ -120,6 +120,10 @@ class AppContextService { return this.securityStart!; } + public getSecuritySetup() { + return this.securitySetup!; + } + public getSecurityLicense() { return this.securitySetup!.license; } diff --git a/x-pack/plugins/fleet/server/services/audit_logging.ts b/x-pack/plugins/fleet/server/services/audit_logging.ts new file mode 100644 index 0000000000000..a9549ade160d1 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/audit_logging.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { AuditLogger } from '@kbn/security-plugin/server'; + +import { appContextService } from './app_context'; +import { getRequestStore } from './request_store'; + +class AuditLoggingService { + /** + * Write a custom audit log record. If a current request is available, the log will include + * user/session data. If not, an unscoped audit logger will be used. + */ + public writeCustomAuditLog(...args: Parameters) { + const securitySetup = appContextService.getSecuritySetup(); + let auditLogger: AuditLogger | undefined; + + const request = getRequestStore().getStore(); + + if (request) { + auditLogger = securitySetup.audit.asScoped(request); + } else { + auditLogger = securitySetup.audit.withoutRequest; + } + + auditLogger.log(...args); + } + + /** + * Helper method for writing saved object related audit logs. Since Fleet + * uses an internal SO client to support its custom RBAC model around Fleet/Integrations + * permissions, we need to implement our own audit logging for saved objects that use the + * internal client. This helper reduces the boilerplate around audit logging in those cases. + * + * @example + * ```ts + * auditLoggingService.writeCustomSoAuditLog({ + * action: 'find', + * id: 'some-id-123', + * savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE + * }); + * ``` + */ + public writeCustomSoAuditLog({ + action, + id, + savedObjectType, + }: { + action: 'find' | 'get' | 'create' | 'update' | 'delete'; + id: string; + savedObjectType: string; + }) { + this.writeCustomAuditLog({ + message: `User ${ + action === 'find' || action === 'get' ? 'has accessed' : 'is accessing' + } ${savedObjectType} [id=${id}]`, + event: { + action: `saved_object_${action}`, + category: ['database'], + outcome: 'unknown', + type: ['access'], + }, + kibana: { + saved_object: { + id, + type: savedObjectType, + }, + }, + }); + } +} + +export const auditLoggingService = new AuditLoggingService(); diff --git a/x-pack/plugins/fleet/server/services/download_source.test.ts b/x-pack/plugins/fleet/server/services/download_source.test.ts index 209a2f0313a0c..500a879cf2c82 100644 --- a/x-pack/plugins/fleet/server/services/download_source.test.ts +++ b/x-pack/plugins/fleet/server/services/download_source.test.ts @@ -7,6 +7,8 @@ import { savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { securityMock } from '@kbn/security-plugin/server/mocks'; + import type { DownloadSourceAttributes } from '../types'; import { DOWNLOAD_SOURCE_SAVED_OBJECT_TYPE } from '../constants'; @@ -19,6 +21,10 @@ jest.mock('./app_context'); jest.mock('./agent_policy'); const mockedAppContextService = appContextService as jest.Mocked; +mockedAppContextService.getSecuritySetup.mockImplementation(() => ({ + ...securityMock.createSetup(), +})); + const mockedAgentPolicyService = agentPolicyService as jest.Mocked; function mockDownloadSourceSO(id: string, attributes: any = {}) { diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts index e3d959682727e..a437d5520ffd7 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/default_settings.test.ts @@ -8,6 +8,8 @@ import { loggerMock } from '@kbn/logging-mocks'; import type { Logger } from '@kbn/core/server'; +import { securityMock } from '@kbn/security-plugin/server/mocks'; + import { appContextService } from '../../../app_context'; import { buildDefaultSettings } from './default_settings'; @@ -15,6 +17,10 @@ import { buildDefaultSettings } from './default_settings'; jest.mock('../../../app_context'); const mockedAppContextService = appContextService as jest.Mocked; +mockedAppContextService.getSecuritySetup.mockImplementation(() => ({ + ...securityMock.createSetup(), +})); + let mockedLogger: jest.Mocked; describe('buildDefaultSettings', () => { beforeEach(() => { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts index 596ad94a067d8..7da483d0794a2 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/_install_package.ts @@ -48,6 +48,8 @@ import { saveArchiveEntries } from '../archive/storage'; import { ConcurrentInstallOperationError } from '../../../errors'; import { appContextService, packagePolicyService } from '../..'; +import { auditLoggingService } from '../../audit_logging'; + import { createInstallation, restartInstallation, @@ -289,6 +291,12 @@ export async function _installPackage({ }) ); + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + const updatedPackage = await withPackageSpan('Update install status', () => savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { version: pkgVersion, diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts index 9a03fd720bc10..e4cd54f90973d 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.test.ts @@ -5,8 +5,6 @@ * 2.0. */ -jest.mock('../registry'); - import type { SavedObjectsClientContract, SavedObjectsFindResult } from '@kbn/core/server'; import { SavedObjectsErrorHelpers } from '@kbn/core/server'; @@ -15,25 +13,39 @@ import { savedObjectsClientMock } from '@kbn/core/server/mocks'; import { PACKAGES_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../../../../common'; import type { PackagePolicySOAttributes, RegistryPackage } from '../../../../common/types'; -import * as Registry from '../registry'; - import { createAppContextStartContractMock } from '../../../mocks'; import { appContextService } from '../../app_context'; - import { PackageNotFoundError } from '../../../errors'; import { getSettings } from '../../settings'; +import { auditLoggingService } from '../../audit_logging'; + +import * as Registry from '../registry'; import { getPackageInfo, getPackages, getPackageUsageStats } from './get'; +jest.mock('../registry'); +jest.mock('../../settings'); +jest.mock('../../audit_logging'); + const MockRegistry = jest.mocked(Registry); -jest.mock('../../settings'); +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; const mockGetSettings = getSettings as jest.Mock; mockGetSettings.mockResolvedValue({ prerelease_integrations_enabled: true }); describe('When using EPM `get` services', () => { + beforeEach(() => { + const mockContract = createAppContextStartContractMock(); + appContextService.start(mockContract); + }); + + afterEach(() => { + appContextService.stop(); + jest.clearAllMocks(); + }); + describe('and invoking getPackageUsageStats()', () => { let soClient: jest.Mocked; @@ -188,9 +200,6 @@ describe('When using EPM `get` services', () => { describe('getPackages', () => { beforeEach(() => { - const mockContract = createAppContextStartContractMock(); - appContextService.start(mockContract); - jest.clearAllMocks(); MockRegistry.fetchList.mockResolvedValue([ { name: 'nginx', @@ -291,6 +300,45 @@ owner: elastic`, { id: 'nginx', name: 'nginx', title: 'Nginx', version: '1.0.0' }, ]); }); + + it('should call audit logger', async () => { + const soClient = savedObjectsClientMock.create(); + + soClient.find.mockResolvedValue({ + saved_objects: [ + { + id: 'elasticsearch', + attributes: { + name: 'elasticsearch', + version: '0.0.1', + install_source: 'upload', + install_version: '0.0.1', + }, + score: 0, + type: PACKAGES_SAVED_OBJECT_TYPE, + references: [], + }, + ], + total: 1, + per_page: 10, + page: 1, + }); + + soClient.get.mockResolvedValue({ + id: 'elasticsearch', + attributes: {}, + references: [], + type: PACKAGES_SAVED_OBJECT_TYPE, + }); + + await getPackages({ savedObjectsClient: soClient }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'get', + id: 'elasticsearch', + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + }); }); describe('getPackageInfo', () => { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index 0cfcee18900c8..351128af7f4bf 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -41,6 +41,8 @@ import { getEsPackage } from '../archive/storage'; import { getArchivePackage } from '../archive'; import { normalizeKuery } from '../../saved_object'; +import { auditLoggingService } from '../../audit_logging'; + import { createInstallableFrom } from '.'; export type { SearchParams } from '../registry'; @@ -114,6 +116,14 @@ export async function getPackages( .concat(uploadedPackagesNotInRegistry as Installable) .sort(sortByName); + for (const pkg of packageList) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'get', + id: pkg.id, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + } + if (!excludeInstallStatus) { return packageList; } @@ -154,18 +164,39 @@ export async function getLimitedPackages(options: { }); }) ); - return installedPackagesInfo.filter(isPackageLimited).map((pkgInfo) => pkgInfo.name); + + const packages = installedPackagesInfo.filter(isPackageLimited).map((pkgInfo) => pkgInfo.name); + + for (const pkg of installedPackages) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'find', + id: pkg.id, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + } + + return packages; } export async function getPackageSavedObjects( savedObjectsClient: SavedObjectsClientContract, options?: Omit ) { - return savedObjectsClient.find({ + const result = await savedObjectsClient.find({ ...(options || {}), type: PACKAGES_SAVED_OBJECT_TYPE, perPage: SO_SEARCH_LIMIT, }); + + for (const savedObject of result.saved_objects) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'find', + id: savedObject.id, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + } + + return result; } export const getInstallations = getPackageSavedObjects; @@ -267,6 +298,14 @@ export const getPackageUsageStats = async ({ filter, }); + for (const packagePolicy of packagePolicies.saved_objects) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'find', + id: packagePolicy.id, + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + } + for (let index = 0, total = packagePolicies.saved_objects.length; index < total; index++) { agentPolicyCount.add(packagePolicies.saved_objects[index].attributes.policy_id); } @@ -375,10 +414,24 @@ export async function getInstallationObject(options: { logger?: Logger; }) { const { savedObjectsClient, pkgName, logger } = options; - return savedObjectsClient.get(PACKAGES_SAVED_OBJECT_TYPE, pkgName).catch((e) => { - logger?.error(e); - return undefined; + const installation = await savedObjectsClient + .get(PACKAGES_SAVED_OBJECT_TYPE, pkgName) + .catch((e) => { + logger?.error(e); + return undefined; + }); + + if (!installation) { + return; + } + + auditLoggingService.writeCustomSoAuditLog({ + action: 'find', + id: installation.id, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, }); + + return installation; } export async function getInstallationObjects(options: { @@ -390,7 +443,17 @@ export async function getInstallationObjects(options: { pkgNames.map((pkgName) => ({ id: pkgName, type: PACKAGES_SAVED_OBJECT_TYPE })) ); - return res.saved_objects.filter((so) => so?.attributes); + const installations = res.saved_objects.filter((so) => so?.attributes); + + for (const installation of installations) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'find', + id: installation.id, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + } + + return installations; } export async function getInstallation(options: { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts index 5466c61313d97..99c711518ff2e 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.test.ts @@ -6,22 +6,24 @@ */ import { savedObjectsClientMock } from '@kbn/core/server/mocks'; - -import type { ElasticsearchClient } from '@kbn/core/server'; - import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; +import type { ElasticsearchClient } from '@kbn/core/server'; -import * as Registry from '../registry'; +import type { InstallablePackage } from '../../../../common'; +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../common'; import { sendTelemetryEvents } from '../../upgrade_sender'; - import { licenseService } from '../../license'; +import { auditLoggingService } from '../../audit_logging'; + +import * as Registry from '../registry'; -import { installPackage } from './install'; +import { createInstallation, installPackage } from './install'; import * as install from './_install_package'; -import * as obj from '.'; import { getBundledPackages } from './bundled_packages'; +import * as obj from '.'; + jest.mock('../../app_context', () => { return { appContextService: { @@ -66,8 +68,59 @@ jest.mock('../archive', () => { deleteVerificationResult: jest.fn(), }; }); +jest.mock('../../audit_logging'); const mockGetBundledPackages = getBundledPackages as jest.MockedFunction; +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; + +describe('createInstallation', () => { + const soClient = savedObjectsClientMock.create(); + + const packageInfo: InstallablePackage = { + name: 'test-package', + version: '1.0.0', + format_version: '1.0.0', + title: 'Test Package', + description: 'A package for testing', + owner: { + github: 'elastic', + }, + }; + + describe('installSource: registry', () => { + it('should call audit logger', async () => { + await createInstallation({ + savedObjectsClient: soClient, + packageInfo, + installSource: 'registry', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'create', + id: 'test-package', + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + }); + }); + + describe('installSource: upload', () => { + it('should call audit logger', async () => { + await createInstallation({ + savedObjectsClient: soClient, + packageInfo, + installSource: 'upload', + spaceId: DEFAULT_SPACE_ID, + }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'create', + id: 'test-package', + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + }); + }); +}); describe('install', () => { beforeEach(() => { diff --git a/x-pack/plugins/fleet/server/services/epm/packages/install.ts b/x-pack/plugins/fleet/server/services/epm/packages/install.ts index 0b5b86407100e..a9b9afcf71ed6 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/install.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/install.ts @@ -68,6 +68,8 @@ import { prepareToInstallPipelines } from '../elasticsearch/ingest_pipeline'; import { prepareToInstallTemplates } from '../elasticsearch/template/install'; +import { auditLoggingService } from '../../audit_logging'; + import { formatVerificationResultForSO } from './package_verification'; import { getInstallation, getInstallationObject } from '.'; @@ -670,6 +672,12 @@ export const updateVersion = async ( pkgName: string, pkgVersion: string ) => { + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + return savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { version: pkgVersion, }); @@ -684,6 +692,12 @@ export const updateInstallStatus = async ({ pkgName: string; status: EpmPackageInstallStatus; }) => { + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + return savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, { install_status: status, }); @@ -713,6 +727,12 @@ export async function restartInstallation(options: { }; } + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, savedObjectUpdate); } @@ -757,6 +777,12 @@ export async function createInstallation(options: { savedObject = { ...savedObject, ...formatVerificationResultForSO(verificationResult) }; } + auditLoggingService.writeCustomSoAuditLog({ + action: 'create', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + const created = await savedObjectsClient.create( PACKAGES_SAVED_OBJECT_TYPE, savedObject, @@ -771,6 +797,12 @@ export const saveKibanaAssetsRefs = async ( pkgName: string, kibanaAssets: Record ) => { + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + const assetRefs = Object.values(kibanaAssets).flat().map(toAssetReference); // Because Kibana assets are installed in parallel with ES assets with refresh: false, we almost always run into an // issue that causes a conflict error due to this issue: https://github.com/elastic/kibana/issues/126240. This is safe @@ -829,6 +861,12 @@ export const updateEsAssetReferences = async ( ({ type, id }) => `${type}-${id}` ); + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + const { attributes: { installed_es: updatedAssets }, } = @@ -864,7 +902,13 @@ export const optimisticallyAddEsAssetReferences = async ( assetsToAdd: EsAssetReference[] ): Promise => { const addEsAssets = async () => { + // TODO: Should this be replaced by a `get()` call from epm/get.ts? const so = await savedObjectsClient.get(PACKAGES_SAVED_OBJECT_TYPE, pkgName); + auditLoggingService.writeCustomSoAuditLog({ + action: 'get', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); const installedEs = so.attributes.installed_es ?? []; @@ -873,6 +917,12 @@ export const optimisticallyAddEsAssetReferences = async ( ({ type, id }) => `${type}-${id}` ); + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + const { attributes: { installed_es: updatedAssets }, } = await savedObjectsClient.update( diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.test.ts index bfc03c1171004..ff03dbf5fd422 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.test.ts @@ -5,7 +5,10 @@ * 2.0. */ +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../common'; + import { packagePolicyService } from '../..'; +import { auditLoggingService } from '../../audit_logging'; import { removeInstallation } from './remove'; @@ -23,7 +26,9 @@ jest.mock('../..', () => { }, }; }); +jest.mock('../../audit_logging'); +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; const mockPackagePolicyService = packagePolicyService as jest.Mocked; describe('removeInstallation', () => { @@ -66,4 +71,20 @@ describe('removeInstallation', () => { `unable to remove package with existing package policy(s) in use by agent(s)` ); }); + + it('should call audit logger', async () => { + await removeInstallation({ + savedObjectsClient: soClientMock, + pkgName: 'system', + pkgVersion: '1.0.0', + esClient: esClientMock, + force: true, + }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'delete', + id: 'system', + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + }); }); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts index 4a4183965913a..0485662b584c7 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/remove.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/remove.ts @@ -41,6 +41,8 @@ import { deletePackageCache } from '../archive'; import { deleteIlms } from '../elasticsearch/datastream_ilm/remove'; import { removeArchiveEntries } from '../archive/storage'; +import { auditLoggingService } from '../../audit_logging'; + import { getInstallation, kibanaSavedObjectTypes } from '.'; export async function removeInstallation(options: { @@ -85,6 +87,11 @@ export async function removeInstallation(options: { // Delete the manager saved object with references to the asset objects // could also update with [] or some other state + auditLoggingService.writeCustomSoAuditLog({ + action: 'delete', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); await savedObjectsClient.delete(PACKAGES_SAVED_OBJECT_TYPE, pkgName); // delete the index patterns if no packages are installed @@ -118,6 +125,14 @@ async function deleteKibanaAssets( { namespace } ); + for (const { saved_object: savedObject } of resolvedObjects) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'get', + id: savedObject.id, + savedObjectType: savedObject.type, + }); + } + const foundObjects = resolvedObjects.filter( ({ saved_object: savedObject }) => savedObject?.error?.statusCode !== 404 ); @@ -126,6 +141,14 @@ async function deleteKibanaAssets( // we filter these out before calling delete const assetsToDelete = foundObjects.map(({ saved_object: { id, type } }) => ({ id, type })); + for (const asset of assetsToDelete) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'delete', + id: asset.id, + savedObjectType: asset.type, + }); + } + return savedObjectsClient.bulkDelete(assetsToDelete, { namespace }); } diff --git a/x-pack/plugins/fleet/server/services/epm/packages/update.test.ts b/x-pack/plugins/fleet/server/services/epm/packages/update.test.ts new file mode 100644 index 0000000000000..123aed4d74414 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/epm/packages/update.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; + +import { createAppContextStartContractMock } from '../../../mocks'; + +import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../common'; +import { appContextService } from '../../app_context'; + +import { auditLoggingService } from '../../audit_logging'; + +import { updatePackage } from './update'; +import { getPackageInfo, getInstallationObject } from './get'; + +jest.mock('./get'); +jest.mock('../../audit_logging'); + +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; +const mockGetPackageInfo = getPackageInfo as jest.MockedFunction; +const mockGetInstallationObject = getInstallationObject as jest.MockedFunction< + typeof getInstallationObject +>; + +describe('updatePackage', () => { + let mockContract: ReturnType; + + beforeEach(() => { + mockContract = createAppContextStartContractMock(); + appContextService.start(mockContract); + }); + + afterEach(() => { + appContextService.stop(); + }); + + it('should call audit logger', async () => { + const savedObjectsClient = savedObjectsClientMock.create(); + + mockGetPackageInfo.mockResolvedValueOnce({ + name: 'test-package', + title: 'Test package', + description: 'Test package', + format_version: '1.0.0', + version: '1.0.0', + latestVersion: '1.0.0', + owner: { + github: 'elastic', + }, + assets: { + elasticsearch: {}, + kibana: {}, + }, + } as any); + + mockGetInstallationObject.mockResolvedValueOnce({ + id: 'test-package', + attributes: {} as any, + references: [], + type: PACKAGES_SAVED_OBJECT_TYPE, + }); + + await updatePackage({ + savedObjectsClient, + pkgName: 'test-package', + keepPoliciesUpToDate: true, + }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'update', + id: 'test-package', + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/epm/packages/update.ts b/x-pack/plugins/fleet/server/services/epm/packages/update.ts index aa75ab3eae1dd..3f3633708b94c 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/update.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/update.ts @@ -14,6 +14,8 @@ import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import type { Installation, UpdatePackageRequestSchema } from '../../../types'; import { FleetError } from '../../../errors'; +import { auditLoggingService } from '../../audit_logging'; + import { getInstallationObject, getPackageInfo } from './get'; export async function updatePackage( @@ -30,6 +32,12 @@ export async function updatePackage( throw new FleetError(`package ${pkgName} is not installed`); } + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: installedPackage.id, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, installedPackage.id, { keep_policies_up_to_date: keepPoliciesUpToDate ?? false, }); @@ -51,6 +59,12 @@ export async function updateDatastreamExperimentalFeatures( features: Record; }> ) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: pkgName, + savedObjectType: PACKAGES_SAVED_OBJECT_TYPE, + }); + await savedObjectsClient.update( PACKAGES_SAVED_OBJECT_TYPE, pkgName, diff --git a/x-pack/plugins/fleet/server/services/index.ts b/x-pack/plugins/fleet/server/services/index.ts index c2abd872d5df9..69f5889bcd210 100644 --- a/x-pack/plugins/fleet/server/services/index.ts +++ b/x-pack/plugins/fleet/server/services/index.ts @@ -50,6 +50,7 @@ export { dataStreamService } from './data_streams'; // Plugin services export { appContextService } from './app_context'; export { licenseService } from './license'; +export { auditLoggingService } from './audit_logging'; // Artifacts services export * from './artifacts'; diff --git a/x-pack/plugins/fleet/server/services/managed_package_policies.test.ts b/x-pack/plugins/fleet/server/services/managed_package_policies.test.ts index 4110438f8f3fd..0882fff0ec34f 100644 --- a/x-pack/plugins/fleet/server/services/managed_package_policies.test.ts +++ b/x-pack/plugins/fleet/server/services/managed_package_policies.test.ts @@ -23,6 +23,7 @@ jest.mock('./app_context', () => { }, }; }); +jest.mock('./audit_logging'); describe('upgradeManagedPackagePolicies', () => { afterEach(() => { diff --git a/x-pack/plugins/fleet/server/services/output.test.ts b/x-pack/plugins/fleet/server/services/output.test.ts index 9c97cbe153284..11d6bf8936ee4 100644 --- a/x-pack/plugins/fleet/server/services/output.test.ts +++ b/x-pack/plugins/fleet/server/services/output.test.ts @@ -7,16 +7,27 @@ import { savedObjectsClientMock, elasticsearchServiceMock } from '@kbn/core/server/mocks'; +import { securityMock } from '@kbn/security-plugin/server/mocks'; + import type { OutputSOAttributes } from '../types'; +import { OUTPUT_SAVED_OBJECT_TYPE } from '../constants'; + import { outputService, outputIdToUuid } from './output'; import { appContextService } from './app_context'; import { agentPolicyService } from './agent_policy'; +import { auditLoggingService } from './audit_logging'; jest.mock('./app_context'); jest.mock('./agent_policy'); +jest.mock('./audit_logging'); +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; const mockedAppContextService = appContextService as jest.Mocked; +mockedAppContextService.getSecuritySetup.mockImplementation(() => ({ + ...securityMock.createSetup(), +})); + const mockedAgentPolicyService = agentPolicyService as jest.Mocked; const CLOUD_ID = @@ -177,6 +188,7 @@ describe('Output Service', () => { mockedAgentPolicyService.removeOutputFromAll.mockReset(); mockedAppContextService.getInternalUserSOClient.mockReset(); mockedAppContextService.getEncryptedSavedObjectsSetup.mockReset(); + mockedAuditLoggingService.writeCustomSoAuditLog.mockReset(); }); describe('create', () => { it('work with a predefined id', async () => { @@ -494,6 +506,28 @@ describe('Output Service', () => { { force: true } ); }); + + it('should call audit logger', async () => { + const soClient = getMockedSoClient(); + + await outputService.create( + soClient, + esClientMock, + { + is_default: false, + is_default_monitoring: true, + name: 'Test', + type: 'elasticsearch', + }, + { id: 'output-test' } + ); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'create', + id: outputIdToUuid('output-test'), + savedObjectType: OUTPUT_SAVED_OBJECT_TYPE, + }); + }); }); describe('update', () => { @@ -823,6 +857,20 @@ describe('Output Service', () => { 'Logstash output cannot be used with Fleet Server integration in fleet server policy. Please create a new ElasticSearch output.' ); }); + + it('should call audit logger', async () => { + const soClient = getMockedSoClient({ defaultOutputId: 'existing-es-output' }); + + await outputService.update(soClient, esClientMock, 'existing-es-output', { + hosts: ['new-host:443'], + }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'update', + id: outputIdToUuid('existing-es-output'), + savedObjectType: OUTPUT_SAVED_OBJECT_TYPE, + }); + }); }); describe('delete', () => { @@ -853,6 +901,17 @@ describe('Output Service', () => { expect(mockedAgentPolicyService.removeOutputFromAll).toBeCalled(); expect(soClient.delete).toBeCalled(); }); + + it('should call audit logger', async () => { + const soClient = getMockedSoClient(); + await outputService.delete(soClient, 'existing-es-output'); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'delete', + id: outputIdToUuid('existing-es-output'), + savedObjectType: OUTPUT_SAVED_OBJECT_TYPE, + }); + }); }); describe('get', () => { @@ -864,6 +923,17 @@ describe('Output Service', () => { expect(output.id).toEqual('output-test'); }); + + it('should call audit logger', async () => { + const soClient = getMockedSoClient(); + await outputService.get(soClient, 'existing-es-output'); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'get', + id: outputIdToUuid('existing-es-output'), + savedObjectType: OUTPUT_SAVED_OBJECT_TYPE, + }); + }); }); describe('getDefaultDataOutputId', () => { diff --git a/x-pack/plugins/fleet/server/services/output.ts b/x-pack/plugins/fleet/server/services/output.ts index a8068681ac941..eebf714716d19 100644 --- a/x-pack/plugins/fleet/server/services/output.ts +++ b/x-pack/plugins/fleet/server/services/output.ts @@ -4,16 +4,17 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { v5 as uuidv5 } from 'uuid'; +import { omit } from 'lodash'; +import { safeLoad } from 'js-yaml'; +import { SavedObjectsUtils } from '@kbn/core/server'; import type { KibanaRequest, SavedObject, SavedObjectsClientContract, ElasticsearchClient, } from '@kbn/core/server'; -import { v5 as uuidv5 } from 'uuid'; -import { omit } from 'lodash'; -import { safeLoad } from 'js-yaml'; import type { NewOutput, Output, OutputSOAttributes, AgentPolicy } from '../types'; import { @@ -33,6 +34,7 @@ import { import { agentPolicyService } from './agent_policy'; import { appContextService } from './app_context'; import { escapeSearchQueryPhrase } from './saved_object'; +import { auditLoggingService } from './audit_logging'; type Nullable = { [P in keyof T]: T[P] | null }; @@ -198,19 +200,39 @@ class OutputService { } private async _getDefaultDataOutputsSO(soClient: SavedObjectsClientContract) { - return await this.encryptedSoClient.find({ + const outputs = await this.encryptedSoClient.find({ type: OUTPUT_SAVED_OBJECT_TYPE, searchFields: ['is_default'], search: 'true', }); + + for (const output of outputs.saved_objects) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'get', + id: output.id, + savedObjectType: OUTPUT_SAVED_OBJECT_TYPE, + }); + } + + return outputs; } private async _getDefaultMonitoringOutputsSO(soClient: SavedObjectsClientContract) { - return await this.encryptedSoClient.find({ + const outputs = await this.encryptedSoClient.find({ type: OUTPUT_SAVED_OBJECT_TYPE, searchFields: ['is_default_monitoring'], search: 'true', }); + + for (const output of outputs.saved_objects) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'get', + id: output.id, + savedObjectType: OUTPUT_SAVED_OBJECT_TYPE, + }); + } + + return outputs; } public async ensureDefaultOutput( @@ -360,9 +382,17 @@ class OutputService { } } + const id = options?.id ? outputIdToUuid(options.id) : SavedObjectsUtils.generateId(); + + auditLoggingService.writeCustomSoAuditLog({ + action: 'create', + id, + savedObjectType: OUTPUT_SAVED_OBJECT_TYPE, + }); + const newSo = await this.encryptedSoClient.create(SAVED_OBJECT_TYPE, data, { overwrite: options?.overwrite || options?.fromPreconfiguration, - id: options?.id ? outputIdToUuid(options.id) : undefined, + id, }); return outputSavedObjectToOutput(newSo); @@ -400,6 +430,14 @@ class OutputService { sortOrder: 'desc', }); + for (const output of outputs.saved_objects) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'get', + id: output.id, + savedObjectType: OUTPUT_SAVED_OBJECT_TYPE, + }); + } + return { items: outputs.saved_objects.map(outputSavedObjectToOutput), total: outputs.total, @@ -417,6 +455,14 @@ class OutputService { search: escapeSearchQueryPhrase(proxyId), }); + for (const output of outputs.saved_objects) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'get', + id: output.id, + savedObjectType: OUTPUT_SAVED_OBJECT_TYPE, + }); + } + return { items: outputs.saved_objects.map(outputSavedObjectToOutput), total: outputs.total, @@ -431,6 +477,12 @@ class OutputService { outputIdToUuid(id) ); + auditLoggingService.writeCustomSoAuditLog({ + action: 'get', + id: outputSO.id, + savedObjectType: OUTPUT_SAVED_OBJECT_TYPE, + }); + if (outputSO.error) { throw new Error(outputSO.error.message); } @@ -467,6 +519,12 @@ class OutputService { id ); + auditLoggingService.writeCustomSoAuditLog({ + action: 'delete', + id: outputIdToUuid(id), + savedObjectType: OUTPUT_SAVED_OBJECT_TYPE, + }); + return this.encryptedSoClient.delete(SAVED_OBJECT_TYPE, outputIdToUuid(id)); } @@ -555,6 +613,12 @@ class OutputService { } } + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: outputIdToUuid(id), + savedObjectType: OUTPUT_SAVED_OBJECT_TYPE, + }); + const outputSO = await this.encryptedSoClient.update>( SAVED_OBJECT_TYPE, outputIdToUuid(id), diff --git a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts index af62a34d73bc2..4fb8073ccbf00 100644 --- a/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policies/experimental_datastream_features.test.ts @@ -7,8 +7,10 @@ import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; import { savedObjectsClientMock } from '@kbn/core-saved-objects-api-server-mocks'; +import { securityMock } from '@kbn/security-plugin/server/mocks'; import type { NewPackagePolicy, PackagePolicy } from '../../types'; +import { appContextService } from '../app_context'; import { updateCurrentWriteIndices } from '../epm/elasticsearch/template/template'; import { getInstallation } from '../epm/packages'; @@ -33,6 +35,11 @@ jest.mock('../epm/packages', () => { }); jest.mock('../app_context'); +const mockedAppContextService = appContextService as jest.Mocked; +mockedAppContextService.getSecuritySetup.mockImplementation(() => ({ + ...securityMock.createSetup(), +})); + jest.mock('../epm/elasticsearch/template/template'); const mockGetInstallation = getInstallation as jest.Mock; diff --git a/x-pack/plugins/fleet/server/services/package_policy.test.ts b/x-pack/plugins/fleet/server/services/package_policy.test.ts index 343b6129d0a1a..314dc6c5f1bd1 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.test.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.test.ts @@ -14,7 +14,6 @@ import { import { produce } from 'immer'; import type { KibanaRequest, - SavedObjectsClient, SavedObjectsClientContract, SavedObjectsUpdateResponse, } from '@kbn/core/server'; @@ -23,7 +22,6 @@ import { SavedObjectsErrorHelpers } from '@kbn/core/server'; import type { PackageInfo, PackagePolicySOAttributes, - AgentPolicySOAttributes, PostPackagePolicyPostDeleteCallback, RegistryDataStream, PackagePolicyInputStream, @@ -49,6 +47,8 @@ import { packageToPackagePolicy } from '../../common/services'; import { FleetError, PackagePolicyIneligibleForUpgradeError } from '../errors'; +import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../constants'; + import { preconfigurePackageInputs, updatePackageInputs, @@ -61,6 +61,8 @@ import { appContextService } from './app_context'; import { getPackageInfo } from './epm/packages'; import { sendTelemetryEvents } from './upgrade_sender'; +import { auditLoggingService } from './audit_logging'; +import { agentPolicyService } from './agent_policy'; const mockedSendTelemetryEvents = sendTelemetryEvents as jest.MockedFunction< typeof sendTelemetryEvents @@ -176,26 +178,8 @@ jest.mock('../../common/services/package_to_package_policy', () => ({ jest.mock('./epm/registry'); -jest.mock('./agent_policy', () => { - return { - agentPolicyService: { - get: async (soClient: SavedObjectsClient, id: string) => { - const agentPolicySO = await soClient.get( - 'ingest-agent-policies', - id - ); - if (!agentPolicySO) { - return null; - } - const agentPolicy = { id: agentPolicySO.id, ...agentPolicySO.attributes }; - agentPolicy.package_policies = []; - return agentPolicy; - }, - bumpRevision: () => {}, - getDefaultAgentPolicyId: () => Promise.resolve('1'), - }, - }; -}); +jest.mock('./agent_policy'); +const mockAgentPolicyService = agentPolicyService as jest.Mocked; jest.mock('./epm/packages/cleanup', () => { return { @@ -209,9 +193,238 @@ jest.mock('./upgrade_sender', () => { }; }); +jest.mock('./audit_logging'); +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; + type CombinedExternalCallback = PutPackagePolicyUpdateCallback | PostPackagePolicyCreateCallback; +const mockAgentPolicyGet = () => { + mockAgentPolicyService.get.mockImplementation( + (_soClient: SavedObjectsClientContract, id: string, _force = false, _errorMessage?: string) => { + return Promise.resolve({ + id, + name: 'Test Agent Policy', + namespace: 'test', + status: 'active', + is_managed: false, + updated_at: new Date().toISOString(), + updated_by: 'test', + revision: 1, + }); + } + ); +}; + describe('Package policy service', () => { + beforeEach(() => { + appContextService.start(createAppContextStartContractMock()); + }); + + afterEach(() => { + appContextService.stop(); + + // `jest.resetAllMocks` breaks a ton of tests in this file 🤷‍♂️ + mockAgentPolicyService.get.mockReset(); + mockedAuditLoggingService.writeCustomSoAuditLog.mockReset(); + }); + + describe('create', () => { + it('should call audit logger', async () => { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + const soClient = savedObjectsClientMock.create(); + + soClient.create.mockResolvedValueOnce({ + id: 'test-package-policy', + attributes: {}, + references: [], + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + + mockAgentPolicyGet(); + + await packagePolicyService.create( + soClient, + esClient, + { + name: 'Test Package Policy', + namespace: 'test', + enabled: true, + policy_id: 'test', + inputs: [], + }, + // Skipping unique name verification just means we have to less mocking/setup + { id: 'test-package-policy', skipUniqueNameVerification: true } + ); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toBeCalledWith({ + action: 'create', + id: 'test-package-policy', + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + }); + }); + + describe('bulkCreate', () => { + it('should call audit logger', async () => { + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + const soClient = savedObjectsClientMock.create(); + + soClient.bulkCreate.mockResolvedValueOnce({ + saved_objects: [ + { + id: 'test-package-policy-1', + attributes: {}, + references: [], + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }, + { + id: 'test-package-policy-2', + attributes: {}, + references: [], + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }, + ], + }); + + mockAgentPolicyGet(); + + await packagePolicyService.bulkCreate(soClient, esClient, [ + { + id: 'test-package-policy-1', + name: 'Test Package Policy 1', + namespace: 'test', + enabled: true, + policy_id: 'test_agent_policy', + inputs: [], + }, + { + id: 'test-package-policy-2', + name: 'Test Package Policy 2', + namespace: 'test', + enabled: true, + policy_id: 'test_agent_policy', + inputs: [], + }, + ]); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenNthCalledWith(1, { + action: 'create', + id: 'test-package-policy-1', + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenNthCalledWith(2, { + action: 'create', + id: 'test-package-policy-2', + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + }); + }); + + describe('get', () => { + it('should call audit logger', async () => { + const soClient = savedObjectsClientMock.create(); + soClient.get.mockResolvedValueOnce({ + id: 'test-package-policy', + attributes: {}, + references: [], + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + + await packagePolicyService.get(soClient, 'test-package-policy'); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toBeCalledWith({ + action: 'get', + id: 'test-package-policy', + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + }); + }); + + describe('getByIDs', () => { + it('should call audit logger', async () => { + const soClient = savedObjectsClientMock.create(); + soClient.bulkGet.mockResolvedValueOnce({ + saved_objects: [ + { + id: 'test-package-policy-1', + attributes: {}, + references: [], + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }, + { + id: 'test-package-policy-2', + attributes: {}, + references: [], + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }, + ], + }); + + await packagePolicyService.getByIDs(soClient, [ + 'test-package-policy-1', + 'test-package-policy-2', + ]); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenNthCalledWith(1, { + action: 'get', + id: 'test-package-policy-1', + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenNthCalledWith(2, { + action: 'get', + id: 'test-package-policy-2', + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + }); + }); + + describe('list', () => { + it('should call audit logger', async () => { + const soClient = savedObjectsClientMock.create(); + soClient.find.mockResolvedValueOnce({ + total: 1, + page: 1, + per_page: 10, + saved_objects: [ + { + id: 'test-package-policy-1', + attributes: {}, + references: [], + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + score: 0, + }, + { + id: 'test-package-policy-2', + attributes: {}, + references: [], + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + score: 0, + }, + ], + }); + + await packagePolicyService.list(soClient, { + page: 1, + perPage: 1, + kuery: '', + }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenNthCalledWith(1, { + action: 'find', + id: 'test-package-policy-1', + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenNthCalledWith(2, { + action: 'find', + id: 'test-package-policy-2', + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + }); + }); + describe('_compilePackagePolicyInputs', () => { it('should work with config variables from the stream', async () => { const inputs = await _compilePackagePolicyInputs( @@ -640,12 +853,6 @@ describe('Package policy service', () => { }); describe('update', () => { - beforeEach(() => { - appContextService.start(createAppContextStartContractMock()); - }); - afterEach(() => { - appContextService.stop(); - }); it('should fail to update on version conflict', async () => { const savedObjectsClient = savedObjectsClientMock.create(); savedObjectsClient.get.mockResolvedValue({ @@ -1377,16 +1584,49 @@ describe('Package policy service', () => { expect(result.name).toEqual('test'); }); + + it('should call audit logger', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + const mockPackagePolicy = createPackagePolicyMock(); + + const attributes = { + ...mockPackagePolicy, + inputs: [], + }; + + soClient.get.mockResolvedValue({ + id: 'test-package-policy', + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + references: [], + attributes, + }); + + soClient.update.mockResolvedValue({ + id: 'test-package-policy', + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + references: [], + attributes, + }); + + await packagePolicyService.update(soClient, esClient, 'test-package-policy', { + ...mockPackagePolicy, + inputs: [], + }); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'update', + id: 'test-package-policy', + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + }); }); describe('bulkUpdate', () => { beforeEach(() => { - appContextService.start(createAppContextStartContractMock()); mockedSendTelemetryEvents.mockReset(); }); - afterEach(() => { - appContextService.stop(); - }); it('should throw if the user try to update input vars that are frozen', async () => { const savedObjectsClient = savedObjectsClientMock.create(); @@ -2113,10 +2353,86 @@ describe('Package policy service', () => { expect(mockedSendTelemetryEvents).not.toBeCalled(); }); + + it('should call audit logger', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + const mockPackagePolicies = [ + { + id: 'test-package-policy-1', + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + attributes: {}, + references: [], + }, + { + id: 'test-package-policy-2', + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + attributes: {}, + references: [], + }, + ]; + + soClient.bulkGet.mockResolvedValueOnce({ + saved_objects: [...mockPackagePolicies], + }); + + soClient.bulkUpdate.mockResolvedValueOnce({ + saved_objects: [...mockPackagePolicies], + }); + + await packagePolicyService.bulkUpdate(soClient, esClient, [ + { + id: 'test-package-policy-1', + name: 'Test Package Policy 1', + namespace: 'test', + enabled: true, + policy_id: 'test-agent-policy', + inputs: [], + }, + { + id: 'test-package-policy-2', + name: 'Test Package Policy 2', + namespace: 'test', + enabled: true, + policy_id: 'test-agent-policy', + inputs: [], + }, + ]); + }); }); describe('delete', () => { + // TODO: Add tests it('should allow to delete a package policy', async () => {}); + + it('should call audit logger', async () => { + const soClient = savedObjectsClientMock.create(); + const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + + const mockPackagePolicy = { + id: 'test-package-policy', + type: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + attributes: {}, + references: [], + }; + + soClient.get.mockResolvedValueOnce({ + ...mockPackagePolicy, + }); + + soClient.delete.mockResolvedValueOnce({ + ...mockPackagePolicy, + }); + + await packagePolicyService.delete(soClient, esClient, ['test-package-policy']); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'delete', + id: 'test-package-policy', + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + }); }); describe('runPostDeleteExternalCallbacks', () => { @@ -2126,7 +2442,6 @@ describe('Package policy service', () => { let deletedPackagePolicies: PostDeletePackagePoliciesResponse; beforeEach(() => { - appContextService.start(createAppContextStartContractMock()); callingOrder = []; deletedPackagePolicies = [ { id: 'a', success: true }, @@ -2142,10 +2457,6 @@ describe('Package policy service', () => { appContextService.addExternalCallback('packagePolicyPostDelete', callbackTwo); }); - afterEach(() => { - appContextService.stop(); - }); - it('should execute external callbacks', async () => { const soClient = savedObjectsClientMock.create(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; @@ -2229,7 +2540,6 @@ describe('Package policy service', () => { let packagePolicies: DeletePackagePoliciesResponse; beforeEach(() => { - appContextService.start(createAppContextStartContractMock()); callingOrder = []; packagePolicies = [{ id: 'a' }, { id: 'a' }] as DeletePackagePoliciesResponse; callbackOne = jest.fn(async (deletedPolicies, soClient, esClient) => { @@ -2242,10 +2552,6 @@ describe('Package policy service', () => { appContextService.addExternalCallback('packagePolicyDelete', callbackTwo); }); - afterEach(() => { - appContextService.stop(); - }); - it('should execute external callbacks', async () => { const soClient = savedObjectsClientMock.create(); const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; @@ -2361,11 +2667,9 @@ describe('Package policy service', () => { beforeEach(() => { context = xpackMocks.createRequestHandlerContext(); request = httpServerMock.createKibanaRequest(); - appContextService.start(createAppContextStartContractMock()); }); afterEach(() => { - appContextService.stop(); jest.clearAllMocks(); callbackCallingOrder.length = 0; }); @@ -2550,11 +2854,9 @@ describe('Package policy service', () => { beforeEach(() => { context = xpackMocks.createRequestHandlerContext(); request = httpServerMock.createKibanaRequest(); - appContextService.start(createAppContextStartContractMock()); }); afterEach(() => { - appContextService.stop(); jest.clearAllMocks(); callbackCallingOrder.length = 0; }); @@ -4521,10 +4823,6 @@ describe('getUpgradeDryRunDiff', () => { let savedObjectsClient: jest.Mocked; beforeEach(() => { savedObjectsClient = savedObjectsClientMock.create(); - appContextService.start(createAppContextStartContractMock()); - }); - afterEach(() => { - appContextService.stop(); }); it('should return no errors if there is no conflict to upgrade', async () => { const res = await packagePolicyService.getUpgradeDryRunDiff( @@ -4650,10 +4948,6 @@ describe('_applyIndexPrivileges()', () => { }; } - beforeAll(async () => { - appContextService.start(createAppContextStartContractMock()); - }); - it('should do nothing if packageStream has no privileges', () => { const packageStream = createPackageStream(); const inputStream = createInputStream(); diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index fe044f012dd11..759a6bc3d05c3 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -17,6 +17,7 @@ import type { Logger, RequestHandlerContext, } from '@kbn/core/server'; +import { SavedObjectsUtils } from '@kbn/core/server'; import { v4 as uuidv4 } from 'uuid'; import { safeLoad } from 'js-yaml'; @@ -104,6 +105,7 @@ import { handleExperimentalDatastreamFeatureOptIn } from './package_policies'; import { updateDatastreamExperimentalFeatures } from './epm/packages/update'; import type { PackagePolicyClient, PackagePolicyService } from './package_policy_service'; import { installAssetsForInputPackagePolicy } from './epm/packages/install'; +import { auditLoggingService } from './audit_logging'; export type InputsOverride = Partial & { vars?: Array; @@ -125,7 +127,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { soClient: SavedObjectsClientContract, esClient: ElasticsearchClient, packagePolicy: NewPackagePolicy, - options?: { + options: { spaceId?: string; id?: string; user?: AuthenticatedUser; @@ -135,10 +137,21 @@ class PackagePolicyClientImpl implements PackagePolicyClient { skipUniqueNameVerification?: boolean; overwrite?: boolean; packageInfo?: PackageInfo; - }, + } = {}, context?: RequestHandlerContext, request?: KibanaRequest ): Promise { + // Ensure an ID is provided, so we can include it in the audit logs below + if (!options.id) { + options.id = SavedObjectsUtils.generateId(); + } + + auditLoggingService.writeCustomSoAuditLog({ + action: 'create', + id: options.id, + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + const logger = appContextService.getLogger(); const enrichedPackagePolicy = await packagePolicyService.runExternalCallbacks( @@ -260,6 +273,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { } const createdPackagePolicy = { id: newSo.id, version: newSo.version, ...newSo.attributes }; + return packagePolicyService.runExternalCallbacks( 'packagePolicyPostCreate', createdPackagePolicy, @@ -278,6 +292,18 @@ class PackagePolicyClientImpl implements PackagePolicyClient { force?: true; } ): Promise { + for (const packagePolicy of packagePolicies) { + if (!packagePolicy.id) { + packagePolicy.id = SavedObjectsUtils.generateId(); + } + + auditLoggingService.writeCustomSoAuditLog({ + action: 'create', + id: packagePolicy.id, + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + } + const agentPolicyIds = new Set(packagePolicies.map((pkgPolicy) => pkgPolicy.policy_id)); for (const agentPolicyId of agentPolicyIds) { @@ -390,6 +416,12 @@ class PackagePolicyClientImpl implements PackagePolicyClient { response.package.experimental_data_stream_features = experimentalFeatures; } + auditLoggingService.writeCustomSoAuditLog({ + action: 'get', + id, + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + return response; } @@ -406,11 +438,21 @@ class PackagePolicyClientImpl implements PackagePolicyClient { return []; } - return packagePolicySO.saved_objects.map((so) => ({ + const packagePolicies = packagePolicySO.saved_objects.map((so) => ({ id: so.id, version: so.version, ...so.attributes, })); + + for (const packagePolicy of packagePolicies) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'find', + id: packagePolicy.id, + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + } + + return packagePolicies; } public async getByIDs( @@ -428,7 +470,7 @@ class PackagePolicyClientImpl implements PackagePolicyClient { return null; } - return packagePolicySO.saved_objects + const packagePolicies = packagePolicySO.saved_objects .map((so): PackagePolicy | null => { if (so.error) { if (options.ignoreMissing && so.error.statusCode === 404) { @@ -447,6 +489,16 @@ class PackagePolicyClientImpl implements PackagePolicyClient { }; }) .filter((packagePolicy): packagePolicy is PackagePolicy => packagePolicy !== null); + + for (const packagePolicy of packagePolicies) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'get', + id: packagePolicy.id, + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + } + + return packagePolicies; } public async list( @@ -464,6 +516,14 @@ class PackagePolicyClientImpl implements PackagePolicyClient { filter: kuery ? normalizeKuery(SAVED_OBJECT_TYPE, kuery) : undefined, }); + for (const packagePolicy of packagePolicies?.saved_objects ?? []) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'find', + id: packagePolicy.id, + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + } + return { items: packagePolicies?.saved_objects.map((packagePolicySO) => ({ id: packagePolicySO.id, @@ -492,6 +552,14 @@ class PackagePolicyClientImpl implements PackagePolicyClient { filter: kuery ? normalizeKuery(SAVED_OBJECT_TYPE, kuery) : undefined, }); + for (const packagePolicy of packagePolicies.saved_objects) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'find', + id: packagePolicy.id, + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + } + return { items: packagePolicies.saved_objects.map((packagePolicySO) => packagePolicySO.id), total: packagePolicies.total, @@ -507,6 +575,12 @@ class PackagePolicyClientImpl implements PackagePolicyClient { packagePolicyUpdate: UpdatePackagePolicy, options?: { user?: AuthenticatedUser; force?: boolean; skipUniqueNameVerification?: boolean } ): Promise { + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id, + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + let enrichedPackagePolicy: UpdatePackagePolicy; try { @@ -647,6 +721,13 @@ class PackagePolicyClientImpl implements PackagePolicyClient { options?: { user?: AuthenticatedUser; force?: boolean }, currentVersion?: string ): Promise { + for (const packagePolicy of packagePolicyUpdates) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: packagePolicy.id, + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + } const oldPackagePolicies = await this.getByIDs( soClient, packagePolicyUpdates.map((p) => p.id) @@ -770,6 +851,14 @@ class PackagePolicyClientImpl implements PackagePolicyClient { context?: RequestHandlerContext, request?: KibanaRequest ): Promise { + for (const id of ids) { + auditLoggingService.writeCustomSoAuditLog({ + action: 'delete', + id, + savedObjectType: PACKAGE_POLICY_SAVED_OBJECT_TYPE, + }); + } + const result: PostDeletePackagePoliciesResponse = []; const logger = appContextService.getLogger(); diff --git a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts index 320720c8dbedb..62758a93faa90 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration.test.ts @@ -273,6 +273,8 @@ jest.mock('./app_context', () => ({ }, })); +jest.mock('./audit_logging'); + const spyAgentPolicyServiceUpdate = jest.spyOn(agentPolicy.agentPolicyService, 'update'); const spyAgentPolicyServicBumpAllAgentPoliciesForOutput = jest.spyOn( agentPolicy.agentPolicyService, diff --git a/x-pack/plugins/fleet/server/services/preconfiguration/fleet_server_host.test.ts b/x-pack/plugins/fleet/server/services/preconfiguration/fleet_server_host.test.ts index 87717860e9be4..aec64226ca5ec 100644 --- a/x-pack/plugins/fleet/server/services/preconfiguration/fleet_server_host.test.ts +++ b/x-pack/plugins/fleet/server/services/preconfiguration/fleet_server_host.test.ts @@ -5,6 +5,7 @@ * 2.0. */ import { savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { securityMock } from '@kbn/security-plugin/server/mocks'; import { appContextService } from '../app_context'; import { getDefaultFleetServerHost, createFleetServerHost } from '../fleet_server_host'; @@ -19,6 +20,10 @@ jest.mock('../fleet_server_host'); jest.mock('../app_context'); const mockedAppContextService = appContextService as jest.Mocked; +mockedAppContextService.getSecuritySetup.mockImplementation(() => ({ + ...securityMock.createSetup(), +})); + const mockedGetDefaultFleetServerHost = getDefaultFleetServerHost as jest.MockedFunction< typeof getDefaultFleetServerHost >; diff --git a/x-pack/plugins/fleet/server/services/request_store.ts b/x-pack/plugins/fleet/server/services/request_store.ts new file mode 100644 index 0000000000000..bed1bc174e764 --- /dev/null +++ b/x-pack/plugins/fleet/server/services/request_store.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AsyncLocalStorage } from 'async_hooks'; + +import type { KibanaRequest } from '@kbn/core-http-server'; + +export function getRequestStore() { + const requestStore = new AsyncLocalStorage(); + + return requestStore; +} diff --git a/x-pack/plugins/fleet/server/services/settings.test.ts b/x-pack/plugins/fleet/server/services/settings.test.ts index ca0be8130b21a..33926b0ec12f7 100644 --- a/x-pack/plugins/fleet/server/services/settings.test.ts +++ b/x-pack/plugins/fleet/server/services/settings.test.ts @@ -6,13 +6,31 @@ */ import { savedObjectsClientMock } from '@kbn/core/server/mocks'; +import { securityMock } from '@kbn/security-plugin/server/mocks'; + +import Boom from '@hapi/boom'; + +import { GLOBAL_SETTINGS_ID, GLOBAL_SETTINGS_SAVED_OBJECT_TYPE } from '../../common/constants'; + +import type { Settings } from '../types'; import { appContextService } from './app_context'; -import { settingsSetup } from './settings'; +import { getSettings, saveSettings, settingsSetup } from './settings'; +import { auditLoggingService } from './audit_logging'; +import { listFleetServerHosts } from './fleet_server_host'; jest.mock('./app_context'); +jest.mock('./audit_logging'); +jest.mock('./fleet_server_host'); +const mockListFleetServerHosts = listFleetServerHosts as jest.MockedFunction< + typeof listFleetServerHosts +>; +const mockedAuditLoggingService = auditLoggingService as jest.Mocked; const mockedAppContextService = appContextService as jest.Mocked; +mockedAppContextService.getSecuritySetup.mockImplementation(() => ({ + ...securityMock.createSetup(), +})); describe('settingsSetup', () => { afterEach(() => { @@ -66,8 +84,145 @@ describe('settingsSetup', () => { type: 'so_type', }); + mockListFleetServerHosts.mockResolvedValueOnce({ + items: [ + { + id: 'fleet-server-host', + name: 'Fleet Server Host', + is_default: true, + is_preconfigured: false, + host_urls: ['http://localhost:8220'], + }, + ], + page: 1, + perPage: 10, + total: 1, + }); + await settingsSetup(soClientMock); expect(soClientMock.create).not.toBeCalled(); }); }); + +describe('getSettings', () => { + it('should call audit logger', async () => { + const soClient = savedObjectsClientMock.create(); + + soClient.find.mockResolvedValueOnce({ + saved_objects: [ + { + id: GLOBAL_SETTINGS_ID, + attributes: {}, + references: [], + type: GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, + score: 0, + }, + ], + page: 1, + per_page: 10, + total: 1, + }); + + mockListFleetServerHosts.mockResolvedValueOnce({ + items: [ + { + id: 'fleet-server-host', + name: 'Fleet Server Host', + is_default: true, + is_preconfigured: false, + host_urls: ['http://localhost:8220'], + }, + ], + page: 1, + perPage: 10, + total: 1, + }); + + await getSettings(soClient); + }); +}); + +describe('saveSettings', () => { + describe('when settings object exists', () => { + it('should call audit logger', async () => { + const soClient = savedObjectsClientMock.create(); + + const newData: Partial> = { + fleet_server_hosts: ['http://localhost:8220'], + }; + + soClient.find.mockResolvedValueOnce({ + saved_objects: [ + { + id: GLOBAL_SETTINGS_ID, + attributes: {}, + references: [], + type: GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, + score: 0, + }, + ], + page: 1, + per_page: 10, + total: 1, + }); + + soClient.update.mockResolvedValueOnce({ + id: GLOBAL_SETTINGS_ID, + attributes: {}, + references: [], + type: GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, + }); + + mockListFleetServerHosts.mockResolvedValueOnce({ + items: [ + { + id: 'fleet-server-host', + name: 'Fleet Server Host', + is_default: true, + is_preconfigured: false, + host_urls: ['http://localhost:8220'], + }, + ], + page: 1, + perPage: 10, + total: 1, + }); + + await saveSettings(soClient, newData); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'create', + id: GLOBAL_SETTINGS_ID, + savedObjectType: GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, + }); + }); + + describe('when settings object does not exist', () => { + it('should call audit logger', async () => { + const soClient = savedObjectsClientMock.create(); + + const newData: Partial> = { + fleet_server_hosts: ['http://localhost:8220'], + }; + + soClient.find.mockRejectedValueOnce(Boom.notFound('not found')); + + soClient.create.mockResolvedValueOnce({ + id: GLOBAL_SETTINGS_ID, + attributes: {}, + references: [], + type: GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, + }); + + await saveSettings(soClient, newData); + + expect(mockedAuditLoggingService.writeCustomSoAuditLog).toHaveBeenCalledWith({ + action: 'create', + id: GLOBAL_SETTINGS_ID, + savedObjectType: GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, + }); + }); + }); + }); +}); diff --git a/x-pack/plugins/fleet/server/services/settings.ts b/x-pack/plugins/fleet/server/services/settings.ts index 7fe6c25d24b86..5b48ee8177885 100644 --- a/x-pack/plugins/fleet/server/services/settings.ts +++ b/x-pack/plugins/fleet/server/services/settings.ts @@ -14,11 +14,17 @@ import type { SettingsSOAttributes, Settings, BaseSettings } from '../../common/ import { appContextService } from './app_context'; import { listFleetServerHosts } from './fleet_server_host'; +import { auditLoggingService } from './audit_logging'; export async function getSettings(soClient: SavedObjectsClientContract): Promise { const res = await soClient.find({ type: GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, }); + auditLoggingService.writeCustomSoAuditLog({ + action: 'get', + id: GLOBAL_SETTINGS_ID, + savedObjectType: GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, + }); if (res.total === 0) { throw Boom.notFound('Global settings not found'); @@ -59,6 +65,12 @@ export async function saveSettings( try { const settings = await getSettings(soClient); + auditLoggingService.writeCustomSoAuditLog({ + action: 'update', + id: settings.id, + savedObjectType: GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, + }); + const res = await soClient.update( GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, settings.id, @@ -72,6 +84,13 @@ export async function saveSettings( } catch (e) { if (e.isBoom && e.output.statusCode === 404) { const defaultSettings = createDefaultSettings(); + + auditLoggingService.writeCustomSoAuditLog({ + action: 'create', + id: GLOBAL_SETTINGS_ID, + savedObjectType: GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, + }); + const res = await soClient.create( GLOBAL_SETTINGS_SAVED_OBJECT_TYPE, { From cd727fa190361956c1228d195a1aedebdae62ed6 Mon Sep 17 00:00:00 2001 From: Patrick Mueller Date: Mon, 3 Apr 2023 09:19:48 -0400 Subject: [PATCH 04/10] [ResponseOps] move alert UUID generation from rule registry to the alerting framework (#143489) resolves https://github.com/elastic/kibana/issues/142874 The alerting framework now generates an alert UUID for every alert it creates. The UUID will be reused for alerts which continue to be active on subsequent runs, until the alert recovers. When the same alert (alert instance id) becomes active again, a new UUID will be generated. These UUIDs then identify a "span" of events for a single alert. The rule registry plugin was already adding these UUIDs to it's own alerts-as-data indices, and that code has now been changed to make use of the new UUID the alerting framework generates. - adds property in the rule task state `alertInstances[alertInstanceId].meta.uuid`; this is where the alert UUID is persisted across runs - adds a new `Alert` method getUuid(): string` that can be used by rule executors to obtain the UUID of the alert they just retrieved from the factory; the rule registry uses this to get the UUID generated by the alerting framework - for the event log, adds the property `kibana.alert.uuid` to `*-instance` event log events; this is the same field the rule registry writes into the alerts-as-data indices - various changes to tests to accommodate new UUID data / methods - migrates the UUID previous stored with lifecycle alerts in the alert state, via the rule registry *INTO* the new `meta.uuid` field in the existing alert state. --- .github/CODEOWNERS | 1 + docs/user/alerting/action-variables.asciidoc | 1 + package.json | 1 + .../group2/check_registered_types.test.ts | 2 +- tsconfig.base.json | 2 + .../kbn-alerting-state-types/README.md | 8 + .../kbn-alerting-state-types/index.ts | 24 ++ .../kbn-alerting-state-types/jest.config.js | 12 + .../kbn-alerting-state-types/kibana.jsonc | 5 + .../kbn-alerting-state-types/package.json | 6 + .../src}/alert_instance.ts | 1 + .../src}/date_from_string.test.ts | 0 .../src}/date_from_string.ts | 0 .../src/lifecycle_state.ts | 48 +++ .../src}/rule_task_instance.ts | 0 .../kbn-alerting-state-types/tsconfig.json | 17 + x-pack/plugins/alerting/README.md | 3 + .../plugins/alerting/common/alert_summary.ts | 1 + x-pack/plugins/alerting/common/index.ts | 22 +- .../alerting/server/alert/alert.test.ts | 47 ++- x-pack/plugins/alerting/server/alert/alert.ts | 19 +- .../server/alert/create_alert_factory.test.ts | 79 +++-- .../lib/alert_summary_from_event_log.test.ts | 97 +++--- .../lib/alert_summary_from_event_log.ts | 10 +- .../alerting_event_logger.test.ts | 2 + .../alerting_event_logger.ts | 2 + .../create_alert_event_log_record_object.ts | 3 + .../lib/get_alerts_for_notification.test.ts | 47 ++- .../server/lib/process_alerts.test.ts | 70 +++- .../server/lib/trim_recovered_alerts.test.ts | 6 +- x-pack/plugins/alerting/server/mocks.ts | 1 + .../rules_client/lib/recover_rule_alerts.ts | 2 + .../server/rules_client/tests/disable.test.ts | 2 + .../tests/get_alert_summary.test.ts | 15 +- .../task_runner/execution_handler.test.ts | 1 + .../server/task_runner/execution_handler.ts | 1 + .../alerting/server/task_runner/fixtures.ts | 2 + .../server/task_runner/log_alerts.test.ts | 51 ++- .../alerting/server/task_runner/log_alerts.ts | 19 +- .../server/task_runner/task_runner.test.ts | 3 + .../transform_action_params.test.ts | 23 +- .../task_runner/transform_action_params.ts | 3 + .../alerting/server/test_utils/index.ts | 24 ++ x-pack/plugins/alerting/tsconfig.json | 1 + .../server/routes/alerts/test_utils/index.ts | 6 +- .../plugins/event_log/generated/mappings.json | 4 + x-pack/plugins/event_log/generated/schemas.ts | 1 + x-pack/plugins/event_log/scripts/mappings.js | 4 + .../server/utils/create_lifecycle_executor.ts | 80 ++--- .../utils/create_lifecycle_rule_type.test.ts | 4 + x-pack/plugins/rule_registry/tsconfig.json | 1 + .../alert_instance_factory_stub.ts | 4 + .../rule_preview/api/preview_rules/route.ts | 1 + .../server/saved_objects/migrations.ts | 127 ++++++- .../saved_objects/migrations_880.test.ts | 327 ++++++++++++++++++ .../migrations_helpers/get_rule_task_state.js | 265 ++++++++++++++ x-pack/plugins/task_manager/tsconfig.json | 3 +- .../tests/alerting/group1/event_log.ts | 90 +++++ .../alerting/group1/get_alert_summary.ts | 35 +- .../tests/alerting/group4/event_log_alerts.ts | 16 +- .../es_archives/task_manager_tasks/data.json | 72 ++++ .../task_manager_tasks/mappings.json | 38 +- .../test_suites/task_manager/migrations.ts | 82 +++++ .../lib/helpers/cleanup_target_indices.ts | 2 +- .../rule_registry/common/lib/helpers/index.ts | 2 + .../common/lib/helpers/mock_alert_factory.ts | 29 ++ .../trial/__snapshots__/create_rule.snap | 32 ++ .../spaces_only/tests/trial/create_rule.ts | 79 ++++- .../tests/trial/get_summarized_alerts.ts | 6 +- .../tests/trial/lifecycle_executor.ts | 27 +- x-pack/test/tsconfig.json | 1 + yarn.lock | 4 + 72 files changed, 1786 insertions(+), 240 deletions(-) create mode 100644 x-pack/packages/kbn-alerting-state-types/README.md create mode 100644 x-pack/packages/kbn-alerting-state-types/index.ts create mode 100644 x-pack/packages/kbn-alerting-state-types/jest.config.js create mode 100644 x-pack/packages/kbn-alerting-state-types/kibana.jsonc create mode 100644 x-pack/packages/kbn-alerting-state-types/package.json rename x-pack/{plugins/alerting/common => packages/kbn-alerting-state-types/src}/alert_instance.ts (98%) rename x-pack/{plugins/alerting/common => packages/kbn-alerting-state-types/src}/date_from_string.test.ts (100%) rename x-pack/{plugins/alerting/common => packages/kbn-alerting-state-types/src}/date_from_string.ts (100%) create mode 100644 x-pack/packages/kbn-alerting-state-types/src/lifecycle_state.ts rename x-pack/{plugins/alerting/common => packages/kbn-alerting-state-types/src}/rule_task_instance.ts (100%) create mode 100644 x-pack/packages/kbn-alerting-state-types/tsconfig.json create mode 100644 x-pack/plugins/task_manager/server/saved_objects/migrations_880.test.ts create mode 100755 x-pack/plugins/task_manager/server/saved_objects/migrations_helpers/get_rule_task_state.js create mode 100644 x-pack/test/rule_registry/common/lib/helpers/mock_alert_factory.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 549670c1607dc..234089fb14b3d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -18,6 +18,7 @@ x-pack/test/alerting_api_integration/common/plugins/alerts @elastic/response-ops x-pack/examples/alerting_example @elastic/response-ops x-pack/test/functional_with_es_ssl/plugins/alerts @elastic/response-ops x-pack/plugins/alerting @elastic/response-ops +x-pack/packages/kbn-alerting-state-types @elastic/response-ops packages/kbn-alerts @elastic/security-solution packages/kbn-alerts-as-data-utils @elastic/response-ops x-pack/test/alerting_api_integration/common/plugins/alerts_restricted @elastic/response-ops diff --git a/docs/user/alerting/action-variables.asciidoc b/docs/user/alerting/action-variables.asciidoc index 24f4da2320995..4eda20454ef93 100644 --- a/docs/user/alerting/action-variables.asciidoc +++ b/docs/user/alerting/action-variables.asciidoc @@ -100,6 +100,7 @@ If the rule's action frequency is not a summary of alerts, it passes the followi `alert.actionSubgroup`:: The action subgroup of the alert that scheduled the action. `alert.flapping`:: A flag on the alert that indicates whether the alert status is changing repeatedly. `alert.id`:: The ID of the alert that scheduled the action. +`alert.uuid`:: A universally unique identifier for the alert. While the alert is active, the UUID value remains unchanged each time the rule runs. preview:[] [float] [[defining-rules-actions-variable-context]] diff --git a/package.json b/package.json index 7583297509c3e..54bbb5f2b7a29 100644 --- a/package.json +++ b/package.json @@ -133,6 +133,7 @@ "@kbn/alerting-example-plugin": "link:x-pack/examples/alerting_example", "@kbn/alerting-fixture-plugin": "link:x-pack/test/functional_with_es_ssl/plugins/alerts", "@kbn/alerting-plugin": "link:x-pack/plugins/alerting", + "@kbn/alerting-state-types": "link:x-pack/packages/kbn-alerting-state-types", "@kbn/alerts": "link:packages/kbn-alerts", "@kbn/alerts-as-data-utils": "link:packages/kbn-alerts-as-data-utils", "@kbn/alerts-restricted-fixtures-plugin": "link:x-pack/test/alerting_api_integration/common/plugins/alerts_restricted", diff --git a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts index 010d101fbe9d7..caa6d371d50b2 100644 --- a/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts +++ b/src/core/server/integration_tests/saved_objects/migrations/group2/check_registered_types.test.ts @@ -141,7 +141,7 @@ describe('checking migration metadata changes on all registered SO types', () => "synthetics-param": "9776c9b571d35f0d0397e8915e035ea1dc026db7", "synthetics-privates-locations": "7d032fc788905e32152029ae7ab3d6038c48ae44", "tag": "87f21f07df9cc37001b15a26e413c18f50d1fbfe", - "task": "ebcc113df12f14bf627dbd335ba78507187b48a3", + "task": "ff760534a44c4cfabcf4baf8cfe8283f717cab02", "telemetry": "561b329aaed3c15b91aaf2075645be3097247612", "ui-metric": "410a8ad28e0f44b161c960ff0ce950c712b17c52", "upgrade-assistant-ml-upgrade-operation": "d8816e5ce32649e7a3a43e2c406c632319ff84bb", diff --git a/tsconfig.base.json b/tsconfig.base.json index 652c16dae834b..b83860768e3ee 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -30,6 +30,8 @@ "@kbn/alerting-fixture-plugin/*": ["x-pack/test/functional_with_es_ssl/plugins/alerts/*"], "@kbn/alerting-plugin": ["x-pack/plugins/alerting"], "@kbn/alerting-plugin/*": ["x-pack/plugins/alerting/*"], + "@kbn/alerting-state-types": ["x-pack/packages/kbn-alerting-state-types"], + "@kbn/alerting-state-types/*": ["x-pack/packages/kbn-alerting-state-types/*"], "@kbn/alerts": ["packages/kbn-alerts"], "@kbn/alerts/*": ["packages/kbn-alerts/*"], "@kbn/alerts-as-data-utils": ["packages/kbn-alerts-as-data-utils"], diff --git a/x-pack/packages/kbn-alerting-state-types/README.md b/x-pack/packages/kbn-alerting-state-types/README.md new file mode 100644 index 0000000000000..e92a02d75a969 --- /dev/null +++ b/x-pack/packages/kbn-alerting-state-types/README.md @@ -0,0 +1,8 @@ +# @kbn/alerting-state-types + +Contains type information for the alerting data persisted in task +manager documents as state. + +Because task manager migrations sometimes need this data, it needs +to be in a package outside of alerting. + diff --git a/x-pack/packages/kbn-alerting-state-types/index.ts b/x-pack/packages/kbn-alerting-state-types/index.ts new file mode 100644 index 0000000000000..c30ea9fb35f2a --- /dev/null +++ b/x-pack/packages/kbn-alerting-state-types/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export type { + ThrottledActions, + LastScheduledActions, + AlertInstanceMeta, + AlertInstanceState, + AlertInstanceContext, + RawAlertInstance, +} from './src/alert_instance'; +export { rawAlertInstance } from './src/alert_instance'; + +export { DateFromString } from './src/date_from_string'; + +export type { TrackedLifecycleAlertState, WrappedLifecycleRuleState } from './src/lifecycle_state'; +export { wrappedStateRt } from './src/lifecycle_state'; + +export type { RuleTaskState, RuleTaskParams } from './src/rule_task_instance'; +export { ActionsCompletion, ruleStateSchema, ruleParamsSchema } from './src/rule_task_instance'; diff --git a/x-pack/packages/kbn-alerting-state-types/jest.config.js b/x-pack/packages/kbn-alerting-state-types/jest.config.js new file mode 100644 index 0000000000000..50f7065f8b992 --- /dev/null +++ b/x-pack/packages/kbn-alerting-state-types/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../../..', + roots: ['/x-pack/packages/kbn-alerting-state-types'], +}; diff --git a/x-pack/packages/kbn-alerting-state-types/kibana.jsonc b/x-pack/packages/kbn-alerting-state-types/kibana.jsonc new file mode 100644 index 0000000000000..91682423ad4b0 --- /dev/null +++ b/x-pack/packages/kbn-alerting-state-types/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-common", + "id": "@kbn/alerting-state-types", + "owner": "@elastic/response-ops" +} diff --git a/x-pack/packages/kbn-alerting-state-types/package.json b/x-pack/packages/kbn-alerting-state-types/package.json new file mode 100644 index 0000000000000..c09089d26390f --- /dev/null +++ b/x-pack/packages/kbn-alerting-state-types/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/alerting-state-types", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0" +} diff --git a/x-pack/plugins/alerting/common/alert_instance.ts b/x-pack/packages/kbn-alerting-state-types/src/alert_instance.ts similarity index 98% rename from x-pack/plugins/alerting/common/alert_instance.ts rename to x-pack/packages/kbn-alerting-state-types/src/alert_instance.ts index 7aad3a05de6f2..2e322f41f512c 100644 --- a/x-pack/plugins/alerting/common/alert_instance.ts +++ b/x-pack/packages/kbn-alerting-state-types/src/alert_instance.ts @@ -37,6 +37,7 @@ const metaSchema = t.partial({ // flapping flag that indicates whether the alert is flapping flapping: t.boolean, pendingRecoveredCount: t.number, + uuid: t.string, }); export type AlertInstanceMeta = t.TypeOf; diff --git a/x-pack/plugins/alerting/common/date_from_string.test.ts b/x-pack/packages/kbn-alerting-state-types/src/date_from_string.test.ts similarity index 100% rename from x-pack/plugins/alerting/common/date_from_string.test.ts rename to x-pack/packages/kbn-alerting-state-types/src/date_from_string.test.ts diff --git a/x-pack/plugins/alerting/common/date_from_string.ts b/x-pack/packages/kbn-alerting-state-types/src/date_from_string.ts similarity index 100% rename from x-pack/plugins/alerting/common/date_from_string.ts rename to x-pack/packages/kbn-alerting-state-types/src/date_from_string.ts diff --git a/x-pack/packages/kbn-alerting-state-types/src/lifecycle_state.ts b/x-pack/packages/kbn-alerting-state-types/src/lifecycle_state.ts new file mode 100644 index 0000000000000..0d8fb3e5aaada --- /dev/null +++ b/x-pack/packages/kbn-alerting-state-types/src/lifecycle_state.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; + +const trackedAlertStateRt = t.type({ + alertId: t.string, + alertUuid: t.string, + started: t.string, + // an array used to track changes in alert state, the order is based on the rule executions + // true - alert has changed from active/recovered + // false - alert is new or the status has remained either active or recovered + flappingHistory: t.array(t.boolean), + // flapping flag that indicates whether the alert is flapping + flapping: t.boolean, + pendingRecoveredCount: t.number, +}); + +export type TrackedLifecycleAlertState = t.TypeOf; + +type RuleTypeState = Record; + +export const alertTypeStateRt = () => + t.record(t.string, t.unknown) as t.Type; + +export const wrappedStateRt = () => + t.type({ + wrapped: alertTypeStateRt(), + // tracks the active alerts + trackedAlerts: t.record(t.string, trackedAlertStateRt), + // tracks the recovered alerts + trackedAlertsRecovered: t.record(t.string, trackedAlertStateRt), + }); + +/** + * This is redefined instead of derived from above `wrappedStateRt` because + * there's no easy way to instantiate generic values such as the runtime type + * factory function. + */ +export type WrappedLifecycleRuleState = RuleTypeState & { + wrapped: State; + trackedAlerts: Record; + trackedAlertsRecovered: Record; +}; diff --git a/x-pack/plugins/alerting/common/rule_task_instance.ts b/x-pack/packages/kbn-alerting-state-types/src/rule_task_instance.ts similarity index 100% rename from x-pack/plugins/alerting/common/rule_task_instance.ts rename to x-pack/packages/kbn-alerting-state-types/src/rule_task_instance.ts diff --git a/x-pack/packages/kbn-alerting-state-types/tsconfig.json b/x-pack/packages/kbn-alerting-state-types/tsconfig.json new file mode 100644 index 0000000000000..6d27b06d5f8ba --- /dev/null +++ b/x-pack/packages/kbn-alerting-state-types/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [] +} diff --git a/x-pack/plugins/alerting/README.md b/x-pack/plugins/alerting/README.md index f547c05e52e09..e1225d8a9af0b 100644 --- a/x-pack/plugins/alerting/README.md +++ b/x-pack/plugins/alerting/README.md @@ -766,6 +766,7 @@ This factory returns an instance of `Alert`. The `Alert` class has the following |Method|Description| |---|---| +|getUuid()|Get the UUID of the alert.| |getState()|Get the current state of the alert.| |scheduleActions(actionGroup, context)|Call this to schedule the execution of actions. The actionGroup is a string `id` that relates to the group of alert `actions` to execute and the context will be used for templating purposes. `scheduleActions` should only be called once per alert.| |replaceState(state)|Used to replace the current state of the alert. This doesn't work like React, the entire state must be provided. Use this feature as you see fit. The state that is set will persist between rule executions whenever you re-create an alert with the same id. The alert state will be erased when `scheduleActions`isn't called during an execution.| @@ -790,6 +791,8 @@ When an alert executes, the first argument is the `group` of actions to execute The templating engine is [mustache]. General definition for the [mustache variable] is a double-brace {{}}. All variables are HTML-escaped by default and if there is a requirement to render unescaped HTML, it should be applied with the triple mustache: `{{{name}}}`. Also, `&` can be used to unescape a variable. +The complete list of variables available has grown, and difficult to keep in synch here as well, so refer to the published documentation for the variables available: https://www.elastic.co/guide/en/kibana/master/rule-action-variables.html + ### Examples The following code would be within a rule type. As you can see `cpuUsage` will replace the state of the alert and `server` is the context for the alert to execute. The difference between the two is that `cpuUsage` will be accessible at the next execution. diff --git a/x-pack/plugins/alerting/common/alert_summary.ts b/x-pack/plugins/alerting/common/alert_summary.ts index 25b00538c16be..b1563882d4f21 100644 --- a/x-pack/plugins/alerting/common/alert_summary.ts +++ b/x-pack/plugins/alerting/common/alert_summary.ts @@ -32,6 +32,7 @@ export interface AlertSummary { } export interface AlertStatus { + uuid?: string; status: AlertStatusValues; muted: boolean; actionGroupId?: string; diff --git a/x-pack/plugins/alerting/common/index.ts b/x-pack/plugins/alerting/common/index.ts index 52d1e4f221db0..32065c40f9310 100644 --- a/x-pack/plugins/alerting/common/index.ts +++ b/x-pack/plugins/alerting/common/index.ts @@ -13,8 +13,26 @@ import { AlertsHealth } from './rule'; export * from './rule'; export * from './rules_settings'; export * from './rule_type'; -export * from './rule_task_instance'; -export * from './alert_instance'; +export type { + ThrottledActions, + LastScheduledActions, + AlertInstanceMeta, + AlertInstanceState, + AlertInstanceContext, + RawAlertInstance, + TrackedLifecycleAlertState, + WrappedLifecycleRuleState, + RuleTaskState, + RuleTaskParams, +} from '@kbn/alerting-state-types'; +export { + rawAlertInstance, + DateFromString, + wrappedStateRt, + ActionsCompletion, + ruleStateSchema, + ruleParamsSchema, +} from '@kbn/alerting-state-types'; export * from './alert_summary'; export * from './builtin_action_groups'; export * from './bulk_edit'; diff --git a/x-pack/plugins/alerting/server/alert/alert.test.ts b/x-pack/plugins/alerting/server/alert/alert.test.ts index 890b4ade280fd..c5b2f97d1b48c 100644 --- a/x-pack/plugins/alerting/server/alert/alert.test.ts +++ b/x-pack/plugins/alerting/server/alert/alert.test.ts @@ -8,6 +8,7 @@ import sinon from 'sinon'; import { Alert } from './alert'; import { AlertInstanceState, AlertInstanceContext, DefaultActionGroupId } from '../../common'; +import { alertWithAnyUUID } from '../test_utils'; let clock: sinon.SinonFakeTimers; @@ -231,6 +232,23 @@ describe('getState()', () => { }); }); +describe('getUUID()', () => { + test('returns a UUID for a new alert', () => { + const alert = new Alert('1'); + const uuid = alert.getUuid(); + expect(uuid).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/); + }); + + test('returns same uuid from previous run of alert', () => { + const uuid = 'previous-uuid'; + const meta = { uuid }; + const alert = new Alert('1', { + meta, + }); + expect(alert.getUuid()).toEqual(uuid); + }); +}); + describe('scheduleActions()', () => { test('makes hasScheduledActions() return true', () => { const alert = new Alert('1', { @@ -320,6 +338,7 @@ describe('updateLastScheduledActions()', () => { expect(alert.toJSON()).toEqual({ state: {}, meta: { + uuid: expect.any(String), lastScheduledActions: { date: new Date().toISOString(), group: 'default', @@ -338,6 +357,7 @@ describe('updateLastScheduledActions()', () => { state: {}, meta: { flappingHistory: [], + uuid: expect.any(String), lastScheduledActions: { date: new Date().toISOString(), group: 'default', @@ -367,6 +387,7 @@ describe('updateLastScheduledActions()', () => { state: {}, meta: { flappingHistory: [], + uuid: expect.any(String), lastScheduledActions: { date: new Date().toISOString(), group: 'default', @@ -468,9 +489,22 @@ describe('toJSON', () => { }, } ); - expect(JSON.stringify(alertInstance)).toEqual( - '{"state":{"foo":true},"meta":{"lastScheduledActions":{"date":"1970-01-01T00:00:00.000Z","group":"default"},"flappingHistory":[false,true],"flapping":false,"pendingRecoveredCount":2}}' - ); + + expect(alertInstance).toMatchObject({ + state: { + foo: true, + }, + meta: { + lastScheduledActions: { + date: expect.any(Date), + group: 'default', + }, + uuid: expect.any(String), + flappingHistory: [false, true], + flapping: false, + pendingRecoveredCount: 2, + }, + }); }); }); @@ -514,6 +548,7 @@ describe('toRaw', () => { meta: { flappingHistory: [false, true, true], flapping: false, + uuid: expect.any(String), }, }); }); @@ -529,12 +564,13 @@ describe('setFlappingHistory', () => { ); alertInstance.setFlappingHistory([false]); expect(alertInstance.getFlappingHistory()).toEqual([false]); - expect(alertInstance.toRaw()).toMatchInlineSnapshot(` + expect(alertWithAnyUUID(alertInstance.toRaw())).toMatchInlineSnapshot(` Object { "meta": Object { "flappingHistory": Array [ false, ], + "uuid": Any, }, "state": Object {}, } @@ -561,11 +597,12 @@ describe('setFlapping', () => { ); alertInstance.setFlapping(false); expect(alertInstance.getFlapping()).toEqual(false); - expect(alertInstance.toRaw()).toMatchInlineSnapshot(` + expect(alertWithAnyUUID(alertInstance.toRaw())).toMatchInlineSnapshot(` Object { "meta": Object { "flapping": false, "flappingHistory": Array [], + "uuid": Any, }, "state": Object {}, } diff --git a/x-pack/plugins/alerting/server/alert/alert.ts b/x-pack/plugins/alerting/server/alert/alert.ts index 3a440f0b31f20..1fff89d527d5a 100644 --- a/x-pack/plugins/alerting/server/alert/alert.ts +++ b/x-pack/plugins/alerting/server/alert/alert.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { v4 as uuidV4 } from 'uuid'; import { get, isEmpty } from 'lodash'; import { ALERT_INSTANCE_ID } from '@kbn/rule-data-utils'; import { CombinedSummarizedAlerts } from '../types'; @@ -36,7 +37,13 @@ export type PublicAlert< ActionGroupIds extends string = DefaultActionGroupId > = Pick< Alert, - 'getState' | 'replaceState' | 'scheduleActions' | 'setContext' | 'getContext' | 'hasContext' + | 'getContext' + | 'getState' + | 'getUuid' + | 'hasContext' + | 'replaceState' + | 'scheduleActions' + | 'setContext' >; export class Alert< @@ -55,6 +62,7 @@ export class Alert< this.state = (state || {}) as State; this.context = {} as Context; this.meta = meta; + this.meta.uuid = meta.uuid ?? uuidV4(); if (!this.meta.flappingHistory) { this.meta.flappingHistory = []; @@ -65,6 +73,10 @@ export class Alert< return this.id; } + getUuid() { + return this.meta.uuid!; + } + hasScheduledActions() { return this.scheduledExecutionOptions !== undefined; } @@ -214,11 +226,12 @@ export class Alert< toRaw(recovered: boolean = false): RawAlertInstance { return recovered ? { - // for a recovered alert, we only care to track the flappingHistory - // and the flapping flag + // for a recovered alert, we only care to track the flappingHistory, + // the flapping flag, and the UUID meta: { flappingHistory: this.meta.flappingHistory, flapping: this.meta.flapping, + uuid: this.meta.uuid, }, } : { diff --git a/x-pack/plugins/alerting/server/alert/create_alert_factory.test.ts b/x-pack/plugins/alerting/server/alert/create_alert_factory.test.ts index c532d1a6640f2..63fd8c7bb1402 100644 --- a/x-pack/plugins/alerting/server/alert/create_alert_factory.test.ts +++ b/x-pack/plugins/alerting/server/alert/create_alert_factory.test.ts @@ -33,14 +33,15 @@ describe('createAlertFactory()', () => { autoRecoverAlerts: true, }); const result = alertFactory.create('1'); - expect(result).toMatchInlineSnapshot(` - Object { - "meta": Object { - "flappingHistory": Array [], - }, - "state": Object {}, - } - `); + expect(result).toMatchObject({ + meta: { + uuid: expect.any(String), + flappingHistory: [], + }, + state: {}, + context: {}, + id: '1', + }); // @ts-expect-error expect(result.getId()).toEqual('1'); }); @@ -48,7 +49,7 @@ describe('createAlertFactory()', () => { test('reuses existing alerts', () => { const alert = new Alert('1', { state: { foo: true }, - meta: { lastScheduledActions: { group: 'default', date: new Date() } }, + meta: { lastScheduledActions: { group: 'default', date: new Date() }, uuid: 'uuid-previous' }, }); const alertFactory = createAlertFactory({ alerts: { @@ -59,20 +60,19 @@ describe('createAlertFactory()', () => { autoRecoverAlerts: true, }); const result = alertFactory.create('1'); - expect(result).toMatchInlineSnapshot(` - Object { - "meta": Object { - "flappingHistory": Array [], - "lastScheduledActions": Object { - "date": "1970-01-01T00:00:00.000Z", - "group": "default", - }, - }, - "state": Object { - "foo": true, + expect(result).toMatchObject({ + meta: { + uuid: 'uuid-previous', + flappingHistory: [], + lastScheduledActions: { + date: expect.any(Date), + group: 'default', }, - } - `); + }, + state: { foo: true }, + context: {}, + id: '1', + }); }); test('mutates given alerts', () => { @@ -84,16 +84,17 @@ describe('createAlertFactory()', () => { autoRecoverAlerts: true, }); alertFactory.create('1'); - expect(alerts).toMatchInlineSnapshot(` - Object { - "1": Object { - "meta": Object { - "flappingHistory": Array [], - }, - "state": Object {}, + expect(alerts).toMatchObject({ + 1: { + meta: { + uuid: expect.any(String), + flappingHistory: [], }, - } - `); + state: {}, + context: {}, + id: '1', + }, + }); }); test('throws error and sets flag when more alerts are created than allowed', () => { @@ -124,9 +125,10 @@ describe('createAlertFactory()', () => { autoRecoverAlerts: true, }); const result = alertFactory.create('1'); - expect(result).toEqual({ + expect(result).toMatchObject({ meta: { flappingHistory: [], + uuid: expect.any(String), }, state: {}, context: {}, @@ -166,9 +168,10 @@ describe('createAlertFactory()', () => { autoRecoverAlerts: true, }); const result = alertFactory.create('1'); - expect(result).toEqual({ + expect(result).toMatchObject({ meta: { flappingHistory: [], + uuid: expect.any(String), }, state: {}, context: {}, @@ -193,9 +196,10 @@ describe('createAlertFactory()', () => { autoRecoverAlerts: true, }); const result = alertFactory.create('1'); - expect(result).toEqual({ + expect(result).toMatchObject({ meta: { flappingHistory: [], + uuid: expect.any(String), }, state: {}, context: {}, @@ -219,9 +223,10 @@ describe('createAlertFactory()', () => { autoRecoverAlerts: true, }); const result = alertFactory.create('1'); - expect(result).toEqual({ + expect(result).toMatchObject({ meta: { flappingHistory: [], + uuid: expect.any(String), }, state: {}, context: {}, @@ -244,9 +249,10 @@ describe('createAlertFactory()', () => { autoRecoverAlerts: true, }); const result = alertFactory.create('1'); - expect(result).toEqual({ + expect(result).toMatchObject({ meta: { flappingHistory: [], + uuid: expect.any(String), }, state: {}, context: {}, @@ -323,6 +329,7 @@ describe('createAlertFactory()', () => { expect(result).toEqual({ meta: { flappingHistory: [], + uuid: expect.any(String), }, state: {}, context: {}, diff --git a/x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.test.ts b/x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.test.ts index fc6518a5d0960..dbcb75a7816e7 100644 --- a/x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.test.ts +++ b/x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.test.ts @@ -125,6 +125,7 @@ describe('alertSummaryFromEventLog', () => { "flapping": false, "muted": true, "status": "OK", + "uuid": undefined, }, "alert-2": Object { "actionGroupId": undefined, @@ -132,6 +133,7 @@ describe('alertSummaryFromEventLog', () => { "flapping": false, "muted": true, "status": "OK", + "uuid": undefined, }, }, "lastRun": undefined, @@ -211,11 +213,11 @@ describe('alertSummaryFromEventLog', () => { const eventsFactory = new EventsFactory(); const events = eventsFactory .addExecute() - .addNewAlert('alert-1') - .addActiveAlert('alert-1', 'action group A') + .addNewAlert('alert-1', 'uuid-1') + .addActiveAlert('alert-1', 'action group A', 'uuid-1') .advanceTime(10000) .addExecute() - .addRecoveredAlert('alert-1') + .addRecoveredAlert('alert-1', 'uuid-1') .getEvents(); const executionEvents = eventsFactory.getEvents(); @@ -237,6 +239,7 @@ describe('alertSummaryFromEventLog', () => { "flapping": false, "muted": false, "status": "OK", + "uuid": "uuid-1", }, }, "lastRun": "2020-06-18T00:00:10.000Z", @@ -252,11 +255,11 @@ describe('alertSummaryFromEventLog', () => { const eventsFactory = new EventsFactory(); const events = eventsFactory .addExecute() - .addNewAlert('alert-1') - .addActiveAlert('alert-1', 'action group A') + .addNewAlert('alert-1', 'uuid-1') + .addActiveAlert('alert-1', 'action group A', 'uuid-1') .advanceTime(10000) .addExecute() - .addLegacyResolvedAlert('alert-1') + .addLegacyResolvedAlert('alert-1', 'uuid-1') .getEvents(); const executionEvents = eventsFactory.getEvents(); @@ -278,6 +281,7 @@ describe('alertSummaryFromEventLog', () => { "flapping": false, "muted": false, "status": "OK", + "uuid": "uuid-1", }, }, "lastRun": "2020-06-18T00:00:10.000Z", @@ -293,10 +297,10 @@ describe('alertSummaryFromEventLog', () => { const eventsFactory = new EventsFactory(); const events = eventsFactory .addExecute() - .addActiveAlert('alert-1', 'action group A') + .addActiveAlert('alert-1', 'action group A', 'uuid-1') .advanceTime(10000) .addExecute() - .addRecoveredAlert('alert-1') + .addRecoveredAlert('alert-1', 'uuid-1') .getEvents(); const executionEvents = eventsFactory.getEvents(); @@ -318,6 +322,7 @@ describe('alertSummaryFromEventLog', () => { "flapping": false, "muted": false, "status": "OK", + "uuid": "uuid-1", }, }, "lastRun": "2020-06-18T00:00:10.000Z", @@ -333,11 +338,11 @@ describe('alertSummaryFromEventLog', () => { const eventsFactory = new EventsFactory(); const events = eventsFactory .addExecute() - .addNewAlert('alert-1') - .addActiveAlert('alert-1', 'action group A') + .addNewAlert('alert-1', 'uuid-1') + .addActiveAlert('alert-1', 'action group A', 'uuid-1') .advanceTime(10000) .addExecute() - .addActiveAlert('alert-1', 'action group A') + .addActiveAlert('alert-1', 'action group A', 'uuid-1') .getEvents(); const executionEvents = eventsFactory.getEvents(); @@ -359,6 +364,7 @@ describe('alertSummaryFromEventLog', () => { "flapping": false, "muted": false, "status": "Active", + "uuid": "uuid-1", }, }, "lastRun": "2020-06-18T00:00:10.000Z", @@ -374,11 +380,11 @@ describe('alertSummaryFromEventLog', () => { const eventsFactory = new EventsFactory(); const events = eventsFactory .addExecute() - .addNewAlert('alert-1') - .addActiveAlert('alert-1', undefined) + .addNewAlert('alert-1', 'uuid-1') + .addActiveAlert('alert-1', undefined, 'uuid-1') .advanceTime(10000) .addExecute() - .addActiveAlert('alert-1', undefined) + .addActiveAlert('alert-1', undefined, 'uuid-1') .getEvents(); const executionEvents = eventsFactory.getEvents(); @@ -400,6 +406,7 @@ describe('alertSummaryFromEventLog', () => { "flapping": false, "muted": false, "status": "Active", + "uuid": "uuid-1", }, }, "lastRun": "2020-06-18T00:00:10.000Z", @@ -415,11 +422,11 @@ describe('alertSummaryFromEventLog', () => { const eventsFactory = new EventsFactory(); const events = eventsFactory .addExecute() - .addNewAlert('alert-1') - .addActiveAlert('alert-1', 'action group A') + .addNewAlert('alert-1', 'uuid-1') + .addActiveAlert('alert-1', 'action group A', 'uuid-1') .advanceTime(10000) .addExecute() - .addActiveAlert('alert-1', 'action group B') + .addActiveAlert('alert-1', 'action group B', 'uuid-1') .getEvents(); const executionEvents = eventsFactory.getEvents(); @@ -441,6 +448,7 @@ describe('alertSummaryFromEventLog', () => { "flapping": false, "muted": false, "status": "Active", + "uuid": "uuid-1", }, }, "lastRun": "2020-06-18T00:00:10.000Z", @@ -456,10 +464,10 @@ describe('alertSummaryFromEventLog', () => { const eventsFactory = new EventsFactory(); const events = eventsFactory .addExecute() - .addActiveAlert('alert-1', 'action group A') + .addActiveAlert('alert-1', 'action group A', 'uuid-1') .advanceTime(10000) .addExecute() - .addActiveAlert('alert-1', 'action group A') + .addActiveAlert('alert-1', 'action group A', 'uuid-1') .getEvents(); const executionEvents = eventsFactory.getEvents(); @@ -480,6 +488,7 @@ describe('alertSummaryFromEventLog', () => { "flapping": false, "muted": false, "status": "Active", + "uuid": "uuid-1", }, }, "lastRun": "2020-06-18T00:00:10.000Z", @@ -495,14 +504,14 @@ describe('alertSummaryFromEventLog', () => { const eventsFactory = new EventsFactory(); const events = eventsFactory .addExecute() - .addNewAlert('alert-1') - .addActiveAlert('alert-1', 'action group A') - .addNewAlert('alert-2') - .addActiveAlert('alert-2', 'action group B') + .addNewAlert('alert-1', 'uuid-1') + .addActiveAlert('alert-1', 'action group A', 'uuid-1') + .addNewAlert('alert-2', 'uuid-2') + .addActiveAlert('alert-2', 'action group B', 'uuid-2') .advanceTime(10000) .addExecute() - .addActiveAlert('alert-1', 'action group A') - .addRecoveredAlert('alert-2') + .addActiveAlert('alert-1', 'action group A', 'uuid-1') + .addRecoveredAlert('alert-2', 'uuid-2') .getEvents(); const executionEvents = eventsFactory.getEvents(); @@ -523,6 +532,7 @@ describe('alertSummaryFromEventLog', () => { "flapping": false, "muted": true, "status": "Active", + "uuid": "uuid-1", }, "alert-2": Object { "actionGroupId": undefined, @@ -530,6 +540,7 @@ describe('alertSummaryFromEventLog', () => { "flapping": false, "muted": true, "status": "OK", + "uuid": "uuid-2", }, }, "lastRun": "2020-06-18T00:00:10.000Z", @@ -545,20 +556,20 @@ describe('alertSummaryFromEventLog', () => { const eventsFactory = new EventsFactory(); const events = eventsFactory .addExecute() - .addNewAlert('alert-1') - .addActiveAlert('alert-1', 'action group A') - .addNewAlert('alert-2') - .addActiveAlert('alert-2', 'action group B') + .addNewAlert('alert-1', 'uuid-1') + .addActiveAlert('alert-1', 'action group A', 'uuid-1') + .addNewAlert('alert-2', 'uuid-2') + .addActiveAlert('alert-2', 'action group B', 'uuid-2') .advanceTime(10000) .addExecute() - .addActiveAlert('alert-1', 'action group A') - .addRecoveredAlert('alert-2') + .addActiveAlert('alert-1', 'action group A', 'uuid-1') + .addRecoveredAlert('alert-2', 'uuid-2') .advanceTime(10000) .addExecute() - .addActiveAlert('alert-1', 'action group B') + .addActiveAlert('alert-1', 'action group B', 'uuid-1') .advanceTime(10000) .addExecute() - .addActiveAlert('alert-1', 'action group B') + .addActiveAlert('alert-1', 'action group B', 'uuid-1') .getEvents(); const executionEvents = eventsFactory.getEvents(); @@ -580,6 +591,7 @@ describe('alertSummaryFromEventLog', () => { "flapping": false, "muted": false, "status": "Active", + "uuid": "uuid-1", }, "alert-2": Object { "actionGroupId": undefined, @@ -587,6 +599,7 @@ describe('alertSummaryFromEventLog', () => { "flapping": false, "muted": false, "status": "OK", + "uuid": "uuid-2", }, }, "lastRun": "2020-06-18T00:00:30.000Z", @@ -602,7 +615,7 @@ describe('alertSummaryFromEventLog', () => { const eventsFactory = new EventsFactory(); const events = eventsFactory .addExecute() - .addActiveAlert('alert-1', 'action group A', true) + .addActiveAlert('alert-1', 'action group A', 'uuid-1', true) .getEvents(); const executionEvents = eventsFactory.getEvents(); @@ -624,6 +637,7 @@ describe('alertSummaryFromEventLog', () => { "flapping": true, "muted": false, "status": "Active", + "uuid": "uuid-1", }, }, "lastRun": "2020-06-18T00:00:00.000Z", @@ -695,6 +709,7 @@ export class EventsFactory { addActiveAlert( alertId: string, actionGroupId: string | undefined, + uuid: string, flapping = false ): EventsFactory { const kibanaAlerting = actionGroupId @@ -706,43 +721,43 @@ export class EventsFactory { provider: EVENT_LOG_PROVIDER, action: EVENT_LOG_ACTIONS.activeInstance, }, - kibana: { alerting: kibanaAlerting, alert: { flapping } }, + kibana: { alerting: kibanaAlerting, alert: { flapping, uuid } }, }); return this; } - addNewAlert(alertId: string): EventsFactory { + addNewAlert(alertId: string, uuid: string): EventsFactory { this.events.push({ '@timestamp': this.date, event: { provider: EVENT_LOG_PROVIDER, action: EVENT_LOG_ACTIONS.newInstance, }, - kibana: { alerting: { instance_id: alertId } }, + kibana: { alerting: { instance_id: alertId }, alert: { uuid } }, }); return this; } - addRecoveredAlert(alertId: string): EventsFactory { + addRecoveredAlert(alertId: string, uuid: string): EventsFactory { this.events.push({ '@timestamp': this.date, event: { provider: EVENT_LOG_PROVIDER, action: EVENT_LOG_ACTIONS.recoveredInstance, }, - kibana: { alerting: { instance_id: alertId } }, + kibana: { alerting: { instance_id: alertId }, alert: { uuid } }, }); return this; } - addLegacyResolvedAlert(alertId: string): EventsFactory { + addLegacyResolvedAlert(alertId: string, uuid: string): EventsFactory { this.events.push({ '@timestamp': this.date, event: { provider: EVENT_LOG_PROVIDER, action: LEGACY_EVENT_LOG_ACTIONS.resolvedInstance, }, - kibana: { alerting: { instance_id: alertId } }, + kibana: { alerting: { instance_id: alertId }, alert: { uuid } }, }); return this; } diff --git a/x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.ts b/x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.ts index 12ea8df57eb8e..7c3df21a3281d 100644 --- a/x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.ts +++ b/x-pack/plugins/alerting/server/lib/alert_summary_from_event_log.ts @@ -79,7 +79,8 @@ export function alertSummaryFromEventLog(params: AlertSummaryFromEventLogParams) const alertId = event?.kibana?.alerting?.instance_id; if (alertId === undefined) continue; - const status = getAlertStatus(alerts, alertId); + const alertUuid = event?.kibana?.alert?.uuid; + const status = getAlertStatus(alerts, alertId, alertUuid); if (event?.kibana?.alert?.flapping) { status.flapping = true; @@ -149,10 +150,15 @@ export function alertSummaryFromEventLog(params: AlertSummaryFromEventLogParams) } // return an alert status object, creating and adding to the map if needed -function getAlertStatus(alerts: Map, alertId: string): AlertStatus { +function getAlertStatus( + alerts: Map, + alertId: string, + alertUuid?: string +): AlertStatus { if (alerts.has(alertId)) return alerts.get(alertId)!; const status: AlertStatus = { + uuid: alertUuid, status: 'OK', muted: false, actionGroupId: undefined, diff --git a/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.test.ts b/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.test.ts index 11f57e8145e95..90c3645e151e5 100644 --- a/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.test.ts +++ b/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.test.ts @@ -59,6 +59,7 @@ const contextWithName = { ...contextWithScheduleDelay, ruleName: 'my-super-cool- const alert = { action: EVENT_LOG_ACTIONS.activeInstance, id: 'aaabbb', + uuid: 'u-u-i-d', message: `.test-rule-type:123: 'my rule' active alert: 'aaabbb' in actionGroup: 'aGroup';`, group: 'aGroup', state: { @@ -1089,6 +1090,7 @@ describe('createAlertRecord', () => { expect(record.event?.provider).toBeUndefined(); expect(record.event?.outcome).toBeUndefined(); expect(record.kibana?.alert?.rule?.execution?.metrics).toBeUndefined(); + expect(record.kibana?.alert?.uuid).toBe(alert.uuid); expect(record.kibana?.server_uuid).toBeUndefined(); expect(record.kibana?.task).toBeUndefined(); expect(record.kibana?.version).toBeUndefined(); diff --git a/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts b/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts index dd8c52bebc0e0..f84936e7ea6e4 100644 --- a/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts +++ b/x-pack/plugins/alerting/server/lib/alerting_event_logger/alerting_event_logger.ts @@ -45,6 +45,7 @@ interface DoneOpts { interface AlertOpts { action: string; id: string; + uuid: string; message: string; group?: string; state?: AlertInstanceState; @@ -239,6 +240,7 @@ export function createAlertRecord(context: RuleContextOpts, alert: AlertOpts) { namespace: context.namespace, spaceId: context.spaceId, executionId: context.executionId, + alertUuid: alert.uuid, action: alert.action, state: alert.state, instanceId: alert.id, diff --git a/x-pack/plugins/alerting/server/lib/create_alert_event_log_record_object.ts b/x-pack/plugins/alerting/server/lib/create_alert_event_log_record_object.ts index 280f498422fd0..f987c297d0217 100644 --- a/x-pack/plugins/alerting/server/lib/create_alert_event_log_record_object.ts +++ b/x-pack/plugins/alerting/server/lib/create_alert_event_log_record_object.ts @@ -25,6 +25,7 @@ interface CreateAlertEventLogRecordParams { group?: string; namespace?: string; timestamp?: string; + alertUuid?: string; task?: { scheduled?: string; scheduleDelay?: number; @@ -57,6 +58,7 @@ export function createAlertEventLogRecordObject(params: CreateAlertEventLogRecor consumer, spaceId, flapping, + alertUuid, alertSummary, } = params; const alerting = @@ -90,6 +92,7 @@ export function createAlertEventLogRecordObject(params: CreateAlertEventLogRecor kibana: { alert: { ...(flapping !== undefined ? { flapping } : {}), + ...(alertUuid ? { uuid: alertUuid } : {}), rule: { rule_type_id: ruleType.id, ...(consumer ? { consumer } : {}), diff --git a/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.test.ts b/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.test.ts index 17c636a187edf..9da07f4707d45 100644 --- a/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.test.ts +++ b/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.test.ts @@ -8,12 +8,15 @@ import { DEFAULT_FLAPPING_SETTINGS, DISABLE_FLAPPING_SETTINGS } from '../../common/rules_settings'; import { getAlertsForNotification } from '.'; import { Alert } from '../alert'; +import { alertsWithAnyUUID } from '../test_utils'; import { RuleNotifyWhen } from '../types'; describe('getAlertsForNotification', () => { test('should set pendingRecoveredCount to zero for all active alerts', () => { - const alert1 = new Alert('1', { meta: { flapping: true, pendingRecoveredCount: 3 } }); - const alert2 = new Alert('2', { meta: { flapping: false } }); + const alert1 = new Alert('1', { + meta: { flapping: true, pendingRecoveredCount: 3, uuid: 'uuid-1' }, + }); + const alert2 = new Alert('2', { meta: { flapping: false, uuid: 'uuid-2' } }); const { newAlerts, activeAlerts } = getAlertsForNotification( DEFAULT_FLAPPING_SETTINGS, @@ -36,6 +39,7 @@ describe('getAlertsForNotification', () => { "flapping": true, "flappingHistory": Array [], "pendingRecoveredCount": 0, + "uuid": "uuid-1", }, "state": Object {}, }, @@ -48,6 +52,7 @@ describe('getAlertsForNotification', () => { "flapping": true, "flappingHistory": Array [], "pendingRecoveredCount": 0, + "uuid": "uuid-1", }, "state": Object {}, }, @@ -56,6 +61,7 @@ describe('getAlertsForNotification', () => { "flapping": false, "flappingHistory": Array [], "pendingRecoveredCount": 0, + "uuid": "uuid-2", }, "state": Object {}, }, @@ -92,14 +98,15 @@ describe('getAlertsForNotification', () => { } ); - expect(newAlerts).toMatchInlineSnapshot(`Object {}`); - expect(activeAlerts).toMatchInlineSnapshot(` + expect(alertsWithAnyUUID(newAlerts)).toMatchInlineSnapshot(`Object {}`); + expect(alertsWithAnyUUID(activeAlerts)).toMatchInlineSnapshot(` Object { "3": Object { "meta": Object { "flapping": true, "flappingHistory": Array [], "pendingRecoveredCount": 1, + "uuid": Any, }, "state": Object {}, }, @@ -115,13 +122,14 @@ describe('getAlertsForNotification', () => { }, ] `); - expect(currentActiveAlerts).toMatchInlineSnapshot(` + expect(alertsWithAnyUUID(currentActiveAlerts)).toMatchInlineSnapshot(` Object { "3": Object { "meta": Object { "flapping": true, "flappingHistory": Array [], "pendingRecoveredCount": 1, + "uuid": Any, }, "state": Object {}, }, @@ -137,13 +145,14 @@ describe('getAlertsForNotification', () => { }, ] `); - expect(recoveredAlerts).toMatchInlineSnapshot(` + expect(alertsWithAnyUUID(recoveredAlerts)).toMatchInlineSnapshot(` Object { "1": Object { "meta": Object { "flapping": true, "flappingHistory": Array [], "pendingRecoveredCount": 0, + "uuid": Any, }, "state": Object {}, }, @@ -151,18 +160,20 @@ describe('getAlertsForNotification', () => { "meta": Object { "flapping": false, "flappingHistory": Array [], + "uuid": Any, }, "state": Object {}, }, } `); - expect(currentRecoveredAlerts).toMatchInlineSnapshot(` + expect(alertsWithAnyUUID(currentRecoveredAlerts)).toMatchInlineSnapshot(` Object { "1": Object { "meta": Object { "flapping": true, "flappingHistory": Array [], "pendingRecoveredCount": 0, + "uuid": Any, }, "state": Object {}, }, @@ -170,6 +181,7 @@ describe('getAlertsForNotification', () => { "meta": Object { "flapping": false, "flappingHistory": Array [], + "uuid": Any, }, "state": Object {}, }, @@ -209,7 +221,7 @@ describe('getAlertsForNotification', () => { expect(newAlerts).toMatchInlineSnapshot(`Object {}`); expect(activeAlerts).toMatchInlineSnapshot(`Object {}`); - expect(recoveredAlerts).toMatchInlineSnapshot(` + expect(alertsWithAnyUUID(recoveredAlerts)).toMatchInlineSnapshot(` Object { "1": Object { "meta": Object { @@ -220,6 +232,7 @@ describe('getAlertsForNotification', () => { true, ], "pendingRecoveredCount": 0, + "uuid": Any, }, "state": Object {}, }, @@ -232,6 +245,7 @@ describe('getAlertsForNotification', () => { true, ], "pendingRecoveredCount": 0, + "uuid": Any, }, "state": Object {}, }, @@ -244,12 +258,13 @@ describe('getAlertsForNotification', () => { true, ], "pendingRecoveredCount": 0, + "uuid": Any, }, "state": Object {}, }, } `); - expect(currentRecoveredAlerts).toMatchInlineSnapshot(` + expect(alertsWithAnyUUID(currentRecoveredAlerts)).toMatchInlineSnapshot(` Object { "1": Object { "meta": Object { @@ -260,6 +275,7 @@ describe('getAlertsForNotification', () => { true, ], "pendingRecoveredCount": 0, + "uuid": Any, }, "state": Object {}, }, @@ -272,6 +288,7 @@ describe('getAlertsForNotification', () => { true, ], "pendingRecoveredCount": 0, + "uuid": Any, }, "state": Object {}, }, @@ -284,6 +301,7 @@ describe('getAlertsForNotification', () => { true, ], "pendingRecoveredCount": 0, + "uuid": Any, }, "state": Object {}, }, @@ -321,13 +339,14 @@ describe('getAlertsForNotification', () => { ); expect(newAlerts).toMatchInlineSnapshot(`Object {}`); - expect(activeAlerts).toMatchInlineSnapshot(` + expect(alertsWithAnyUUID(activeAlerts)).toMatchInlineSnapshot(` Object { "3": Object { "meta": Object { "flapping": true, "flappingHistory": Array [], "pendingRecoveredCount": 1, + "uuid": Any, }, "state": Object {}, }, @@ -347,13 +366,14 @@ describe('getAlertsForNotification', () => { expect( Object.values(currentActiveAlerts).map((a) => a.getScheduledActionOptions()) ).toMatchInlineSnapshot(`Array []`); - expect(recoveredAlerts).toMatchInlineSnapshot(` + expect(alertsWithAnyUUID(recoveredAlerts)).toMatchInlineSnapshot(` Object { "1": Object { "meta": Object { "flapping": true, "flappingHistory": Array [], "pendingRecoveredCount": 0, + "uuid": Any, }, "state": Object {}, }, @@ -361,18 +381,20 @@ describe('getAlertsForNotification', () => { "meta": Object { "flapping": false, "flappingHistory": Array [], + "uuid": Any, }, "state": Object {}, }, } `); - expect(currentRecoveredAlerts).toMatchInlineSnapshot(` + expect(alertsWithAnyUUID(currentRecoveredAlerts)).toMatchInlineSnapshot(` Object { "1": Object { "meta": Object { "flapping": true, "flappingHistory": Array [], "pendingRecoveredCount": 0, + "uuid": Any, }, "state": Object {}, }, @@ -380,6 +402,7 @@ describe('getAlertsForNotification', () => { "meta": Object { "flapping": false, "flappingHistory": Array [], + "uuid": Any, }, "state": Object {}, }, diff --git a/x-pack/plugins/alerting/server/lib/process_alerts.test.ts b/x-pack/plugins/alerting/server/lib/process_alerts.test.ts index b1136b6acde48..2aaed9b915111 100644 --- a/x-pack/plugins/alerting/server/lib/process_alerts.test.ts +++ b/x-pack/plugins/alerting/server/lib/process_alerts.test.ts @@ -719,7 +719,9 @@ describe('processAlerts', () => { describe('updating flappingHistory', () => { test('if new alert, set flapping state to true', () => { - const activeAlert = new Alert('1'); + const activeAlert = new Alert('1', { + meta: { uuid: 'uuid-1' }, + }); const alerts = cloneDeep({ '1': activeAlert }); alerts['1'].scheduleActions('default' as never, { foo: '1' }); @@ -741,6 +743,7 @@ describe('processAlerts', () => { "flappingHistory": Array [ true, ], + "uuid": "uuid-1", }, "state": Object { "duration": "0", @@ -756,6 +759,7 @@ describe('processAlerts', () => { "flappingHistory": Array [ true, ], + "uuid": "uuid-1", }, "state": Object { "duration": "0", @@ -769,7 +773,7 @@ describe('processAlerts', () => { test('if alert is still active, set flapping state to false', () => { const activeAlert = new Alert('1', { - meta: { flappingHistory: [false] }, + meta: { flappingHistory: [false], uuid: 'uuid-1' }, }); const alerts = cloneDeep({ '1': activeAlert }); @@ -793,6 +797,7 @@ describe('processAlerts', () => { false, false, ], + "uuid": "uuid-1", }, "state": Object {}, }, @@ -803,9 +808,11 @@ describe('processAlerts', () => { }); test('if alert is active and previously recovered, set flapping state to true', () => { - const activeAlert = new Alert('1'); + const activeAlert = new Alert('1', { + meta: { uuid: 'uuid-1' }, + }); const recoveredAlert = new Alert('1', { - meta: { flappingHistory: [false] }, + meta: { flappingHistory: [false], uuid: 'uuid-2' }, }); const alerts = cloneDeep({ '1': activeAlert }); @@ -830,6 +837,7 @@ describe('processAlerts', () => { false, true, ], + "uuid": "uuid-1", }, "state": Object { "duration": "0", @@ -846,6 +854,7 @@ describe('processAlerts', () => { false, true, ], + "uuid": "uuid-1", }, "state": Object { "duration": "0", @@ -859,11 +868,11 @@ describe('processAlerts', () => { test('if alert is recovered and previously active, set flapping state to true', () => { const activeAlert = new Alert('1', { - meta: { flappingHistory: [false] }, + meta: { flappingHistory: [false], uuid: 'uuid-1' }, }); activeAlert.scheduleActions('default' as never, { foo: '1' }); const recoveredAlert = new Alert('1', { - meta: { flappingHistory: [false] }, + meta: { flappingHistory: [false], uuid: 'uuid-1' }, }); const alerts = cloneDeep({ '1': recoveredAlert }); @@ -888,6 +897,7 @@ describe('processAlerts', () => { false, true, ], + "uuid": "uuid-1", }, "state": Object {}, }, @@ -897,7 +907,7 @@ describe('processAlerts', () => { test('if alert is still recovered, set flapping state to false', () => { const recoveredAlert = new Alert('1', { - meta: { flappingHistory: [false] }, + meta: { flappingHistory: [false], uuid: 'uuid-1' }, }); const alerts = cloneDeep({ '1': recoveredAlert }); @@ -922,6 +932,7 @@ describe('processAlerts', () => { false, false, ], + "uuid": "uuid-1", }, "state": Object {}, }, @@ -930,14 +941,16 @@ describe('processAlerts', () => { }); test('if setFlapping is false should not update flappingHistory', () => { - const activeAlert1 = new Alert('1'); + const activeAlert1 = new Alert('1', { + meta: { uuid: 'uuid-1' }, + }); activeAlert1.scheduleActions('default' as never, { foo: '1' }); const activeAlert2 = new Alert('2', { - meta: { flappingHistory: [false] }, + meta: { flappingHistory: [false], uuid: 'uuid-2' }, }); activeAlert2.scheduleActions('default' as never, { foo: '1' }); const recoveredAlert = new Alert('3', { - meta: { flappingHistory: [false] }, + meta: { flappingHistory: [false], uuid: 'uuid-3' }, }); const previouslyRecoveredAlerts = cloneDeep({ '3': recoveredAlert }); @@ -959,6 +972,7 @@ describe('processAlerts', () => { "1": Object { "meta": Object { "flappingHistory": Array [], + "uuid": "uuid-1", }, "state": Object { "duration": "0", @@ -970,6 +984,7 @@ describe('processAlerts', () => { "flappingHistory": Array [ false, ], + "uuid": "uuid-2", }, "state": Object {}, }, @@ -980,6 +995,7 @@ describe('processAlerts', () => { "1": Object { "meta": Object { "flappingHistory": Array [], + "uuid": "uuid-1", }, "state": Object { "duration": "0", @@ -995,6 +1011,7 @@ describe('processAlerts', () => { "flappingHistory": Array [ false, ], + "uuid": "uuid-3", }, "state": Object {}, }, @@ -1005,7 +1022,7 @@ describe('processAlerts', () => { describe('when hasReachedAlertLimit is true', () => { test('if alert is still active, set flapping state to false', () => { const activeAlert = new Alert('1', { - meta: { flappingHistory: [false] }, + meta: { flappingHistory: [false], uuid: 'uuid-1' }, }); const alerts = cloneDeep({ '1': activeAlert }); @@ -1029,6 +1046,7 @@ describe('processAlerts', () => { false, false, ], + "uuid": "uuid-1", }, "state": Object {}, }, @@ -1040,10 +1058,12 @@ describe('processAlerts', () => { test('if new alert, set flapping state to true', () => { const activeAlert1 = new Alert('1', { - meta: { flappingHistory: [false] }, + meta: { flappingHistory: [false], uuid: 'uuid-1' }, }); activeAlert1.scheduleActions('default' as never, { foo: '1' }); - const activeAlert2 = new Alert('1'); + const activeAlert2 = new Alert('2', { + meta: { flappingHistory: [false], uuid: 'uuid-2' }, + }); activeAlert2.scheduleActions('default' as never, { foo: '1' }); const alerts = cloneDeep({ '1': activeAlert1, '2': activeAlert2 }); @@ -1066,14 +1086,17 @@ describe('processAlerts', () => { false, false, ], + "uuid": "uuid-1", }, "state": Object {}, }, "2": Object { "meta": Object { "flappingHistory": Array [ + false, true, ], + "uuid": "uuid-2", }, "state": Object { "duration": "0", @@ -1087,8 +1110,10 @@ describe('processAlerts', () => { "2": Object { "meta": Object { "flappingHistory": Array [ + false, true, ], + "uuid": "uuid-2", }, "state": Object { "duration": "0", @@ -1102,10 +1127,12 @@ describe('processAlerts', () => { test('if alert is active and previously recovered, set flapping state to true', () => { const activeAlert1 = new Alert('1', { - meta: { flappingHistory: [false] }, + meta: { flappingHistory: [false], uuid: 'uuid-1' }, }); activeAlert1.scheduleActions('default' as never, { foo: '1' }); - const activeAlert2 = new Alert('1'); + const activeAlert2 = new Alert('1', { + meta: { uuid: 'uuid-2' }, + }); activeAlert2.scheduleActions('default' as never, { foo: '1' }); const alerts = cloneDeep({ '1': activeAlert1, '2': activeAlert2 }); @@ -1128,6 +1155,7 @@ describe('processAlerts', () => { false, true, ], + "uuid": "uuid-1", }, "state": Object { "duration": "0", @@ -1139,6 +1167,7 @@ describe('processAlerts', () => { "flappingHistory": Array [ true, ], + "uuid": "uuid-2", }, "state": Object { "duration": "0", @@ -1155,6 +1184,7 @@ describe('processAlerts', () => { false, true, ], + "uuid": "uuid-1", }, "state": Object { "duration": "0", @@ -1166,6 +1196,7 @@ describe('processAlerts', () => { "flappingHistory": Array [ true, ], + "uuid": "uuid-2", }, "state": Object { "duration": "0", @@ -1179,10 +1210,12 @@ describe('processAlerts', () => { test('if setFlapping is false should not update flappingHistory', () => { const activeAlert1 = new Alert('1', { - meta: { flappingHistory: [false] }, + meta: { flappingHistory: [false], uuid: 'uuid-1' }, }); activeAlert1.scheduleActions('default' as never, { foo: '1' }); - const activeAlert2 = new Alert('1'); + const activeAlert2 = new Alert('1', { + meta: { uuid: 'uuid-2' }, + }); activeAlert2.scheduleActions('default' as never, { foo: '1' }); const alerts = cloneDeep({ '1': activeAlert1, '2': activeAlert2 }); @@ -1204,12 +1237,14 @@ describe('processAlerts', () => { "flappingHistory": Array [ false, ], + "uuid": "uuid-1", }, "state": Object {}, }, "2": Object { "meta": Object { "flappingHistory": Array [], + "uuid": "uuid-2", }, "state": Object { "duration": "0", @@ -1223,6 +1258,7 @@ describe('processAlerts', () => { "2": Object { "meta": Object { "flappingHistory": Array [], + "uuid": "uuid-2", }, "state": Object { "duration": "0", diff --git a/x-pack/plugins/alerting/server/lib/trim_recovered_alerts.test.ts b/x-pack/plugins/alerting/server/lib/trim_recovered_alerts.test.ts index 3c42a5664477c..9ed58b968f147 100644 --- a/x-pack/plugins/alerting/server/lib/trim_recovered_alerts.test.ts +++ b/x-pack/plugins/alerting/server/lib/trim_recovered_alerts.test.ts @@ -40,7 +40,11 @@ describe('trimRecoveredAlerts', () => { trimmedAlertsRecovered: { 1: alert1, 3: alert3 }, earlyRecoveredAlerts: { 2: new Alert('2', { - meta: { flappingHistory: new Array(20).fill(false), flapping: false }, + meta: { + flappingHistory: new Array(20).fill(false), + flapping: false, + uuid: expect.any(String), + }, }), }, }); diff --git a/x-pack/plugins/alerting/server/mocks.ts b/x-pack/plugins/alerting/server/mocks.ts index a5cadcd9bafa1..0a224d9120ff6 100644 --- a/x-pack/plugins/alerting/server/mocks.ts +++ b/x-pack/plugins/alerting/server/mocks.ts @@ -80,6 +80,7 @@ export const createAlertFactoryMock = { getScheduledActionOptions: jest.fn(), unscheduleActions: jest.fn(), getState: jest.fn(), + getUuid: jest.fn(), scheduleActions: jest.fn(), replaceState: jest.fn(), updateLastScheduledActions: jest.fn(), diff --git a/x-pack/plugins/alerting/server/rules_client/lib/recover_rule_alerts.ts b/x-pack/plugins/alerting/server/rules_client/lib/recover_rule_alerts.ts index e7027905ff956..bf8d10a341597 100644 --- a/x-pack/plugins/alerting/server/rules_client/lib/recover_rule_alerts.ts +++ b/x-pack/plugins/alerting/server/rules_client/lib/recover_rule_alerts.ts @@ -38,6 +38,7 @@ export const recoverRuleAlerts = async ( const { group: actionGroup } = recoveredAlerts[alertId].getLastScheduledActions() ?? {}; const instanceState = recoveredAlerts[alertId].getState(); const message = `instance '${alertId}' has recovered due to the rule was disabled`; + const alertUuid = recoveredAlerts[alertId].getUuid(); const event = createAlertEventLogRecordObject({ ruleId: id, @@ -45,6 +46,7 @@ export const recoverRuleAlerts = async ( ruleType: context.ruleTypeRegistry.get(attributes.alertTypeId), consumer: attributes.consumer, instanceId: alertId, + alertUuid, action: EVENT_LOG_ACTIONS.recoveredInstance, message, state: instanceState, diff --git a/x-pack/plugins/alerting/server/rules_client/tests/disable.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/disable.test.ts index 5af5ec3e60bd1..57906bcae3b97 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/disable.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/disable.test.ts @@ -253,6 +253,7 @@ describe('disable()', () => { group: 'default', date: new Date().toISOString(), }, + uuid: 'uuid-1', }, state: { bar: false }, }, @@ -313,6 +314,7 @@ describe('disable()', () => { }, kibana: { alert: { + uuid: 'uuid-1', rule: { consumer: 'myApp', rule_type_id: '123', diff --git a/x-pack/plugins/alerting/server/rules_client/tests/get_alert_summary.test.ts b/x-pack/plugins/alerting/server/rules_client/tests/get_alert_summary.test.ts index c7e2c65980afb..f9dcfaeb77a50 100644 --- a/x-pack/plugins/alerting/server/rules_client/tests/get_alert_summary.test.ts +++ b/x-pack/plugins/alerting/server/rules_client/tests/get_alert_summary.test.ts @@ -122,14 +122,14 @@ describe('getAlertSummary()', () => { const eventsFactory = new EventsFactory(mockedDateString); const events = eventsFactory .addExecute() - .addNewAlert('alert-currently-active') - .addNewAlert('alert-previously-active') - .addActiveAlert('alert-currently-active', 'action group A') - .addActiveAlert('alert-previously-active', 'action group B') + .addNewAlert('alert-currently-active', 'uuid-1') + .addNewAlert('alert-previously-active', 'uuid-2') + .addActiveAlert('alert-currently-active', 'action group A', 'uuid-1') + .addActiveAlert('alert-previously-active', 'action group B', 'uuid-2') .advanceTime(10000) .addExecute() - .addRecoveredAlert('alert-previously-active') - .addActiveAlert('alert-currently-active', 'action group A', true) + .addRecoveredAlert('alert-previously-active', 'uuid-2') + .addActiveAlert('alert-currently-active', 'action group A', 'uuid-1', true) .getEvents(); const eventsResult = { ...AlertSummaryFindEventsResult, @@ -161,6 +161,7 @@ describe('getAlertSummary()', () => { "flapping": true, "muted": false, "status": "Active", + "uuid": "uuid-1", }, "alert-muted-no-activity": Object { "actionGroupId": undefined, @@ -168,6 +169,7 @@ describe('getAlertSummary()', () => { "flapping": false, "muted": true, "status": "OK", + "uuid": undefined, }, "alert-previously-active": Object { "actionGroupId": undefined, @@ -175,6 +177,7 @@ describe('getAlertSummary()', () => { "flapping": false, "muted": false, "status": "OK", + "uuid": "uuid-2", }, }, "consumer": "rule-consumer", diff --git a/x-pack/plugins/alerting/server/task_runner/execution_handler.test.ts b/x-pack/plugins/alerting/server/task_runner/execution_handler.test.ts index d46edd8f43223..de3e05dae82ce 100644 --- a/x-pack/plugins/alerting/server/task_runner/execution_handler.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/execution_handler.test.ts @@ -112,6 +112,7 @@ const defaultExecutionParams = { apiKey, ruleConsumer: 'rule-consumer', executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', + alertUuid: 'uuid-1', ruleLabel: 'rule-label', request: {} as KibanaRequest, alertingEventLogger, diff --git a/x-pack/plugins/alerting/server/task_runner/execution_handler.ts b/x-pack/plugins/alerting/server/task_runner/execution_handler.ts index d1a81964b11ba..18c06c0b36f56 100644 --- a/x-pack/plugins/alerting/server/task_runner/execution_handler.ts +++ b/x-pack/plugins/alerting/server/task_runner/execution_handler.ts @@ -261,6 +261,7 @@ export class ExecutionHandler< spaceId, tags: this.rule.tags, alertInstanceId: executableAlert.getId(), + alertUuid: executableAlert.getUuid(), alertActionGroup: actionGroup, alertActionGroupName: this.ruleTypeActionGroups!.get(actionGroup)!, context: executableAlert.getContext(), diff --git a/x-pack/plugins/alerting/server/task_runner/fixtures.ts b/x-pack/plugins/alerting/server/task_runner/fixtures.ts index 5289a92c83585..dacddea634fa4 100644 --- a/x-pack/plugins/alerting/server/task_runner/fixtures.ts +++ b/x-pack/plugins/alerting/server/task_runner/fixtures.ts @@ -250,6 +250,7 @@ export const generateAlertOpts = ({ action, group, state, id }: GeneratorParams return { action, id, + uuid: expect.any(String), message, state, ...(group ? { group } : {}), @@ -355,6 +356,7 @@ export const generateAlertInstance = ( ) => ({ [String(id)]: { meta: { + uuid: expect.any(String), lastScheduledActions: { date: new Date(DATE_1970), group: 'default', diff --git a/x-pack/plugins/alerting/server/task_runner/log_alerts.test.ts b/x-pack/plugins/alerting/server/task_runner/log_alerts.test.ts index 163cadf1d084b..ffeb5db16fed3 100644 --- a/x-pack/plugins/alerting/server/task_runner/log_alerts.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/log_alerts.test.ts @@ -5,6 +5,13 @@ * 2.0. */ +jest.mock('uuid', () => { + let counter = 1; + return { + v4: () => `uuid-module-v4-called-${counter++}`, + }; +}); + import { loggingSystemMock } from '@kbn/core-logging-server-mocks'; import { Alert } from '../alert'; import { alertingEventLoggerMock } from '../lib/alerting_event_logger/alerting_event_logger.mock'; @@ -124,6 +131,8 @@ describe('logAlerts', () => { }); test('should correctly set values in ruleRunMetricsStore and call alertingEventLogger.logAlert if shouldPersistAlerts is true', () => { + jest.clearAllMocks(); + logAlerts({ logger, alertingEventLogger, @@ -131,15 +140,15 @@ describe('logAlerts', () => { '4': new Alert<{}, {}, DefaultActionGroupId>('4'), }, activeAlerts: { - '1': new Alert<{}, {}, DefaultActionGroupId>('1'), - '2': new Alert<{}, {}, DefaultActionGroupId>('2'), + '1': new Alert<{}, {}, DefaultActionGroupId>('1', { meta: { uuid: 'uuid-1' } }), + '2': new Alert<{}, {}, DefaultActionGroupId>('2', { meta: { uuid: 'uuid-2' } }), '4': new Alert<{}, {}, DefaultActionGroupId>('4'), }, recoveredAlerts: { - '7': new Alert<{}, {}, DefaultActionGroupId>('7'), - '8': new Alert<{}, {}, DefaultActionGroupId>('8'), - '9': new Alert<{}, {}, DefaultActionGroupId>('9'), - '10': new Alert<{}, {}, DefaultActionGroupId>('10'), + '7': new Alert<{}, {}, DefaultActionGroupId>('7', { meta: { uuid: 'uuid-7' } }), + '8': new Alert<{}, {}, DefaultActionGroupId>('8', { meta: { uuid: 'uuid-8' } }), + '9': new Alert<{}, {}, DefaultActionGroupId>('9', { meta: { uuid: 'uuid-9' } }), + '10': new Alert<{}, {}, DefaultActionGroupId>('10', { meta: { uuid: 'uuid-10' } }), }, ruleLogPrefix: `test-rule-type-id:123: 'test rule'`, ruleRunMetricsStore, @@ -159,6 +168,7 @@ describe('logAlerts', () => { message: "test-rule-type-id:123: 'test rule' alert '7' has recovered", state: {}, flapping: false, + uuid: 'uuid-7', }); expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(2, { action: 'recovered-instance', @@ -166,6 +176,7 @@ describe('logAlerts', () => { message: "test-rule-type-id:123: 'test rule' alert '8' has recovered", state: {}, flapping: false, + uuid: 'uuid-8', }); expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(3, { action: 'recovered-instance', @@ -173,6 +184,7 @@ describe('logAlerts', () => { message: "test-rule-type-id:123: 'test rule' alert '9' has recovered", state: {}, flapping: false, + uuid: 'uuid-9', }); expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(4, { action: 'recovered-instance', @@ -180,6 +192,7 @@ describe('logAlerts', () => { message: "test-rule-type-id:123: 'test rule' alert '10' has recovered", state: {}, flapping: false, + uuid: 'uuid-10', }); expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(5, { action: 'new-instance', @@ -187,6 +200,7 @@ describe('logAlerts', () => { message: "test-rule-type-id:123: 'test rule' created new alert: '4'", state: {}, flapping: false, + uuid: expect.any(String), }); expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(6, { action: 'active-instance', @@ -194,6 +208,7 @@ describe('logAlerts', () => { message: "test-rule-type-id:123: 'test rule' active alert: '1' in actionGroup: 'undefined'", state: {}, flapping: false, + uuid: 'uuid-1', }); expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(7, { action: 'active-instance', @@ -201,6 +216,7 @@ describe('logAlerts', () => { message: "test-rule-type-id:123: 'test rule' active alert: '2' in actionGroup: 'undefined'", state: {}, flapping: false, + uuid: 'uuid-2', }); expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(8, { action: 'active-instance', @@ -208,7 +224,14 @@ describe('logAlerts', () => { message: "test-rule-type-id:123: 'test rule' active alert: '4' in actionGroup: 'undefined'", state: {}, flapping: false, + uuid: expect.any(String), }); + + // check the two calls for alert 4 used the same UUID + const actualUuid1 = alertingEventLogger.logAlert.mock.calls[4][0].uuid; + const actualUuid2 = alertingEventLogger.logAlert.mock.calls[7][0].uuid; + expect(actualUuid1).toEqual(actualUuid2); + expect(actualUuid1).toMatch(/^uuid-module-v4-called-\d+$/); }); test('should not call alertingEventLogger.logAlert or update ruleRunMetricsStore if shouldPersistAlerts is false', () => { @@ -272,6 +295,8 @@ describe('logAlerts', () => { message: "test-rule-type-id:123: 'test rule' alert '7' has recovered", state: {}, flapping: false, + group: undefined, + uuid: expect.any(String), }); expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(2, { action: 'recovered-instance', @@ -279,6 +304,8 @@ describe('logAlerts', () => { message: "test-rule-type-id:123: 'test rule' alert '8' has recovered", state: {}, flapping: true, + group: undefined, + uuid: expect.any(String), }); expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(3, { action: 'recovered-instance', @@ -286,6 +313,8 @@ describe('logAlerts', () => { message: "test-rule-type-id:123: 'test rule' alert '9' has recovered", state: {}, flapping: false, + group: undefined, + uuid: expect.any(String), }); expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(4, { action: 'recovered-instance', @@ -293,6 +322,8 @@ describe('logAlerts', () => { message: "test-rule-type-id:123: 'test rule' alert '10' has recovered", state: {}, flapping: false, + group: undefined, + uuid: expect.any(String), }); expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(5, { action: 'new-instance', @@ -300,6 +331,8 @@ describe('logAlerts', () => { message: "test-rule-type-id:123: 'test rule' created new alert: '4'", state: {}, flapping: false, + group: undefined, + uuid: expect.any(String), }); expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(6, { action: 'active-instance', @@ -307,6 +340,8 @@ describe('logAlerts', () => { message: "test-rule-type-id:123: 'test rule' active alert: '1' in actionGroup: 'undefined'", state: {}, flapping: true, + group: undefined, + uuid: expect.any(String), }); expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(7, { action: 'active-instance', @@ -314,6 +349,8 @@ describe('logAlerts', () => { message: "test-rule-type-id:123: 'test rule' active alert: '2' in actionGroup: 'undefined'", state: {}, flapping: false, + group: undefined, + uuid: expect.any(String), }); expect(alertingEventLogger.logAlert).toHaveBeenNthCalledWith(8, { action: 'active-instance', @@ -321,6 +358,8 @@ describe('logAlerts', () => { message: "test-rule-type-id:123: 'test rule' active alert: '4' in actionGroup: 'undefined'", state: {}, flapping: false, + group: undefined, + uuid: expect.any(String), }); }); }); diff --git a/x-pack/plugins/alerting/server/task_runner/log_alerts.ts b/x-pack/plugins/alerting/server/task_runner/log_alerts.ts index 7f8f5d93a5d9f..a939ad511da34 100644 --- a/x-pack/plugins/alerting/server/task_runner/log_alerts.ts +++ b/x-pack/plugins/alerting/server/task_runner/log_alerts.ts @@ -91,12 +91,15 @@ export function logAlerts< ruleRunMetricsStore.setNumberOfActiveAlerts(activeAlertIds.length); ruleRunMetricsStore.setNumberOfRecoveredAlerts(recoveredAlertIds.length); for (const id of recoveredAlertIds) { - const { group: actionGroup } = recoveredAlerts[id].getLastScheduledActions() ?? {}; + const alert = recoveredAlerts[id]; + const { group: actionGroup } = alert.getLastScheduledActions() ?? {}; + const uuid = alert.getUuid(); const state = recoveredAlerts[id].getState(); const message = `${ruleLogPrefix} alert '${id}' has recovered`; alertingEventLogger.logAlert({ action: EVENT_LOG_ACTIONS.recoveredInstance, id, + uuid, group: actionGroup, message, state, @@ -105,12 +108,15 @@ export function logAlerts< } for (const id of newAlertIds) { - const { actionGroup } = activeAlerts[id].getScheduledActionOptions() ?? {}; - const state = activeAlerts[id].getState(); + const alert = activeAlerts[id]; + const { actionGroup } = alert.getScheduledActionOptions() ?? {}; + const state = alert.getState(); + const uuid = alert.getUuid(); const message = `${ruleLogPrefix} created new alert: '${id}'`; alertingEventLogger.logAlert({ action: EVENT_LOG_ACTIONS.newInstance, id, + uuid, group: actionGroup, message, state, @@ -119,12 +125,15 @@ export function logAlerts< } for (const id of activeAlertIds) { - const { actionGroup } = activeAlerts[id].getScheduledActionOptions() ?? {}; - const state = activeAlerts[id].getState(); + const alert = activeAlerts[id]; + const { actionGroup } = alert.getScheduledActionOptions() ?? {}; + const state = alert.getState(); + const uuid = alert.getUuid(); const message = `${ruleLogPrefix} active alert: '${id}' in actionGroup: '${actionGroup}'`; alertingEventLogger.logAlert({ action: EVENT_LOG_ACTIONS.activeInstance, id, + uuid, group: actionGroup, message, state, diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts index addbd8fccc03b..d99bc21d5d3d4 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts @@ -2654,6 +2654,7 @@ describe('Task Runner', () => { alertInstances: { '1': { meta: { + uuid: expect.any(String), lastScheduledActions: { date: new Date(DATE_1970), group: 'default', @@ -2820,6 +2821,7 @@ describe('Task Runner', () => { alertInstances: { '1': { meta: { + uuid: expect.any(String), lastScheduledActions: { date: new Date(DATE_1970), group: 'default', @@ -2835,6 +2837,7 @@ describe('Task Runner', () => { }, '2': { meta: { + uuid: expect.any(String), lastScheduledActions: { date: new Date(DATE_1970), group: 'default', diff --git a/x-pack/plugins/alerting/server/task_runner/transform_action_params.test.ts b/x-pack/plugins/alerting/server/task_runner/transform_action_params.test.ts index 13caac8e39116..a240105f7cab2 100644 --- a/x-pack/plugins/alerting/server/task_runner/transform_action_params.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/transform_action_params.test.ts @@ -46,6 +46,7 @@ describe('transformActionParams', () => { tags: ['tag-A', 'tag-B'], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: { @@ -83,6 +84,7 @@ describe('transformActionParams', () => { tags: ['tag-A', 'tag-B'], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, @@ -113,6 +115,7 @@ describe('transformActionParams', () => { tags: ['tag-A', 'tag-B'], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, @@ -142,6 +145,7 @@ describe('transformActionParams', () => { tags: ['tag-A', 'tag-B'], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, @@ -171,6 +175,7 @@ describe('transformActionParams', () => { tags: ['tag-A', 'tag-B'], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, @@ -200,6 +205,7 @@ describe('transformActionParams', () => { tags: ['tag-A', 'tag-B'], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, @@ -229,6 +235,7 @@ describe('transformActionParams', () => { tags: ['tag-A', 'tag-B'], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, @@ -257,6 +264,7 @@ describe('transformActionParams', () => { alertName: 'alert-name', spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, @@ -286,6 +294,7 @@ describe('transformActionParams', () => { tags: [], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, @@ -315,6 +324,7 @@ describe('transformActionParams', () => { tags: ['tag-A', 'tag-B'], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, @@ -344,6 +354,7 @@ describe('transformActionParams', () => { tags: ['tag-A', 'tag-B'], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, @@ -373,6 +384,7 @@ describe('transformActionParams', () => { tags: ['tag-A', 'tag-B'], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, @@ -402,6 +414,7 @@ describe('transformActionParams', () => { tags: ['tag-A', 'tag-B'], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, @@ -431,6 +444,7 @@ describe('transformActionParams', () => { tags: ['tag-A', 'tag-B'], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, @@ -446,7 +460,7 @@ describe('transformActionParams', () => { test('rule alert variables are passed to templates', () => { const actionParams = { message: - 'Value "{{alert.id}}", "{{alert.actionGroup}}" and "{{alert.actionGroupName}}" exist', + 'Value "{{alert.id}}", "{{alert.actionGroup}}", "{{alert.uuid}}" and "{{alert.actionGroupName}}" exist', }; const result = transformActionParams({ actionsPlugin, @@ -461,6 +475,7 @@ describe('transformActionParams', () => { tags: ['tag-A', 'tag-B'], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, @@ -468,7 +483,7 @@ describe('transformActionParams', () => { }); expect(result).toMatchInlineSnapshot(` Object { - "message": "Value \\"2\\", \\"action-group\\" and \\"Action Group\\" exist", + "message": "Value \\"2\\", \\"action-group\\", \\"uuid-1\\" and \\"Action Group\\" exist", } `); }); @@ -491,6 +506,7 @@ describe('transformActionParams', () => { tags: ['tag-A', 'tag-B'], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, @@ -522,6 +538,7 @@ describe('transformActionParams', () => { tags: ['tag-A', 'tag-B'], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, @@ -555,6 +572,7 @@ describe('transformActionParams', () => { tags: ['tag-A', 'tag-B'], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, @@ -588,6 +606,7 @@ describe('transformActionParams', () => { tags: ['tag-A', 'tag-B'], spaceId: 'spaceId-A', alertInstanceId: '2', + alertUuid: 'uuid-1', alertActionGroup: 'action-group', alertActionGroupName: 'Action Group', alertParams: {}, diff --git a/x-pack/plugins/alerting/server/task_runner/transform_action_params.ts b/x-pack/plugins/alerting/server/task_runner/transform_action_params.ts index fc4591e8856e8..9f037d39b75d1 100644 --- a/x-pack/plugins/alerting/server/task_runner/transform_action_params.ts +++ b/x-pack/plugins/alerting/server/task_runner/transform_action_params.ts @@ -24,6 +24,7 @@ interface TransformActionParamsOptions { spaceId: string; tags?: string[]; alertInstanceId: string; + alertUuid: string; alertActionGroup: string; alertActionGroupName: string; actionParams: RuleActionParams; @@ -64,6 +65,7 @@ export function transformActionParams({ spaceId, tags, alertInstanceId, + alertUuid, alertActionGroup, alertActionGroupName, context, @@ -101,6 +103,7 @@ export function transformActionParams({ }, alert: { id: alertInstanceId, + uuid: alertUuid, actionGroup: alertActionGroup, actionGroupName: alertActionGroupName, flapping, diff --git a/x-pack/plugins/alerting/server/test_utils/index.ts b/x-pack/plugins/alerting/server/test_utils/index.ts index 5babc70f9a4eb..589dae529cee6 100644 --- a/x-pack/plugins/alerting/server/test_utils/index.ts +++ b/x-pack/plugins/alerting/server/test_utils/index.ts @@ -5,6 +5,8 @@ * 2.0. */ +import { RawAlertInstance } from '../../common'; + interface Resolvable { resolve: (arg: T) => void; } @@ -21,3 +23,25 @@ export function resolvable(): Promise & Resolvable { }, }); } + +// Used to convert a raw Rule's UUID to something that can be used +// to compare with a jest snapshot. +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function alertWithAnyUUID(rawAlert: Record): Record { + if (!rawAlert?.meta?.uuid) return rawAlert; + + const newAlert = JSON.parse(JSON.stringify(rawAlert)); + newAlert.meta.uuid = expect.any(String); + return newAlert; +} + +export function alertsWithAnyUUID( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + rawAlerts: Record +): Record { + const newAlerts: Record = {}; + for (const id of Object.keys(rawAlerts)) { + newAlerts[id] = alertWithAnyUUID(rawAlerts[id]); + } + return newAlerts; +} diff --git a/x-pack/plugins/alerting/tsconfig.json b/x-pack/plugins/alerting/tsconfig.json index 8790eddc30203..5425e31e5cdd5 100644 --- a/x-pack/plugins/alerting/tsconfig.json +++ b/x-pack/plugins/alerting/tsconfig.json @@ -39,6 +39,7 @@ "@kbn/data-views-plugin", "@kbn/share-plugin", "@kbn/safer-lodash-set", + "@kbn/alerting-state-types", "@kbn/alerts-as-data-utils", "@kbn/core-elasticsearch-client-server-mocks", "@kbn/core-saved-objects-utils-server", diff --git a/x-pack/plugins/apm/server/routes/alerts/test_utils/index.ts b/x-pack/plugins/apm/server/routes/alerts/test_utils/index.ts index a61dea317f662..e0bb678a1be2f 100644 --- a/x-pack/plugins/apm/server/routes/alerts/test_utils/index.ts +++ b/x-pack/plugins/apm/server/routes/alerts/test_utils/index.ts @@ -38,13 +38,17 @@ export const createRuleTypeMocks = () => { } as AlertingPluginSetupContract; const scheduleActions = jest.fn(); + const getUuid = jest.fn(); const services = { scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), savedObjectsClient: { get: () => ({ attributes: { consumer: APM_SERVER_FEATURE_ID } }), }, - alertFactory: { create: jest.fn(() => ({ scheduleActions })), done: {} }, + alertFactory: { + create: jest.fn(() => ({ scheduleActions, getUuid })), + done: {}, + }, alertWithLifecycle: jest.fn(), logger: loggerMock, shouldWriteAlerts: () => true, diff --git a/x-pack/plugins/event_log/generated/mappings.json b/x-pack/plugins/event_log/generated/mappings.json index 8c61d6385d80b..c9758df202a3d 100644 --- a/x-pack/plugins/event_log/generated/mappings.json +++ b/x-pack/plugins/event_log/generated/mappings.json @@ -304,6 +304,10 @@ "flapping": { "type": "boolean" }, + "uuid": { + "type": "keyword", + "ignore_above": 1024 + }, "rule": { "properties": { "consumer": { diff --git a/x-pack/plugins/event_log/generated/schemas.ts b/x-pack/plugins/event_log/generated/schemas.ts index 27c3813dae658..7471390a7bc88 100644 --- a/x-pack/plugins/event_log/generated/schemas.ts +++ b/x-pack/plugins/event_log/generated/schemas.ts @@ -140,6 +140,7 @@ export const EventSchema = schema.maybe( alert: schema.maybe( schema.object({ flapping: ecsBoolean(), + uuid: ecsString(), rule: schema.maybe( schema.object({ consumer: ecsString(), diff --git a/x-pack/plugins/event_log/scripts/mappings.js b/x-pack/plugins/event_log/scripts/mappings.js index ecc6a5fc30792..2e236d7b4eff8 100644 --- a/x-pack/plugins/event_log/scripts/mappings.js +++ b/x-pack/plugins/event_log/scripts/mappings.js @@ -86,6 +86,10 @@ exports.EcsCustomPropertyMappings = { flapping: { type: 'boolean', }, + uuid: { + type: 'keyword', + ignore_above: 1024, + }, rule: { properties: { consumer: { diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts index 0db7256e03e66..1b9c03745cda0 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts @@ -8,7 +8,6 @@ import type { Logger } from '@kbn/logging'; import type { PublicContract } from '@kbn/utility-types'; import { getOrElse } from 'fp-ts/lib/Either'; -import * as rt from 'io-ts'; import { v4 } from 'uuid'; import { difference } from 'lodash'; import { @@ -20,6 +19,11 @@ import { RuleTypeState, } from '@kbn/alerting-plugin/server'; import { isFlapping } from '@kbn/alerting-plugin/server/lib'; +import { wrappedStateRt, WrappedLifecycleRuleState } from '@kbn/alerting-state-types'; +export type { + TrackedLifecycleAlertState, + WrappedLifecycleRuleState, +} from '@kbn/alerting-state-types'; import { ParsedExperimentalFields } from '../../common/parse_experimental_fields'; import { ParsedTechnicalFields } from '../../common/parse_technical_fields'; import { @@ -97,44 +101,6 @@ export type LifecycleRuleExecutor< > ) => Promise<{ state: State }>; -const trackedAlertStateRt = rt.type({ - alertId: rt.string, - alertUuid: rt.string, - started: rt.string, - // an array used to track changes in alert state, the order is based on the rule executions - // true - alert has changed from active/recovered - // false - alert is new or the status has remained either active or recovered - flappingHistory: rt.array(rt.boolean), - // flapping flag that indicates whether the alert is flapping - flapping: rt.boolean, - pendingRecoveredCount: rt.number, -}); - -export type TrackedLifecycleAlertState = rt.TypeOf; - -const alertTypeStateRt = () => - rt.record(rt.string, rt.unknown) as rt.Type; - -const wrappedStateRt = () => - rt.type({ - wrapped: alertTypeStateRt(), - // tracks the active alerts - trackedAlerts: rt.record(rt.string, trackedAlertStateRt), - // tracks the recovered alerts - trackedAlertsRecovered: rt.record(rt.string, trackedAlertStateRt), - }); - -/** - * This is redefined instead of derived from above `wrappedStateRt` because - * there's no easy way to instantiate generic values such as the runtime type - * factory function. - */ -export type WrappedLifecycleRuleState = RuleTypeState & { - wrapped: State; - trackedAlerts: Record; - trackedAlertsRecovered: Record; -}; - export const createLifecycleExecutor = (logger: Logger, ruleDataClient: PublicContract) => < @@ -165,6 +131,7 @@ export const createLifecycleExecutor = services: { alertFactory, shouldWriteAlerts }, state: previousState, flappingSettings, + rule, } = options; const ruleDataClientWriter = await ruleDataClient.getWriter(); @@ -180,8 +147,7 @@ export const createLifecycleExecutor = const commonRuleFields = getCommonAlertFields(options); const currentAlerts: Record = {}; - - const newAlertUuids: Record = {}; + const alertUuidMap: Map = new Map(); const lifecycleAlertServices: LifecycleAlertServices< InstanceState, @@ -190,18 +156,33 @@ export const createLifecycleExecutor = > = { alertWithLifecycle: ({ id, fields }) => { currentAlerts[id] = fields; - return alertFactory.create(id); + const alert = alertFactory.create(id); + const uuid = alert.getUuid(); + alertUuidMap.set(id, uuid); + return alert; }, getAlertStartedDate: (alertId: string) => state.trackedAlerts[alertId]?.started ?? null, getAlertUuid: (alertId: string) => { - let existingUuid = state.trackedAlerts[alertId]?.alertUuid || newAlertUuids[alertId]; + const uuid = alertUuidMap.get(alertId); + if (uuid) { + return uuid; + } - if (!existingUuid) { - existingUuid = v4(); - newAlertUuids[alertId] = existingUuid; + const trackedAlert = state.trackedAlerts[alertId]; + if (trackedAlert) { + return trackedAlert.alertUuid; } - return existingUuid; + const trackedRecoveredAlert = state.trackedAlertsRecovered[alertId]; + if (trackedRecoveredAlert) { + return trackedRecoveredAlert.alertUuid; + } + + const alertInfo = `alert ${alertId} of rule ${rule.ruleTypeId}:${rule.id}`; + logger.warn( + `[Rule Registry] requesting uuid for ${alertInfo} which is not tracked, generating dynamically` + ); + return v4(); }, getAlertByAlertUuid: async (alertUuid: string) => { try { @@ -259,13 +240,14 @@ export const createLifecycleExecutor = alertIds.map((alertId) => { const alertData = trackedAlertsDataMap[alertId]; const currentAlertData = currentAlerts[alertId]; + const trackedAlert = state.trackedAlerts[alertId]; if (!alertData) { logger.debug(`[Rule Registry] Could not find alert data for ${alertId}`); } - const isNew = !state.trackedAlerts[alertId]; - const isRecovered = !currentAlerts[alertId]; + const isNew = !trackedAlert; + const isRecovered = !currentAlertData; const isActive = !isRecovered; const flappingHistory = getUpdatedFlappingHistory( diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts index 787fd17c3d74f..d78617f9f8e9d 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts @@ -77,10 +77,14 @@ function createRule(shouldWriteAlerts: boolean = true) { const scheduleActions = jest.fn(); + let uuidCounter = 1; + const getUuid = jest.fn(() => `uuid-${uuidCounter++}`); + const alertFactory = { create: () => { return { scheduleActions, + getUuid, } as any; }, alertLimit: { diff --git a/x-pack/plugins/rule_registry/tsconfig.json b/x-pack/plugins/rule_registry/tsconfig.json index 1bb9b96e6aa92..eb42db1856f5f 100644 --- a/x-pack/plugins/rule_registry/tsconfig.json +++ b/x-pack/plugins/rule_registry/tsconfig.json @@ -31,6 +31,7 @@ "@kbn/logging", "@kbn/securitysolution-io-ts-utils", "@kbn/share-plugin", + "@kbn/alerting-state-types", "@kbn/alerts-as-data-utils", ], "exclude": [ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/alert_instance_factory_stub.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/alert_instance_factory_stub.ts index f350d2a3cb35f..c1d63acf00bdb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/alert_instance_factory_stub.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/alert_instance_factory_stub.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { v4 as uuidV4 } from 'uuid'; import type { AlertInstanceContext, AlertInstanceState, @@ -49,4 +50,7 @@ export const alertInstanceFactoryStub = < hasContext() { return false; }, + getUuid() { + return uuidV4(); + }, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/route.ts index 469d25e4b1c3f..86477bfda3d08 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_preview/api/preview_rules/route.ts @@ -195,6 +195,7 @@ export const previewRulesRoute = async ( | 'setContext' | 'getContext' | 'hasContext' + | 'getUuid' >; alertLimit: { getValue: () => number; diff --git a/x-pack/plugins/task_manager/server/saved_objects/migrations.ts b/x-pack/plugins/task_manager/server/saved_objects/migrations.ts index a147a6bdabc6b..bcbd493f83160 100644 --- a/x-pack/plugins/task_manager/server/saved_objects/migrations.ts +++ b/x-pack/plugins/task_manager/server/saved_objects/migrations.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { v4 as uuidv4 } from 'uuid'; import { LogMeta, SavedObjectMigrationContext, @@ -13,16 +14,22 @@ import { SavedObjectsUtils, SavedObjectUnsanitizedDoc, } from '@kbn/core/server'; +import type { + RuleTaskState, + TrackedLifecycleAlertState, + WrappedLifecycleRuleState, +} from '@kbn/alerting-state-types'; + import { REMOVED_TYPES } from '../task_type_dictionary'; -import { ConcreteTaskInstance, TaskStatus } from '../task'; +import { SerializedConcreteTaskInstance, TaskStatus } from '../task'; interface TaskInstanceLogMeta extends LogMeta { - migrations: { taskInstanceDocument: SavedObjectUnsanitizedDoc }; + migrations: { taskInstanceDocument: SavedObjectUnsanitizedDoc }; } type TaskInstanceMigration = ( - doc: SavedObjectUnsanitizedDoc -) => SavedObjectUnsanitizedDoc; + doc: SavedObjectUnsanitizedDoc +) => SavedObjectUnsanitizedDoc; export function getMigrations(): SavedObjectMigrationMap { return { @@ -43,15 +50,19 @@ export function getMigrations(): SavedObjectMigrationMap { '8.2.0' ), '8.5.0': executeMigrationWithErrorHandling(pipeMigrations(addEnabledField), '8.5.0'), + '8.8.0': executeMigrationWithErrorHandling(pipeMigrations(addAlertUUID), '8.8.0'), }; } function executeMigrationWithErrorHandling( - migrationFunc: SavedObjectMigrationFn, + migrationFunc: SavedObjectMigrationFn< + SerializedConcreteTaskInstance, + SerializedConcreteTaskInstance + >, version: string ) { return ( - doc: SavedObjectUnsanitizedDoc, + doc: SavedObjectUnsanitizedDoc, context: SavedObjectMigrationContext ) => { try { @@ -71,8 +82,8 @@ function executeMigrationWithErrorHandling( } function alertingTaskLegacyIdToSavedObjectIds( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { if (doc.attributes.taskType.startsWith('alerting:')) { let params: { spaceId?: string; alertId?: string } = {}; params = JSON.parse(doc.attributes.params as unknown as string); @@ -97,8 +108,8 @@ function alertingTaskLegacyIdToSavedObjectIds( } function actionsTasksLegacyIdToSavedObjectIds( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { if (doc.attributes.taskType.startsWith('actions:')) { let params: { spaceId?: string; actionTaskParamsId?: string } = {}; params = JSON.parse(doc.attributes.params as unknown as string); @@ -129,7 +140,7 @@ function actionsTasksLegacyIdToSavedObjectIds( function moveIntervalIntoSchedule({ attributes: { interval, ...attributes }, ...doc -}: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc { +}: SavedObjectUnsanitizedDoc): SavedObjectUnsanitizedDoc { return { ...doc, attributes: { @@ -146,8 +157,8 @@ function moveIntervalIntoSchedule({ } function resetUnrecognizedStatus( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { const status = doc?.attributes?.status; if (status && status === 'unrecognized') { const taskType = doc.attributes.taskType; @@ -162,20 +173,20 @@ function resetUnrecognizedStatus( ...doc.attributes, status: 'idle', }, - } as SavedObjectUnsanitizedDoc; + } as SavedObjectUnsanitizedDoc; } return doc; } function pipeMigrations(...migrations: TaskInstanceMigration[]): TaskInstanceMigration { - return (doc: SavedObjectUnsanitizedDoc) => + return (doc: SavedObjectUnsanitizedDoc) => migrations.reduce((migratedDoc, nextMigration) => nextMigration(migratedDoc), doc); } function resetAttemptsAndStatusForTheTasksWithoutSchedule( - doc: SavedObjectUnsanitizedDoc -): SavedObjectUnsanitizedDoc { + doc: SavedObjectUnsanitizedDoc +): SavedObjectUnsanitizedDoc { if (doc.attributes.taskType.startsWith('alerting:')) { if ( !doc.attributes.schedule?.interval && @@ -195,7 +206,7 @@ function resetAttemptsAndStatusForTheTasksWithoutSchedule( return doc; } -function addEnabledField(doc: SavedObjectUnsanitizedDoc) { +function addEnabledField(doc: SavedObjectUnsanitizedDoc) { if ( doc.attributes.status === TaskStatus.Failed || doc.attributes.status === TaskStatus.Unrecognized @@ -211,3 +222,83 @@ function addEnabledField(doc: SavedObjectUnsanitizedDoc) { }, }; } + +function addAlertUUID(doc: SavedObjectUnsanitizedDoc) { + if (!doc.attributes.taskType.startsWith('alerting:')) return doc; + if (!doc.attributes.state) return doc; + + const taskState: RuleTaskState = JSON.parse(doc.attributes.state); + const ruleState = taskState?.alertTypeState; + if (!ruleState) return doc; + + // get existing alert uuid's from the rule registry's rule state wrapper + const alertToTrackedMap = getAlertsToTrackedMap(ruleState); + + // we are iterating over two collections of alerts, so in case there are + // duplicates, keep track of all uuid's assigned, so the same one will be used + const currentUUIDs = new Map(); + + // add the uuids to the framework's meta object; the objects are mutated in-line + addAlertUUIDsToAlerts(taskState.alertInstances, alertToTrackedMap, currentUUIDs); + addAlertUUIDsToAlerts(taskState.alertRecoveredInstances, alertToTrackedMap, currentUUIDs); + + return { + ...doc, + attributes: { + ...doc.attributes, + state: JSON.stringify(taskState), + }, + }; +} + +// mutates alerts passed in +function addAlertUUIDsToAlerts( + alerts: RuleTaskState['alertInstances'] | undefined, + alertToTrackedMap: Map, + currentUUIDs: Map +): void { + if (!alerts) return; + + for (const [id, alert] of Object.entries(alerts)) { + if (!alert.meta) alert.meta = {}; + + // get alert info from tracked map (rule registry) + const trackedAlert = alertToTrackedMap.get(id); + // get uuid for current alert, if we've already seen it + const recentUUID = currentUUIDs.get(id); + + if (trackedAlert?.alertUuid) { + alert.meta.uuid = trackedAlert.alertUuid; + } else if (recentUUID) { + alert.meta.uuid = recentUUID; + } else { + alert.meta.uuid = uuidv4(); + } + + currentUUIDs.set(id, alert.meta.uuid); + } +} + +// gets a map of alertId => tracked alert state, which is from the +// rule registry wrapper, which contains the uuid and other info +function getAlertsToTrackedMap( + ruleState: Record +): Map { + const result = new Map(); + + if (!isRuleRegistryWrappedState(ruleState)) return result; + + return new Map([ + ...Object.entries(ruleState.trackedAlerts || {}), + ...Object.entries(ruleState.trackedAlertsRecovered || {}), + ]); +} + +function isRuleRegistryWrappedState( + ruleState: Record +): ruleState is WrappedLifecycleRuleState { + return ( + ruleState.wrapped != null && + (ruleState.trackedAlerts != null || ruleState.trackedAlertsRecovered != null) + ); +} diff --git a/x-pack/plugins/task_manager/server/saved_objects/migrations_880.test.ts b/x-pack/plugins/task_manager/server/saved_objects/migrations_880.test.ts new file mode 100644 index 0000000000000..928e30b89325f --- /dev/null +++ b/x-pack/plugins/task_manager/server/saved_objects/migrations_880.test.ts @@ -0,0 +1,327 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { omit, cloneDeep } from 'lodash'; +import { SavedObjectUnsanitizedDoc } from '@kbn/core/server'; +import { migrationMocks } from '@kbn/core/server/mocks'; +import type { + RuleTaskState, + WrappedLifecycleRuleState, + RawAlertInstance, +} from '@kbn/alerting-state-types'; + +import { getMigrations } from './migrations'; +import { SerializedConcreteTaskInstance, TaskStatus } from '../task'; + +type RawAlertInstances = Record; + +const migrationContext = migrationMocks.createContext(); +const migration880 = getMigrations()['8.8.0']; + +describe('successful migrations for 8.8.0', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + test('validate test data', () => { + // the RuleState docs are included to make it easier to read + // the tests; we validate they are the parsed JSON here + const mtState = JSON.parse(TaskDocMetricThreshold.task.state); + expect(mtState).toStrictEqual(RuleStateMetricThreshold); + + const itState = JSON.parse(TaskDocIndexThreshold.task.state); + expect(itState).toStrictEqual(RuleStateIndexThreshold); + }); + + describe('move rule alert uuid to framework meta', () => { + test('does not change non-rule tasks', () => { + const task = getMockData(); + expect(migration880(task, migrationContext)).toEqual(task); + }); + + test('does not change rule task with no state', () => { + const task = getMockData({ taskType: 'alerting:some-rule-id', state: undefined }); + expect(migration880(task, migrationContext)).toEqual(task); + }); + + test('does not change rule task with empty state', () => { + const task = getMockData({ taskType: 'alerting:some-rule-id', state: '{}' }); + expect(migration880(task, migrationContext)).toEqual(task); + }); + + test('for non-lifecycle rules, adds new uuid to alert meta', () => { + const task = getMockData(TaskDocIndexThreshold.task); + const taskMigrated = migration880(task, migrationContext); + const state = JSON.parse(taskMigrated.attributes.state) as RuleTaskState; + + checkMetaInRuleTaskState(state, RuleStateIndexThreshold); + }); + + test('for lifecycle rules, copies uuid to alert meta', () => { + const task = getMockData(TaskDocMetricThreshold.task); + const taskMigrated = migration880(task, migrationContext); + const state = JSON.parse(taskMigrated.attributes.state); + + checkMetaInRuleTaskStateWrapped(state, RuleStateMetricThreshold); + }); + }); +}); + +function checkMetaInRuleTaskState( + actual: RuleTaskState, + original: RuleTaskState, + wrappedUUIDs?: Map +) { + // delete the uuids from actual (a copy of it) to compare to original + const copy = cloneDeep(actual); + + // make sure every alertInstance element has a UUID, and that's the only change + for (const [id, alert] of Object.entries(actual.alertInstances || {})) { + checkAlert(id, alert, original.alertInstances); + delete copy?.alertInstances?.[id].meta?.uuid; + } + + // make sure every alertRecoveredInstance element has a UUID, and that's the only change + for (const [id, alert] of Object.entries(actual.alertRecoveredInstances || {})) { + checkAlert(id, alert, original.alertRecoveredInstances); + delete copy?.alertRecoveredInstances?.[id].meta?.uuid; + } + + // after deleting the uuids, should be same as the original + expect(copy).toStrictEqual(original); + + function checkAlert(id: string, alert: RawAlertInstance, instances?: RawAlertInstances) { + expect(alert.meta?.uuid).toMatch(/^.{36}$/); + + const expectedAlert = instances?.[id]; + expect(omit(alert, 'meta.uuid')).toStrictEqual(expectedAlert); + + if (wrappedUUIDs) { + expect(alert.meta?.uuid).toBe(wrappedUUIDs.get(id)); + } + } +} + +function checkMetaInRuleTaskStateWrapped(actual: RuleTaskState, expected: RuleTaskState) { + const wrappedState = expected.alertTypeState as WrappedLifecycleRuleState; + + const wrappedUUIDs = new Map(); + for (const [id, alert] of Object.entries(wrappedState.trackedAlerts || {})) { + wrappedUUIDs.set(id, alert.alertUuid); + } + + for (const [id, alert] of Object.entries(wrappedState.trackedAlertsRecovered || {})) { + wrappedUUIDs.set(id, alert.alertUuid); + } + + checkMetaInRuleTaskState(actual, expected, wrappedUUIDs); +} + +export function getMockData( + overwrites: Record = {} +): SavedObjectUnsanitizedDoc { + return { + id: 'some-uuid', + type: 'task', + attributes: { + id: 'some-id', + status: TaskStatus.Idle, + taskType: 'some-taskType', + state: JSON.stringify({}), + params: JSON.stringify({ prop: true }), + traceparent: 'some-traceparent', + scheduledAt: new Date().toISOString(), + startedAt: null, + retryAt: null, + runAt: new Date().toISOString(), + attempts: 0, + ownerId: null, + ...cloneDeep(overwrites), + }, + }; +} + +// data below generated by ../migrations_helpers/get-rule-task-state.js + +const TaskDocIndexThreshold = { + migrationVersion: { task: '8.5.0' }, + task: { + retryAt: null, + runAt: '2023-02-22T03:38:19.334Z', + startedAt: null, + params: + '{"alertId":"4f11f730-b262-11ed-b5fa-5de11bbd3e96","spaceId":"default","consumer":"alerts"}', + ownerId: null, + enabled: true, + schedule: { interval: '3s' }, + taskType: 'alerting:.index-threshold', + scope: ['alerting'], + traceparent: '', + state: + '{"alertTypeState":{},"alertInstances":{"host-C":{"state":{"start":"2023-02-22T03:38:10.433Z","duration":"6006000000"},"meta":{"flappingHistory":[true,false],"flapping":false,"pendingRecoveredCount":0,"lastScheduledActions":{"group":"threshold met","date":"2023-02-22T03:38:16.442Z"}}}},"alertRecoveredInstances":{"host-A":{"meta":{"flappingHistory":[true,false,true],"flapping":false}},"host-B":{"meta":{"flappingHistory":[true,false,true],"flapping":false}}},"summaryActions":{},"previousStartedAt":"2023-02-22T03:38:16.334Z"}', + scheduledAt: '2023-02-22T03:38:13.331Z', + attempts: 0, + status: 'idle', + }, + references: [], + updated_at: '2023-02-22T03:38:16.570Z', + coreMigrationVersion: '8.7.0', + created_at: '2023-02-22T03:38:03.160Z', + type: 'task', +}; + +// included just so the `state` JSON data ^^^ is readable +const RuleStateIndexThreshold: RuleTaskState = { + alertTypeState: {}, + alertInstances: { + 'host-C': { + state: { + start: '2023-02-22T03:38:10.433Z', + duration: '6006000000', + }, + meta: { + flappingHistory: [true, false], + flapping: false, + pendingRecoveredCount: 0, + lastScheduledActions: { + group: 'threshold met', + date: '2023-02-22T03:38:16.442Z', + }, + }, + }, + }, + alertRecoveredInstances: { + 'host-A': { + meta: { + flappingHistory: [true, false, true], + flapping: false, + }, + }, + 'host-B': { + meta: { + flappingHistory: [true, false, true], + flapping: false, + }, + }, + }, + summaryActions: {}, + previousStartedAt: '2023-02-22T03:38:16.334Z', + + // This cast is needed as RuleTaskState defines dates as Date, but + // they are stored as strings. There is no "serialized" version + // of this, it's an io-ts generated type. You can check the rest + // of the types by deleting the "as unknown as ..." bits below. +} as unknown as RuleTaskState; + +const TaskDocMetricThreshold = { + migrationVersion: { task: '8.5.0' }, + task: { + retryAt: null, + runAt: '2023-02-22T03:38:16.328Z', + startedAt: null, + params: + '{"alertId":"4dd72d40-b262-11ed-b5fa-5de11bbd3e96","spaceId":"default","consumer":"infrastructure"}', + ownerId: null, + enabled: true, + schedule: { interval: '3s' }, + taskType: 'alerting:metrics.alert.threshold', + scope: ['alerting'], + traceparent: '', + state: + '{"alertTypeState":{"wrapped":{"lastRunTimestamp":1677037093328,"missingGroups":[],"groupBy":["network.name"]},"trackedAlerts":{"host-A":{"alertId":"host-A","alertUuid":"f8b420a4-a596-4c96-8e42-6da5a167dd56","started":"2023-02-22T03:38:13.328Z","flappingHistory":[true,true,true],"flapping":false,"pendingRecoveredCount":0},"host-B":{"alertId":"host-B","alertUuid":"1ecbe90f-a196-40db-aede-094577ccbf14","started":"2023-02-22T03:38:13.328Z","flappingHistory":[true,true,true],"flapping":false,"pendingRecoveredCount":0}},"trackedAlertsRecovered":{"host-C":{"alertId":"host-C","alertUuid":"95669714-2ab6-4155-8928-107aea504f39","started":"2023-02-22T03:38:07.330Z","flappingHistory":[true,true],"flapping":false,"pendingRecoveredCount":0}}},"alertInstances":{"host-A":{"state":{"start":"2023-02-22T03:38:13.587Z","duration":"0"},"meta":{"flappingHistory":[true,true,true],"flapping":false,"pendingRecoveredCount":0,"lastScheduledActions":{"group":"metrics.threshold.fired","date":"2023-02-22T03:38:13.590Z"}}},"host-B":{"state":{"start":"2023-02-22T03:38:13.587Z","duration":"0"},"meta":{"flappingHistory":[true,true,true],"flapping":false,"pendingRecoveredCount":0,"lastScheduledActions":{"group":"metrics.threshold.fired","date":"2023-02-22T03:38:13.591Z"}}}},"alertRecoveredInstances":{"host-C":{"meta":{"flappingHistory":[true,true],"flapping":false}}},"summaryActions":{},"previousStartedAt":"2023-02-22T03:38:13.328Z"}', + scheduledAt: '2023-02-22T03:38:10.330Z', + attempts: 0, + status: 'idle', + }, + references: [], + updated_at: '2023-02-22T03:38:13.699Z', + coreMigrationVersion: '8.7.0', + created_at: '2023-02-22T03:38:01.102Z', + type: 'task', +}; + +// included just so the `state` JSON data ^^^ is readable +const RuleStateMetricThreshold: RuleTaskState = { + alertTypeState: { + wrapped: { + lastRunTimestamp: 1677037093328, + missingGroups: [], + groupBy: ['network.name'], + }, + trackedAlerts: { + 'host-A': { + alertId: 'host-A', + alertUuid: 'f8b420a4-a596-4c96-8e42-6da5a167dd56', + started: '2023-02-22T03:38:13.328Z', + flappingHistory: [true, true, true], + flapping: false, + pendingRecoveredCount: 0, + }, + 'host-B': { + alertId: 'host-B', + alertUuid: '1ecbe90f-a196-40db-aede-094577ccbf14', + started: '2023-02-22T03:38:13.328Z', + flappingHistory: [true, true, true], + flapping: false, + pendingRecoveredCount: 0, + }, + }, + trackedAlertsRecovered: { + 'host-C': { + alertId: 'host-C', + alertUuid: '95669714-2ab6-4155-8928-107aea504f39', + started: '2023-02-22T03:38:07.330Z', + flappingHistory: [true, true], + flapping: false, + pendingRecoveredCount: 0, + }, + }, + }, + alertInstances: { + 'host-A': { + state: { + start: '2023-02-22T03:38:13.587Z', + duration: '0', + }, + meta: { + flappingHistory: [true, true, true], + flapping: false, + pendingRecoveredCount: 0, + lastScheduledActions: { + group: 'metrics.threshold.fired', + date: '2023-02-22T03:38:13.590Z', + }, + }, + }, + 'host-B': { + state: { + start: '2023-02-22T03:38:13.587Z', + duration: '0', + }, + meta: { + flappingHistory: [true, true, true], + flapping: false, + pendingRecoveredCount: 0, + lastScheduledActions: { + group: 'metrics.threshold.fired', + date: '2023-02-22T03:38:13.591Z', + }, + }, + }, + }, + alertRecoveredInstances: { + 'host-C': { + meta: { + flappingHistory: [true, true], + flapping: false, + }, + }, + }, + summaryActions: {}, + previousStartedAt: '2023-02-22T03:38:13.328Z', + // see ^^^ for the index threshold rule why this cast is needed +} as unknown as RuleTaskState; diff --git a/x-pack/plugins/task_manager/server/saved_objects/migrations_helpers/get_rule_task_state.js b/x-pack/plugins/task_manager/server/saved_objects/migrations_helpers/get_rule_task_state.js new file mode 100755 index 0000000000000..a30da03fff940 --- /dev/null +++ b/x-pack/plugins/task_manager/server/saved_objects/migrations_helpers/get_rule_task_state.js @@ -0,0 +1,265 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* +Used to get data for a migration test. + +This script will create a metric threshold and index threshold rule, +for the purposes of looking at their task state. It arranges to get +the rules active and then recovered, so that the alert data will show +up in both the "active" and "recovered but tracking for flapping" +containers. + +It then will send data for the alerts to go active on, then recover, +forever. + +It will then query the task documents for the rules, waiting till it +gets some in both of the containers. It prints the task document as +a single line of JSON, and the task state as pretty JSON, just so you +can "read it" - because it's stored as a JSON string in the task doc. + +- env var `$KBN_URL` should be set to your Kibana URL, with user/pass. +- env var `$ES_URL` should be set to your Elasticsearch URL, with user/pass. +*/ + +const path = require('path'); +const https = require('node:https'); +const fetch = require('node-fetch'); + +const KB_URL = process.env.KB_URL || process.env.KBN_URL; +const ES_URL = process.env.ES_URL; + +const dataIndex = 'rule-task-state--dev'; +const dataAlias = `metrics-${dataIndex}`; // metrics rules look for metrics-* indices +const mappings = { + properties: { + '@timestamp': { type: 'date' }, + 'network.packets': { type: 'long' }, + 'network.name': { type: 'keyword' }, + }, +}; + +let Active = true; +let Conn; +let MtRule; +let ItRule; + +main(); + +async function main() { + const createdIndex = await putEs(dataIndex, { mappings }); + console.log(`created index: ${JSON.stringify(createdIndex)}`); + + const alias = await putEs(`${dataIndex}/_alias/${dataAlias}`); + console.log(`alias for metrics: ${JSON.stringify(alias)}`); + + // write data @ 1s, alternating active / not active @ 5s + setInterval(writeData, 1000); + setInterval(() => (Active = !Active), 5000); + + const createConnPayload = { + name: 'server log for rule-task-state', + connector_type_id: '.server-log', + }; + + Conn = await postKbn(`api/actions/connector`, createConnPayload); + console.log(`server log id: ${Conn.id}`); + + MtRule = await postKbn(`api/alerting/rule`, getMtRulePayload()); + console.log(`metric threshold rule id: ${MtRule.id}`); + + ItRule = await postKbn(`api/alerting/rule`, getItRulePayload()); + console.log(`index threshold rule id: ${ItRule.id}`); + + setInterval(getTaskDocs, 3000); +} + +function writeData() { + const date = new Date().toISOString(); + postEs(`${dataIndex}/_doc`, { + '@timestamp': date, + network: { name: 'host-A', packets: Active ? 1 : 0 }, + }); + postEs(`${dataIndex}/_doc`, { + '@timestamp': date, + network: { name: 'host-B', packets: Active ? 1 : 0 }, + }); + postEs(`${dataIndex}/_doc`, { + '@timestamp': date, + network: { name: 'host-C', packets: Active ? 0 : 1 }, + }); +} + +async function getTaskDocs() { + const { task: mtTaskState, ruleState: mtRuleState } = await getTask(MtRule.id); + const { task: itTaskState, ruleState: itRuleState } = await getTask(ItRule.id); + + console.log('--------------------------------------------------------'); + console.log(JSON.stringify(itTaskState._source)); + console.log(JSON.stringify(itRuleState, null, 4)); + console.log(); + console.log(JSON.stringify(mtTaskState._source)); + console.log(JSON.stringify(mtRuleState, null, 4)); + console.log(); + console.log('waiting for better task docs'); + console.log(); + + if ( + Object.keys(itRuleState.alertInstances).length > 0 && + Object.keys(mtRuleState.alertInstances).length > 0 && + Object.keys(itRuleState.alertRecoveredInstances).length > 0 && + Object.keys(mtRuleState.alertRecoveredInstances).length > 0 + ) { + console.log('that last one is a keeper!'); + + console.log('full docs for es archive:'); + console.log(''); + console.log(JSON.stringify(itTaskState, null, 4)); + console.log(''); + console.log(JSON.stringify(mtTaskState, null, 4)); + console.log(''); + process.exit(0); + } +} + +function getMtRulePayload() { + return { + consumer: 'infrastructure', + name: 'rule-mt', + schedule: { + interval: '3s', + }, + params: { + criteria: [ + { + aggType: 'max', + comparator: '>', + threshold: [0], + timeSize: 3, + timeUnit: 's', + metric: 'network.packets', + }, + ], + sourceId: 'default', + alertOnNoData: false, + alertOnGroupDisappear: false, + groupBy: ['network.name'], + }, + rule_type_id: 'metrics.alert.threshold', + notify_when: 'onActiveAlert', + actions: [ + { + group: 'metrics.threshold.fired', + id: Conn.id, + params: { + message: + '{{alertName}} - {{context.group}} is in a state of {{context.alertState}}\n\nReason:\n{{context.reason}}\n', + }, + }, + ], + }; +} + +function getItRulePayload() { + return { + rule_type_id: '.index-threshold', + name: 'rule-it', + notify_when: 'onActiveAlert', + consumer: 'alerts', + schedule: { interval: '3s' }, + actions: [ + { + group: 'threshold met', + id: Conn.id, + params: { message: '{{context.message}}' }, + }, + ], + params: { + index: [dataIndex], + timeField: '@timestamp', + aggType: 'max', + aggField: 'network.packets', + groupBy: 'top', + termSize: 100, + termField: 'network.name', + timeWindowSize: 3, + timeWindowUnit: 's', + thresholdComparator: '>', + threshold: [0], + }, + }; +} + +/** @type { (id: string) => Promise } */ +async function getTask(id) { + await getURL(`${ES_URL}/_refresh`); + + const task = await getEs(`/.kibana_task_manager/_doc/task:${id}`); + + const ruleStateJ = task._source.task.state; + const ruleState = JSON.parse(ruleStateJ); + + return { task, ruleState }; +} + +async function getEs(url) { + return getURL(path.join(ES_URL, url)); +} +async function postEs(url, body) { + return postURL(path.join(ES_URL, url), body); +} +async function putEs(url, body) { + return putURL(path.join(ES_URL, url), body); +} + +// eslint-disable-next-line no-unused-vars +async function getKbn(url) { + return getURL(path.join(KB_URL, url)); +} +async function postKbn(url, body) { + return postURL(path.join(KB_URL, url), body); +} +// eslint-disable-next-line no-unused-vars +async function putKbn(url, body) { + return putURL(path.join(KB_URL, url), body); +} + +async function getURL(url) { + return sendURL(url, 'GET'); +} +async function postURL(url, body) { + return sendURL(url, 'POST', body); +} +async function putURL(url, body) { + return sendURL(url, 'PUT', body); +} + +async function sendURL(urlWithPass, method, body) { + const purl = new URL(urlWithPass); + const userPass = `${purl.username}:${purl.password}`; + const userPassEn = Buffer.from(userPass).toString('base64'); + const auth = `Basic ${userPassEn}`; + const url = `${purl.origin}${purl.pathname}${purl.search}`; + const headers = { + 'content-type': 'application/json', + 'kbn-xsrf': 'foo', + authorization: auth, + }; + + const fetchOptions = { method, headers }; + if (body) fetchOptions.body = JSON.stringify(body); + + if (purl.protocol === 'https:') { + fetchOptions.agent = new https.Agent({ rejectUnauthorized: false }); + } + + // console.log(`fetch("${url}", ${JSON.stringify(fetchOptions, null, 4)}`) + const response = await fetch(url, fetchOptions); + const object = await response.json(); + // console.log(`fetch(...): ${JSON.stringify(object, null, 4)}`) + return object; +} diff --git a/x-pack/plugins/task_manager/tsconfig.json b/x-pack/plugins/task_manager/tsconfig.json index 0caec8f729150..9e6604c03121f 100644 --- a/x-pack/plugins/task_manager/tsconfig.json +++ b/x-pack/plugins/task_manager/tsconfig.json @@ -9,6 +9,7 @@ "server/**/*.json", ], "kbn_references": [ + "@kbn/alerting-state-types", "@kbn/core", "@kbn/usage-collection-plugin", "@kbn/config-schema", @@ -17,7 +18,7 @@ "@kbn/safer-lodash-set", "@kbn/es-types", "@kbn/apm-utils", - "@kbn/core-saved-objects-common", + "@kbn/core-saved-objects-common" ], "exclude": [ "target/**/*", diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/event_log.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/event_log.ts index 27d4fc59d239a..7d832ccb450b4 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/event_log.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/event_log.ts @@ -19,6 +19,12 @@ import { } from '../../../../common/lib'; import { FtrProviderContext } from '../../../../common/ftr_provider_context'; +const InstanceActions = new Set([ + 'new-instance', + 'active-instance', + 'recovered-instance', +]); + // eslint-disable-next-line import/no-default-export export default function eventLogTests({ getService }: FtrProviderContext) { const supertest = getService('supertest'); @@ -1096,6 +1102,84 @@ export default function eventLogTests({ getService }: FtrProviderContext) { .map((event) => event?.kibana?.alert?.flapping); expect(flapping).to.eql([false, false, false, false, false, true, true, true]); }); + + it('should generate expected uuids for events for flapping alerts that go active while flapping and eventually recover', async () => { + await supertest + .post(`${getUrlPrefix(space.id)}/internal/alerting/rules/settings/_flapping`) + .set('kbn-xsrf', 'foo') + .auth('superuser', 'superuser') + .send({ + enabled: true, + look_back_window: 6, + status_change_threshold: 4, + }) + .expect(200); + + // flap and then recover, then active again + const instance = [true, false, true, false, true].concat( + ...new Array(6).fill(false), + true + ); + const pattern = { instance }; + + const response = await supertest + .post(`${getUrlPrefix(space.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send( + getTestRuleData({ + rule_type_id: 'test.patternFiring', + schedule: { interval: '1s' }, + throttle: null, + params: { pattern }, + actions: [], + notify_when: RuleNotifyWhen.CHANGE, + }) + ); + + expect(response.status).to.eql(200); + const alertId = response.body.id; + objectRemover.add(space.id, alertId, 'rule', 'alerting'); + + // get the events we're expecting + const events = await retry.try(async () => { + return await getEventLog({ + getService, + spaceId: space.id, + type: 'alert', + id: alertId, + provider: 'alerting', + actions: new Map([ + ['execute', { gte: 10 }], + ['new-instance', { gte: 4 }], + ['active-instance', { gte: 3 }], + ['recovered-instance', { gte: 3 }], + ]), + }); + }); + + let currentUuid: string | undefined; + const seenUuids = new Set(); + for (const event of events) { + const action = event?.event?.action; + const uuid = event?.kibana?.alert?.uuid; + + if (!InstanceActions.has(action)) continue; + + expect(uuid).to.be.ok(); + + if (action === 'new-instance') { + expect(currentUuid).to.be(undefined); + expect(seenUuids.has(uuid!)).to.be(false); + currentUuid = uuid; + seenUuids.add(uuid!); + } else if (action === 'active-instance') { + expect(uuid).to.be(currentUuid); + } else if (action === 'recovered-instance') { + expect(uuid).to.be(currentUuid); + currentUuid = undefined; + } + } + }); }); } }); @@ -1181,6 +1265,12 @@ export function validateEvent(event: IValidatedEvent, params: ValidateEventLogPa expect(event?.kibana?.alerting?.instance_id).to.be(instanceId); } + if (InstanceActions.has(event?.event?.action)) { + expect(typeof event?.kibana?.alert?.uuid).to.be('string'); + } else { + expect(event?.kibana?.alert?.uuid).to.be(undefined); + } + if (reason) { expect(event?.event?.reason).to.be(reason); } diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/get_alert_summary.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/get_alert_summary.ts index 712616329929a..75fd92ec61eaa 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/get_alert_summary.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/get_alert_summary.ts @@ -233,7 +233,12 @@ export default function createGetAlertSummaryTests({ getService }: FtrProviderCo `${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${createdRule.id}/_alert_summary` ); - const actualAlerts = response.body.alerts; + const actualAlerts = checkAndCleanActualAlerts(response.body.alerts, [ + 'alertA', + 'alertB', + 'alertC', + ]); + const expectedAlerts = { alertA: { status: 'Active', @@ -292,7 +297,11 @@ export default function createGetAlertSummaryTests({ getService }: FtrProviderCo `${getUrlPrefix(Spaces.space1.id)}/api/alerts/alert/${createdRule.id}/_instance_summary` ); - const actualAlerts = response.body.instances; + const actualAlerts = checkAndCleanActualAlerts(response.body.instances, [ + 'alertA', + 'alertB', + 'alertC', + ]); const expectedAlerts = { alertA: { status: 'Active', @@ -337,3 +346,25 @@ export default function createGetAlertSummaryTests({ getService }: FtrProviderCo }); } } + +function checkAndCleanActualAlerts(actualAlerts: any, idsWithUuids: string[]) { + const uuids = new Set(); + const idsWithUuidsSet = new Set(idsWithUuids); + + for (const alertId of Object.keys(actualAlerts)) { + const alert = actualAlerts[alertId]; + + if (idsWithUuidsSet.has(alertId)) { + const uuid = alert?.uuid; + expect(typeof uuid).to.be('string'); + + if (uuid) { + expect(uuids.has(uuid)).to.be(false); + uuids.add(uuid); + delete actualAlerts[alertId].uuid; + } + } + } + + return actualAlerts; +} diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/event_log_alerts.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/event_log_alerts.ts index cc5f3108ddd54..b012abb2e2cde 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/event_log_alerts.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/event_log_alerts.ts @@ -88,29 +88,41 @@ export default function eventLogAlertTests({ getService }: FtrProviderContext) { // Ensure every execution actually had a unique id from the others expect(totalUniqueExecutionIds.size).to.equal(totalExecutionEventCount); + const allAlertUuids = new Set(); const currentAlertSpan: { alertId?: string; start?: string; durationToDate?: string; + uuid?: string; } = {}; const flapping = []; for (let i = 0; i < instanceEvents.length; ++i) { + expect(typeof instanceEvents[i]?.kibana?.alert?.uuid).to.be('string'); + const uuid = instanceEvents[i]?.kibana?.alert?.uuid!; + flapping.push(instanceEvents[i]?.kibana?.alert?.flapping); switch (instanceEvents[i]?.event?.action) { case 'new-instance': expect(instanceEvents[i]?.kibana?.alerting?.instance_id).to.equal('instance'); - // a new alert should generate a unique UUID for the duration of its activeness + expect(instanceEvents[i]?.kibana?.alert?.flapping).to.equal(false); expect(instanceEvents[i]?.event?.end).to.be(undefined); + // uuid should be unique for new instances, reused for active/recovered + expect(currentAlertSpan.uuid).to.be(undefined); + expect(allAlertUuids.has(uuid)).to.be(false); + allAlertUuids.add(uuid); + currentAlertSpan.alertId = instanceEvents[i]?.kibana?.alerting?.instance_id; currentAlertSpan.start = instanceEvents[i]?.event?.start; currentAlertSpan.durationToDate = `${instanceEvents[i]?.event?.duration}`; + currentAlertSpan.uuid = uuid; break; case 'active-instance': expect(instanceEvents[i]?.kibana?.alerting?.instance_id).to.equal('instance'); expect(instanceEvents[i]?.event?.start).to.equal(currentAlertSpan.start); expect(instanceEvents[i]?.event?.end).to.be(undefined); + expect(instanceEvents[i]?.kibana?.alert?.uuid).to.be(currentAlertSpan.uuid); if (instanceEvents[i]?.event?.duration! !== '0') { expect( @@ -125,10 +137,12 @@ export default function eventLogAlertTests({ getService }: FtrProviderContext) { expect(instanceEvents[i]?.kibana?.alerting?.instance_id).to.equal('instance'); expect(instanceEvents[i]?.event?.start).to.equal(currentAlertSpan.start); expect(instanceEvents[i]?.event?.end).not.to.be(undefined); + expect(instanceEvents[i]?.kibana?.alert?.uuid).to.be(currentAlertSpan.uuid); expect( new Date(instanceEvents[i]?.event?.end!).valueOf() - new Date(instanceEvents[i]?.event?.start!).valueOf() ).to.equal(nanosToMillis(instanceEvents[i]?.event?.duration!)); + currentAlertSpan.uuid = undefined; break; } } diff --git a/x-pack/test/functional/es_archives/task_manager_tasks/data.json b/x-pack/test/functional/es_archives/task_manager_tasks/data.json index 974642e98c07f..eb0db58dfb385 100644 --- a/x-pack/test/functional/es_archives/task_manager_tasks/data.json +++ b/x-pack/test/functional/es_archives/task_manager_tasks/data.json @@ -214,3 +214,75 @@ } } } + +{ + "type": "doc", + "value": { + "id": "task:d0487a50-c7f1-11ed-aefe-691acd8d4e25", + "index": ".kibana_task_manager_1", + "source": { + "migrationVersion": { + "task": "8.5.0" + }, + "task": { + "retryAt": null, + "runAt": "2023-03-21T14:08:29.777Z", + "startedAt": null, + "params": "{\"alertId\":\"d0487a50-c7f1-11ed-aefe-691acd8d4e25\",\"spaceId\":\"default\",\"consumer\":\"alerts\"}", + "ownerId": null, + "enabled": true, + "schedule": { + "interval": "3s" + }, + "taskType": "alerting:.index-threshold", + "scope": [ + "alerting" + ], + "traceparent": "", + "state": "{\"alertTypeState\":{},\"alertInstances\":{\"host-C\":{\"state\":{\"start\":\"2023-03-21T14:08:20.886Z\",\"duration\":\"6010000000\"},\"meta\":{\"flappingHistory\":[true,false],\"flapping\":false,\"pendingRecoveredCount\":0,\"lastScheduledActions\":{\"group\":\"threshold met\",\"date\":\"2023-03-21T14:08:26.903Z\"}}}},\"alertRecoveredInstances\":{\"host-A\":{\"meta\":{\"flappingHistory\":[true,false,true],\"flapping\":false}},\"host-B\":{\"meta\":{\"flappingHistory\":[true,false,true],\"flapping\":false}}},\"summaryActions\":{},\"previousStartedAt\":\"2023-03-21T14:08:26.777Z\"}", + "scheduledAt": "2023-03-21T14:08:23.770Z", + "attempts": 0, + "status": "idle" + }, + "references": [], + "updated_at": "2023-03-21T14:08:27.022Z", + "type": "task" + } + } +} + +{ + "type": "doc", + "value": { + "id": "task:cf0ced10-c7f1-11ed-aefe-691acd8d4e25", + "index": ".kibana_task_manager_1", + "source": { + "migrationVersion": { + "task": "8.5.0" + }, + "task": { + "retryAt": null, + "runAt": "2023-03-21T14:08:29.777Z", + "startedAt": null, + "params": "{\"alertId\":\"cf0ced10-c7f1-11ed-aefe-691acd8d4e25\",\"spaceId\":\"default\",\"consumer\":\"infrastructure\"}", + "ownerId": null, + "enabled": true, + "schedule": { + "interval": "3s" + }, + "taskType": "alerting:metrics.alert.threshold", + "scope": [ + "alerting" + ], + "traceparent": "", + "state": "{\"alertTypeState\":{\"wrapped\":{\"lastRunTimestamp\":1679407706777,\"missingGroups\":[],\"groupBy\":[\"network.name\"]},\"trackedAlerts\":{\"host-C\":{\"alertId\":\"host-C\",\"alertUuid\":\"34f86489-cf44-4760-a9df-d1a2166230d9\",\"started\":\"2023-03-21T14:08:20.770Z\",\"flappingHistory\":[true,false],\"flapping\":false,\"pendingRecoveredCount\":0}},\"trackedAlertsRecovered\":{\"host-A\":{\"alertId\":\"host-A\",\"alertUuid\":\"d3a6a7d0-71d3-475d-a8db-5ea7a03462d5\",\"started\":\"2023-03-21T14:08:11.767Z\",\"flappingHistory\":[true,false,false,true],\"flapping\":false,\"pendingRecoveredCount\":0},\"host-B\":{\"alertId\":\"host-B\",\"alertUuid\":\"41d17c71-12b6-4bbc-8fd7-1eee8498c17f\",\"started\":\"2023-03-21T14:08:11.767Z\",\"flappingHistory\":[true,false,false,true],\"flapping\":false,\"pendingRecoveredCount\":0}}},\"alertInstances\":{\"host-C\":{\"state\":{\"start\":\"2023-03-21T14:08:21.701Z\",\"duration\":\"6049000000\"},\"meta\":{\"flappingHistory\":[true,false],\"flapping\":false,\"pendingRecoveredCount\":0,\"lastScheduledActions\":{\"group\":\"metrics.threshold.fired\",\"date\":\"2023-03-21T14:08:27.755Z\"}}}},\"alertRecoveredInstances\":{\"host-A\":{\"meta\":{\"flappingHistory\":[true,false,false,true],\"flapping\":false}},\"host-B\":{\"meta\":{\"flappingHistory\":[true,false,false,true],\"flapping\":false}}},\"summaryActions\":{},\"previousStartedAt\":\"2023-03-21T14:08:26.777Z\"}", + "scheduledAt": "2023-03-21T14:08:23.770Z", + "attempts": 0, + "status": "idle" + }, + "references": [], + "updated_at": "2023-03-21T14:08:27.877Z", + "type": "task" + } + } +} diff --git a/x-pack/test/functional/es_archives/task_manager_tasks/mappings.json b/x-pack/test/functional/es_archives/task_manager_tasks/mappings.json index 6ec81326d1ca4..4272c9cb1b912 100644 --- a/x-pack/test/functional/es_archives/task_manager_tasks/mappings.json +++ b/x-pack/test/functional/es_archives/task_manager_tasks/mappings.json @@ -161,21 +161,24 @@ }, "task": { "properties": { - "attempts": { - "type": "integer" - }, - "ownerId": { + "taskType": { "type": "keyword" }, - "params": { - "type": "text" - }, - "retryAt": { + "scheduledAt": { "type": "date" }, "runAt": { "type": "date" }, + "startedAt": { + "type": "date" + }, + "retryAt": { + "type": "date" + }, + "enabled": { + "type": "boolean" + }, "schedule": { "properties": { "interval": { @@ -183,25 +186,28 @@ } } }, - "scheduledAt": { - "type": "date" + "attempts": { + "type": "integer" }, - "scope": { + "status": { "type": "keyword" }, - "startedAt": { - "type": "date" + "traceparent": { + "type": "text" + }, + "params": { + "type": "text" }, "state": { "type": "text" }, - "status": { + "user": { "type": "keyword" }, - "taskType": { + "scope": { "type": "keyword" }, - "user": { + "ownerId": { "type": "keyword" } } diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/migrations.ts b/x-pack/test/plugin_api_integration/test_suites/task_manager/migrations.ts index 232db750d5425..8497f8bdc9678 100644 --- a/x-pack/test/plugin_api_integration/test_suites/task_manager/migrations.ts +++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/migrations.ts @@ -10,10 +10,12 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { TransportResult } from '@elastic/elasticsearch'; import { ConcreteTaskInstance, + SerializedConcreteTaskInstance, TaskInstanceWithDeprecatedFields, TaskStatus, } from '@kbn/task-manager-plugin/server/task'; import { SavedObjectsUtils } from '@kbn/core/server'; +import type { RuleTaskState, WrappedLifecycleRuleState } from '@kbn/alerting-state-types'; import { FtrProviderContext } from '../../../common/ftr_provider_context'; export default function createGetTests({ getService }: FtrProviderContext) { @@ -190,5 +192,85 @@ export default function createGetTests({ getService }: FtrProviderContext) { expect(task._source?.task.enabled).to.be(undefined); }); }); + + describe('8.8.0', async () => { + it('adds UUIDs to all alerts', async () => { + const response = await es.search<{ task: SerializedConcreteTaskInstance }>( + { + index: '.kibana_task_manager', + size: 100, + body: { query: { match_all: {} } }, + }, + { meta: true } + ); + expect(response.statusCode).to.eql(200); + const tasks = response.body.hits.hits; + tasks.forEach((task) => { + const stateString = task._source?.task.state; + expect(stateString).to.be.ok(); + const state: RuleTaskState = JSON.parse(stateString!); + const uuids = new Set(); + + for (const alert of Object.values(state.alertInstances || {})) { + const uuid = alert?.meta?.uuid || 'uuid-is-missing'; + expect(uuid).to.match(/^.{8}-.{4}-.{4}-.{4}-.{12}$/); + expect(uuids.has(uuid)).to.be(false); + uuids.add(uuid); + } + + for (const alert of Object.values(state.alertRecoveredInstances || {})) { + const uuid = alert?.meta?.uuid || 'uuid-is-missing'; + expect(uuid).to.match(/^.{8}-.{4}-.{4}-.{4}-.{12}$/); + expect(uuids.has(uuid)).to.be(false); + uuids.add(uuid); + } + }); + }); + + it('copies UUIDs from rule registry wrapper to alerting framework', async () => { + const response = await es.search<{ task: SerializedConcreteTaskInstance }>( + { + index: '.kibana_task_manager', + size: 100, + body: { query: { match_all: {} } }, + }, + { meta: true } + ); + expect(response.statusCode).to.eql(200); + const tasks = response.body.hits.hits; + tasks.forEach((task) => { + const stateString = task._source?.task.state; + expect(stateString).to.be.ok(); + + const state: RuleTaskState = JSON.parse(stateString!); + if (!state?.alertTypeState?.wrapped) return; + + const wrappedUUIDs = new Map(); + const wrappedState = state.alertTypeState as WrappedLifecycleRuleState; + + for (const alert of Object.values(wrappedState.trackedAlerts || {})) { + const id = alert.alertId; + const uuid = alert.alertUuid; + wrappedUUIDs.set(id, uuid); + } + + for (const alert of Object.values(wrappedState.trackedAlertsRecovered || {})) { + const id = alert.alertId; + const uuid = alert.alertUuid; + wrappedUUIDs.set(id, uuid); + } + + for (const [id, alert] of Object.entries(state.alertInstances || {})) { + const uuid = alert?.meta?.uuid || 'uuid-is-missing'; + expect(uuid).to.be(wrappedUUIDs.get(id)); + } + + for (const [id, alert] of Object.entries(state.alertRecoveredInstances || {})) { + const uuid = alert?.meta?.uuid || 'uuid-is-missing'; + expect(uuid).to.be(wrappedUUIDs.get(id)); + } + }); + }); + }); }); } diff --git a/x-pack/test/rule_registry/common/lib/helpers/cleanup_target_indices.ts b/x-pack/test/rule_registry/common/lib/helpers/cleanup_target_indices.ts index a249c57d8c3dc..efebb438f98a7 100644 --- a/x-pack/test/rule_registry/common/lib/helpers/cleanup_target_indices.ts +++ b/x-pack/test/rule_registry/common/lib/helpers/cleanup_target_indices.ts @@ -17,7 +17,7 @@ export const cleanupTargetIndices = async (getService: GetService, user: User, s const aliasMap = await es.indices.getAlias({ name: targetIndices, allow_no_indices: true, - expand_wildcards: 'open', + expand_wildcards: 'all', }); const indices = Object.keys(aliasMap); expect(indices.length > 0).to.be(true); diff --git a/x-pack/test/rule_registry/common/lib/helpers/index.ts b/x-pack/test/rule_registry/common/lib/helpers/index.ts index 695f71021d5ab..25128a5807320 100644 --- a/x-pack/test/rule_registry/common/lib/helpers/index.ts +++ b/x-pack/test/rule_registry/common/lib/helpers/index.ts @@ -11,4 +11,6 @@ export * from './create_transaction_metric'; export * from './get_alerts_target_indices'; export * from './wait_until_next_execution'; export * from './cleanup_target_indices'; +export * from './cleanup_registry_indices'; export * from './delete_alert'; +export * from './mock_alert_factory'; diff --git a/x-pack/test/rule_registry/common/lib/helpers/mock_alert_factory.ts b/x-pack/test/rule_registry/common/lib/helpers/mock_alert_factory.ts new file mode 100644 index 0000000000000..cdc9d15257120 --- /dev/null +++ b/x-pack/test/rule_registry/common/lib/helpers/mock_alert_factory.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +// implements a minimal mock alert factory used by a few tests + +let UUIDCounter = 1; + +class MockAlert { + id: string; + + constructor(alertId: string) { + this.id = alertId; + } + getUuid() { + return `uuid-${UUIDCounter++}`; + } +} + +export function getMockAlertFactory() { + return { + create(alertId: string) { + return new MockAlert(alertId); + }, + }; +} diff --git a/x-pack/test/rule_registry/spaces_only/tests/trial/__snapshots__/create_rule.snap b/x-pack/test/rule_registry/spaces_only/tests/trial/__snapshots__/create_rule.snap index a080cbf7247a6..e76a791652f93 100644 --- a/x-pack/test/rule_registry/spaces_only/tests/trial/__snapshots__/create_rule.snap +++ b/x-pack/test/rule_registry/spaces_only/tests/trial/__snapshots__/create_rule.snap @@ -17,6 +17,9 @@ Object { "kibana.alert.evaluation.value": Array [ 50, ], + "kibana.alert.flapping": Array [ + false, + ], "kibana.alert.instance.id": Array [ "apm.transaction_error_rate_opbeans-go_request_ENVIRONMENT_NOT_DEFINED", ], @@ -32,9 +35,22 @@ Object { "kibana.alert.rule.name": Array [ "Failed transaction rate threshold | opbeans-go", ], + "kibana.alert.rule.parameters": Array [ + Object { + "environment": "ENVIRONMENT_ALL", + "serviceName": "opbeans-go", + "threshold": 30, + "transactionType": "request", + "windowSize": 5, + "windowUnit": "m", + }, + ], "kibana.alert.rule.producer": Array [ "apm", ], + "kibana.alert.rule.revision": Array [ + 0, + ], "kibana.alert.rule.rule_type_id": Array [ "apm.transaction_error_rate", ], @@ -81,6 +97,9 @@ Object { "kibana.alert.evaluation.value": Array [ 50, ], + "kibana.alert.flapping": Array [ + false, + ], "kibana.alert.instance.id": Array [ "apm.transaction_error_rate_opbeans-go_request_ENVIRONMENT_NOT_DEFINED", ], @@ -96,9 +115,22 @@ Object { "kibana.alert.rule.name": Array [ "Failed transaction rate threshold | opbeans-go", ], + "kibana.alert.rule.parameters": Array [ + Object { + "environment": "ENVIRONMENT_ALL", + "serviceName": "opbeans-go", + "threshold": 30, + "transactionType": "request", + "windowSize": 5, + "windowUnit": "m", + }, + ], "kibana.alert.rule.producer": Array [ "apm", ], + "kibana.alert.rule.revision": Array [ + 0, + ], "kibana.alert.rule.rule_type_id": Array [ "apm.transaction_error_rate", ], diff --git a/x-pack/test/rule_registry/spaces_only/tests/trial/create_rule.ts b/x-pack/test/rule_registry/spaces_only/tests/trial/create_rule.ts index 2e8358b239561..9a6f1dfa6e972 100644 --- a/x-pack/test/rule_registry/spaces_only/tests/trial/create_rule.ts +++ b/x-pack/test/rule_registry/spaces_only/tests/trial/create_rule.ts @@ -9,16 +9,20 @@ import expect from '@kbn/expect'; import { ALERT_DURATION, ALERT_END, + ALERT_INSTANCE_ID, ALERT_RULE_EXECUTION_UUID, ALERT_RULE_UUID, ALERT_START, ALERT_STATUS, + ALERT_TIME_RANGE, ALERT_UUID, EVENT_KIND, VERSION, } from '@kbn/rule-data-utils'; import { omit } from 'lodash'; import { Rule } from '@kbn/alerting-plugin/common'; +import { SerializedConcreteTaskInstance } from '@kbn/task-manager-plugin/server/task'; +import type { RuleTaskState } from '@kbn/alerting-state-types'; import type { FtrProviderContext } from '../../../common/ftr_provider_context'; import { getAlertsTargetIndices, @@ -32,6 +36,7 @@ import { import { AlertDef, AlertParams } from '../../../common/types'; import { APM_METRIC_INDEX_NAME } from '../../../common/constants'; import { obsOnly } from '../../../common/lib/authentication/users'; +import { getEventLog } from '../../../../alerting_api_integration/common/lib/get_event_log'; const SPACE_ID = 'space1'; @@ -39,8 +44,7 @@ const SPACE_ID = 'space1'; export default function registryRulesApiTest({ getService }: FtrProviderContext) { const es = getService('es'); - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/125851 - describe.skip('Rule Registry API', () => { + describe('Rule Registry API', async () => { describe('with write permissions', () => { it('does not bootstrap indices on plugin startup', async () => { const { body: targetIndices } = await getAlertsTargetIndices(getService, obsOnly, SPACE_ID); @@ -119,7 +123,7 @@ export default function registryRulesApiTest({ getService }: FtrProviderContext) }, }, }); - expect(res).to.be.empty(); + expect(res.hits.hits).to.be.empty(); } catch (exc) { expect(exc.message).contain('index_not_found_exception'); } @@ -151,7 +155,7 @@ export default function registryRulesApiTest({ getService }: FtrProviderContext) }, }, }); - expect(res).to.be.empty(); + expect(res.hits.hits).to.be.empty(); } catch (exc) { expect(exc.message).contain('index_not_found_exception'); } @@ -195,9 +199,25 @@ export default function registryRulesApiTest({ getService }: FtrProviderContext) ALERT_UUID, ALERT_RULE_EXECUTION_UUID, ALERT_RULE_UUID, + ALERT_TIME_RANGE, VERSION, ]; + const alertInstanceId = alertEvent[ALERT_INSTANCE_ID]?.[0]; + const alertUuid = alertEvent[ALERT_UUID]?.[0]; + const executionUuid = alertEvent[ALERT_RULE_EXECUTION_UUID]?.[0]; + expect(typeof alertUuid).to.be('string'); + expect(typeof executionUuid).to.be('string'); + + await checkEventLogAlertUuids( + getService, + SPACE_ID, + createResponse.alert.id, + alertInstanceId, + alertUuid, + executionUuid + ); + const toCompare = omit(alertEvent, exclude); expectSnapshot(toCompare).toMatch(); @@ -259,3 +279,54 @@ export default function registryRulesApiTest({ getService }: FtrProviderContext) }); }); } + +async function checkEventLogAlertUuids( + getService: FtrProviderContext['getService'], + spaceId: string, + ruleId: string, + alertInstanceId: string, + alertUuid: string, + executionUuid: string +) { + const es = getService('es'); + const retry = getService('retry'); + + const docs: Awaited> = []; + await retry.waitFor('getting event log docs', async () => { + docs.push(...(await getEventLogDocs())); + return docs.length > 0; + }); + + expect(docs.length).to.be.greaterThan(0); + for (const doc of docs) { + expect(doc?.kibana?.alert?.uuid).to.be(alertUuid); + } + + // check that the task doc has the same UUID + const taskDoc = await es.get<{ task: SerializedConcreteTaskInstance }>({ + index: '.kibana_task_manager', + id: `task:${ruleId}`, + }); + + const ruleStateString = taskDoc._source?.task.state || 'task-state-is-missing'; + const ruleState: RuleTaskState = JSON.parse(ruleStateString); + if (ruleState.alertInstances?.[alertInstanceId]) { + expect(ruleState.alertInstances[alertInstanceId].meta?.uuid).to.be(alertUuid); + } else if (ruleState.alertRecoveredInstances?.[alertInstanceId]) { + expect(ruleState.alertRecoveredInstances[alertInstanceId].meta?.uuid).to.be(alertUuid); + } else { + expect(false).to.be('alert instance not found in task doc'); + } + + function getEventLogDocs() { + return getEventLog({ + getService, + spaceId, + type: 'alert', + id: ruleId, + provider: 'alerting', + actions: new Map([['active-instance', { equal: 1 }]]), + filter: `kibana.alert.rule.execution.uuid: ${executionUuid}`, + }); + } +} diff --git a/x-pack/test/rule_registry/spaces_only/tests/trial/get_summarized_alerts.ts b/x-pack/test/rule_registry/spaces_only/tests/trial/get_summarized_alerts.ts index 3adff40258ca5..32f47b708e545 100644 --- a/x-pack/test/rule_registry/spaces_only/tests/trial/get_summarized_alerts.ts +++ b/x-pack/test/rule_registry/spaces_only/tests/trial/get_summarized_alerts.ts @@ -37,7 +37,7 @@ import { MockAlertState, MockAllowedActionGroups, } from '../../../common/types'; -import { cleanupRegistryIndices } from '../../../common/lib/helpers/cleanup_registry_indices'; +import { cleanupRegistryIndices, getMockAlertFactory } from '../../../common/lib/helpers'; // eslint-disable-next-line import/no-default-export export default function createGetSummarizedAlertsTest({ getService }: FtrProviderContext) { @@ -173,7 +173,7 @@ export default function createGetSummarizedAlertsTest({ getService }: FtrProvide producer: 'observability.test', }, services: { - alertFactory: { create: sinon.stub() }, + alertFactory: getMockAlertFactory(), shouldWriteAlerts: sinon.stub().returns(true), }, flappingSettings: DEFAULT_FLAPPING_SETTINGS, @@ -332,7 +332,7 @@ export default function createGetSummarizedAlertsTest({ getService }: FtrProvide producer: 'observability.test', }, services: { - alertFactory: { create: sinon.stub() }, + alertFactory: getMockAlertFactory(), shouldWriteAlerts: sinon.stub().returns(true), }, flappingSettings: DEFAULT_FLAPPING_SETTINGS, diff --git a/x-pack/test/rule_registry/spaces_only/tests/trial/lifecycle_executor.ts b/x-pack/test/rule_registry/spaces_only/tests/trial/lifecycle_executor.ts index 22810df3dd95d..80334e09f6999 100644 --- a/x-pack/test/rule_registry/spaces_only/tests/trial/lifecycle_executor.ts +++ b/x-pack/test/rule_registry/spaces_only/tests/trial/lifecycle_executor.ts @@ -5,6 +5,14 @@ * 2.0. */ +// WARNING: This test running in Function Test Runner is building a live +// LifecycleRuleExecutor, feeding it some mock data, but letting it write +// it's various alerts to indices. I suspect it's quite fragile, and I +// added this comment to fix some fragility in the way the alert factory +// was built. I suspect it will suffer more such things in the future. +// I fixed this as a drive-by, but opened an issue to do something later, +// if needed: https://github.com/elastic/kibana/issues/144557 + import { type Subject, ReplaySubject } from 'rxjs'; import type { ElasticsearchClient, Logger, LogMeta } from '@kbn/core/server'; import sinon from 'sinon'; @@ -29,7 +37,7 @@ import { MockAlertState, MockAllowedActionGroups, } from '../../../common/types'; -import { cleanupRegistryIndices } from '../../../common/lib/helpers/cleanup_registry_indices'; +import { cleanupRegistryIndices, getMockAlertFactory } from '../../../common/lib/helpers'; // eslint-disable-next-line import/no-default-export export default function createLifecycleExecutorApiTest({ getService }: FtrProviderContext) { @@ -57,8 +65,7 @@ export default function createLifecycleExecutorApiTest({ getService }: FtrProvid return Promise.resolve(client); }; - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/125851 - describe.skip('createLifecycleExecutor', () => { + describe('createLifecycleExecutor', () => { let ruleDataClient: IRuleDataClient; let pluginStop$: Subject; @@ -170,13 +177,15 @@ export default function createLifecycleExecutorApiTest({ getService }: FtrProvid return Promise.resolve({ state }); }); + const ruleId = 'rule-id'; // Create the options with the minimal amount of values to test the lifecycle executor const options = { - alertId: id, + alertId: ruleId, spaceId: 'default', tags: ['test'], startedAt: new Date(), rule: { + id: ruleId, name: 'test rule', ruleTypeId: 'observability.test.fake', ruleTypeName: 'test', @@ -184,9 +193,14 @@ export default function createLifecycleExecutorApiTest({ getService }: FtrProvid producer: 'observability.test', }, services: { - alertFactory: { create: sinon.stub() }, + alertFactory: getMockAlertFactory(), shouldWriteAlerts: sinon.stub().returns(true), }, + flappingSettings: { + enabled: false, + lookBackWindow: 20, + statusChangeThreshold: 4, + }, } as unknown as RuleExecutorOptions< MockRuleParams, WrappedLifecycleRuleState, @@ -205,6 +219,9 @@ export default function createLifecycleExecutorApiTest({ getService }: FtrProvid }, }); + const alertUuid = executorResult.state.trackedAlerts['host-01'].alertUuid; + expect(alertUuid).to.be('uuid-1'); + // We need to refresh the index so the data is available for the next call await es.indices.refresh({ index: `${ruleDataClient.indexName}*` }); diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 15b63aac62602..2e92bd5f38c3d 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -120,6 +120,7 @@ "@kbn/files-plugin", "@kbn/shared-ux-file-types", "@kbn/securitysolution-io-ts-alerting-types", + "@kbn/alerting-state-types", "@kbn/assetManager-plugin", ] } diff --git a/yarn.lock b/yarn.lock index ff341cfeb0026..5d5602ee6a37b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2797,6 +2797,10 @@ version "0.0.0" uid "" +"@kbn/alerting-state-types@link:x-pack/packages/kbn-alerting-state-types": + version "0.0.0" + uid "" + "@kbn/alerts-as-data-utils@link:packages/kbn-alerts-as-data-utils": version "0.0.0" uid "" From eee78fc58c33f23afc6df1a380711a0b67add976 Mon Sep 17 00:00:00 2001 From: Rachel Shen Date: Mon, 3 Apr 2023 07:35:41 -0600 Subject: [PATCH 05/10] [Reporting][Docs] update readme with testing on windows information (#154074) ## Summary Adding information for the future for people who need to test reporting on a Windows machine --- x-pack/build_chromium/README.md | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/x-pack/build_chromium/README.md b/x-pack/build_chromium/README.md index df097e1cbef8b..5a240ffc06a0f 100644 --- a/x-pack/build_chromium/README.md +++ b/x-pack/build_chromium/README.md @@ -119,6 +119,48 @@ Here's the steps on how to test a Puppeteer upgrade, run these tests on Mac, Win - All functional and API tests that generate PDF and PNG files should pass. - Use a VM to run Kibana in a low-memory environment and try to generate a PNG of a dashboard that outputs as a 4MB file. Document the minimum requirements in the PR. +## Testing Chromium upgrades on a Windows Machine + +Directions on creating a build of Kibana off an existing PR can be found here: +https://www.elastic.co/guide/en/kibana/current/building-kibana.html +You will need this build to install on your windows device to test the in progress PR. + +The default extractor for Windows might give `Path too long errors`. +- Install the zipped file onto your C:\ directory in case the path actually is too long. +- Use 7Zip or WinZip to extract the contents of the kibana build. +Reference: This article can be helpful: +https://www.partitionwizard.com/disk-recovery/error-0x80010135-path-too-long.html + +For an elasticsearch cluster to base the latest kibana build with, you can use a snapshot.sh bash script to generate the latest build. Create a file called snapshot.sh and put the following into the file: + +``` +runQuery() { + curl --silent -XGET https://artifacts-api.elastic.co${1} +} +BUILD_HASH=$(runQuery /v1/versions/${VERSION}-SNAPSHOT/builds | jq -r '.builds[0]') +echo "Latest build hash :: $BUILD_HASH" +KBN_DOWNLOAD=$(runQuery /v1/versions/${VERSION}-SNAPSHOT/builds/$BUILD_HASH/projects/elasticsearch/packages/elasticsearch-${VERSION}-SNAPSHOT-windows-x86_64.zip) +echo $KBN_DOWNLOAD | jq -r '.package.url' +``` + +In the terminal once you have the snapshot.sh file written run: +chmod a+x snapshot.sh to make the file executable +Then set the version variable within the script to what you need by typing the following (in this example 8.8.0): +VERSION=8.8.0 ./snapshot.sh + +In the terminal you should see a web address that will give you a download of elasticsearch. + +You may need to disable xpack security in the elasticsearch.yml +xpack.security.enabled: false + +Make sure nothing is set in the kibana.yml + +Run `.\bin\elasticsearch.bat` in the elasticsearch directory first and then once it's up run `.\bin\kibana.bat` + +Navigate to localhost:5601 and there shouldn't be any prompts to set up security etc. To test PNG reporting, you may need to upload a license. Navigate to https://wiki.elastic.co/display/PM/Internal+License+-+X-Pack+and+Endgame and download the license.json from Internal Licenses. + +Navigate to Stack Management in Kibana and you can upload the license.json from internal licenses. You won't need to restart the cluster and should be able to test the Kibana feature as needed at this point. + ## Resources The following links provide helpful context about how the Chromium build works, and its prerequisites: From ff738dc971159265a7279d24dbd76b74e793453e Mon Sep 17 00:00:00 2001 From: Mark Hopkin Date: Mon, 3 Apr 2023 14:43:32 +0100 Subject: [PATCH 06/10] [Fleet] Fix functional test failures after new custom logs package released (#154240) These tests rely on the custom logs package being an integration package, 2.0.0 was released today which is an input package so the index templates are not created on install. --- .../apis/epm/custom_ingest_pipeline.ts | 19 +++---------------- .../apis/epm/final_pipeline.ts | 18 ++++-------------- 2 files changed, 7 insertions(+), 30 deletions(-) diff --git a/x-pack/test/fleet_api_integration/apis/epm/custom_ingest_pipeline.ts b/x-pack/test/fleet_api_integration/apis/epm/custom_ingest_pipeline.ts index 8d7a6323d5f73..4afcf4759d2a0 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/custom_ingest_pipeline.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/custom_ingest_pipeline.ts @@ -14,14 +14,12 @@ const TEST_INDEX = 'logs-log.log-test'; const CUSTOM_PIPELINE = 'logs-log.log@custom'; -let pkgVersion: string; - export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; const supertest = getService('supertest'); const es = getService('es'); const esArchiver = getService('esArchiver'); - + const LOG_INTEGRATION_VERSION = '1.1.2'; describe('custom ingest pipeline for fleet managed datastreams', () => { skipIfNoDockerRegistry(providerContext); before(async () => { @@ -29,27 +27,16 @@ export default function (providerContext: FtrProviderContext) { }); setupFleetAndAgents(providerContext); - // Use the custom log package to test the custom ingest pipeline before(async () => { - const { body: getPackagesRes } = await supertest.get( - `/api/fleet/epm/packages?prerelease=true` - ); - const logPackage = getPackagesRes.items.find((p: any) => p.name === 'log'); - if (!logPackage) { - throw new Error('No log package'); - } - - pkgVersion = logPackage.version; - await supertest - .post(`/api/fleet/epm/packages/log/${pkgVersion}`) + .post(`/api/fleet/epm/packages/log/${LOG_INTEGRATION_VERSION}`) .set('kbn-xsrf', 'xxxx') .send({ force: true }) .expect(200); }); after(async () => { await supertest - .delete(`/api/fleet/epm/packages/log/${pkgVersion}`) + .delete(`/api/fleet/epm/packages/log/${LOG_INTEGRATION_VERSION}`) .set('kbn-xsrf', 'xxxx') .send({ force: true }) .expect(200); diff --git a/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts b/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts index 5e677bd96d54a..164df10481be5 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts @@ -14,9 +14,9 @@ const TEST_INDEX = 'logs-log.log-test'; const FINAL_PIPELINE_ID = '.fleet_final_pipeline-1'; -const FINAL_PIPELINE_VERSION = 1; +const LOG_INTEGRATION_VERSION = '1.1.2'; -let pkgVersion: string; +const FINAL_PIPELINE_VERSION = 1; export default function (providerContext: FtrProviderContext) { const { getService } = providerContext; @@ -42,25 +42,15 @@ export default function (providerContext: FtrProviderContext) { // Use the custom log package to test the fleet final pipeline before(async () => { - const { body: getPackagesRes } = await supertest.get( - `/api/fleet/epm/packages?prerelease=true` - ); - const logPackage = getPackagesRes.items.find((p: any) => p.name === 'log'); - if (!logPackage) { - throw new Error('No log package'); - } - - pkgVersion = logPackage.version; - await supertest - .post(`/api/fleet/epm/packages/log/${pkgVersion}`) + .post(`/api/fleet/epm/packages/log/${LOG_INTEGRATION_VERSION}`) .set('kbn-xsrf', 'xxxx') .send({ force: true }) .expect(200); }); after(async () => { await supertest - .delete(`/api/fleet/epm/packages/log/${pkgVersion}`) + .delete(`/api/fleet/epm/packages/log/${LOG_INTEGRATION_VERSION}`) .set('kbn-xsrf', 'xxxx') .send({ force: true }) .expect(200); From b0e9b28a1be04ffa31ead9b857d09be4f020e766 Mon Sep 17 00:00:00 2001 From: Kevin Delemme Date: Mon, 3 Apr 2023 09:45:29 -0400 Subject: [PATCH 07/10] feat(slo): create rule from slo details page (#154175) --- .../slo_details/components/header_control.tsx | 150 ++++++++++++++---- .../pages/slo_details/slo_details.test.tsx | 41 ++++- .../public/pages/slo_details/slo_details.tsx | 45 +----- 3 files changed, 153 insertions(+), 83 deletions(-) diff --git a/x-pack/plugins/observability/public/pages/slo_details/components/header_control.tsx b/x-pack/plugins/observability/public/pages/slo_details/components/header_control.tsx index a995fa91c3a61..931b503caa1a5 100644 --- a/x-pack/plugins/observability/public/pages/slo_details/components/header_control.tsx +++ b/x-pack/plugins/observability/public/pages/slo_details/components/header_control.tsx @@ -10,6 +10,10 @@ import React, { useState } from 'react'; import { i18n } from '@kbn/i18n'; import { SLOWithSummaryResponse } from '@kbn/slo-schema'; +import { isApmIndicatorType } from '../../../utils/slo/indicator'; +import { convertSliApmParamsToApmAppDeeplinkUrl } from '../../../utils/slo/convert_sli_apm_params_to_apm_app_deeplink_url'; +import { SLO_BURN_RATE_RULE_ID } from '../../../../common/constants'; +import { sloFeatureId } from '../../../../common'; import { paths } from '../../../config/paths'; import { useKibana } from '../../../utils/kibana_react'; import { ObservabilityAppServices } from '../../../application/types'; @@ -24,9 +28,11 @@ export function HeaderControl({ isLoading, slo }: Props) { const { application: { navigateToUrl }, http: { basePath }, + triggersActionsUi: { getAddRuleFlyout: AddRuleFlyout }, } = useKibana().services; const { hasWriteCapabilities } = useCapabilities(); const [isPopoverOpen, setIsPopoverOpen] = useState(false); + const [isRuleFlyoutVisible, setRuleFlyoutVisibility] = useState(false); const handleActionsClick = () => setIsPopoverOpen((value) => !value); const closePopover = () => setIsPopoverOpen(false); @@ -37,43 +43,117 @@ export function HeaderControl({ isLoading, slo }: Props) { } }; + const onCloseRuleFlyout = () => { + setRuleFlyoutVisibility(false); + }; + + const handleOpenRuleFlyout = () => { + closePopover(); + setRuleFlyoutVisibility(true); + }; + + const handleNavigateToApm = () => { + if ( + slo?.indicator.type === 'sli.apm.transactionDuration' || + slo?.indicator.type === 'sli.apm.transactionErrorRate' + ) { + const { + indicator: { + params: { environment, filter, service, transactionName, transactionType }, + }, + timeWindow: { duration }, + } = slo; + + const url = convertSliApmParamsToApmAppDeeplinkUrl({ + duration, + environment, + filter, + service, + transactionName, + transactionType, + }); + + navigateToUrl(basePath.prepend(url)); + } + }; + return ( - - {i18n.translate('xpack.observability.slo.sloDetails.headerControl.actions', { - defaultMessage: 'Actions', - })} - - } - isOpen={isPopoverOpen} - closePopover={closePopover} - > - + - {i18n.translate('xpack.observability.slo.sloDetails.headerControl.edit', { - defaultMessage: 'Edit', + {i18n.translate('xpack.observability.slo.sloDetails.headerControl.actions', { + defaultMessage: 'Actions', })} - , - ]} - /> - + + } + isOpen={isPopoverOpen} + closePopover={closePopover} + > + + {i18n.translate('xpack.observability.slo.sloDetails.headerControl.edit', { + defaultMessage: 'Edit', + })} + , + + {i18n.translate( + 'xpack.observability.slo.sloDetails.headerControl.createBurnRateRule', + { + defaultMessage: 'Create alert rule', + } + )} + , + ].concat( + !!slo && isApmIndicatorType(slo.indicator.type) + ? [ + + {i18n.translate( + 'xpack.observability.slos.sloDetails.headerControl.exploreInApm', + { + defaultMessage: 'Explore in APM', + } + )} + , + ] + : [] + )} + /> + + {!!slo && isRuleFlyoutVisible ? ( + + ) : null} + ); } diff --git a/x-pack/plugins/observability/public/pages/slo_details/slo_details.test.tsx b/x-pack/plugins/observability/public/pages/slo_details/slo_details.test.tsx index 55eb42ec95654..517a00b9cad70 100644 --- a/x-pack/plugins/observability/public/pages/slo_details/slo_details.test.tsx +++ b/x-pack/plugins/observability/public/pages/slo_details/slo_details.test.tsx @@ -6,7 +6,7 @@ */ import React from 'react'; -import { screen } from '@testing-library/react'; +import { fireEvent, screen } from '@testing-library/react'; import { useKibana } from '../../utils/kibana_react'; import { useParams } from 'react-router-dom'; @@ -57,6 +57,11 @@ const mockKibana = () => { prepend: mockBasePathPrepend, }, }, + triggersActionsUi: { + getAddRuleFlyout: jest.fn(() => ( +
mocked component
+ )), + }, uiSettings: { get: (settings: string) => { if (settings === 'dateFormat') return 'YYYY-MM-DD'; @@ -149,8 +154,32 @@ describe('SLO Details Page', () => { expect(screen.queryAllByTestId('wideChartLoading').length).toBe(0); }); + it("renders a 'Edit' button under actions menu", async () => { + const slo = buildSlo(); + useParamsMock.mockReturnValue(slo.id); + useFetchSloDetailsMock.mockReturnValue({ isLoading: false, slo }); + useLicenseMock.mockReturnValue({ hasAtLeast: () => true }); + + render(); + + fireEvent.click(screen.getByTestId('o11yHeaderControlActionsButton')); + expect(screen.queryByTestId('sloDetailsHeaderControlPopoverEdit')).toBeTruthy(); + }); + + it("renders a 'Create alert rule' button under actions menu", async () => { + const slo = buildSlo(); + useParamsMock.mockReturnValue(slo.id); + useFetchSloDetailsMock.mockReturnValue({ isLoading: false, slo }); + useLicenseMock.mockReturnValue({ hasAtLeast: () => true }); + + render(); + + fireEvent.click(screen.getByTestId('o11yHeaderControlActionsButton')); + expect(screen.queryByTestId('sloDetailsHeaderControlPopoverCreateRule')).toBeTruthy(); + }); + describe('when an APM SLO is loaded', () => { - it("should render a 'Explore in APM' button", async () => { + it("renders a 'Explore in APM' button under actions menu", async () => { const slo = buildSlo({ indicator: buildApmAvailabilityIndicator() }); useParamsMock.mockReturnValue(slo.id); useFetchSloDetailsMock.mockReturnValue({ isLoading: false, slo }); @@ -158,12 +187,13 @@ describe('SLO Details Page', () => { render(); - expect(screen.queryByTestId('sloDetailsExploreInApmButton')).toBeTruthy(); + fireEvent.click(screen.getByTestId('o11yHeaderControlActionsButton')); + expect(screen.queryByTestId('sloDetailsHeaderControlPopoverExploreInApm')).toBeTruthy(); }); }); describe('when an Custom KQL SLO is loaded', () => { - it("should not render a 'Explore in APM' button", async () => { + it("does not render a 'Explore in APM' button under actions menu", async () => { const slo = buildSlo(); useParamsMock.mockReturnValue(slo.id); useFetchSloDetailsMock.mockReturnValue({ isLoading: false, slo }); @@ -171,7 +201,8 @@ describe('SLO Details Page', () => { render(); - expect(screen.queryByTestId('sloDetailsExploreInApmButton')).toBeFalsy(); + fireEvent.click(screen.getByTestId('o11yHeaderControlActionsButton')); + expect(screen.queryByTestId('sloDetailsHeaderControlPopoverExploreInApm')).toBeFalsy(); }); }); }); diff --git a/x-pack/plugins/observability/public/pages/slo_details/slo_details.tsx b/x-pack/plugins/observability/public/pages/slo_details/slo_details.tsx index bbb606e76c079..fec93a1fb6b2c 100644 --- a/x-pack/plugins/observability/public/pages/slo_details/slo_details.tsx +++ b/x-pack/plugins/observability/public/pages/slo_details/slo_details.tsx @@ -8,7 +8,7 @@ import React from 'react'; import { useParams } from 'react-router-dom'; import { EuiBreadcrumbProps } from '@elastic/eui/src/components/breadcrumbs/breadcrumb'; -import { EuiButtonEmpty, EuiLoadingSpinner } from '@elastic/eui'; +import { EuiLoadingSpinner } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import type { IBasePath } from '@kbn/core-http-browser'; import type { SLOWithSummaryResponse } from '@kbn/slo-schema'; @@ -25,8 +25,6 @@ import { HeaderControl } from './components/header_control'; import { paths } from '../../config/paths'; import type { SloDetailsPathParams } from './types'; import type { ObservabilityAppServices } from '../../application/types'; -import { isApmIndicatorType } from '../../utils/slo/indicator'; -import { convertSliApmParamsToApmAppDeeplinkUrl } from '../../utils/slo/convert_sli_apm_params_to_apm_app_deeplink_url'; export function SloDetailsPage() { const { @@ -50,50 +48,11 @@ export function SloDetailsPage() { navigateToUrl(basePath.prepend(paths.observability.slos)); } - const handleNavigateToApm = () => { - if ( - slo?.indicator.type === 'sli.apm.transactionDuration' || - slo?.indicator.type === 'sli.apm.transactionErrorRate' - ) { - const { - indicator: { - params: { environment, filter, service, transactionName, transactionType }, - }, - timeWindow: { duration }, - } = slo; - - const url = convertSliApmParamsToApmAppDeeplinkUrl({ - duration, - environment, - filter, - service, - transactionName, - transactionType, - }); - - navigateToUrl(basePath.prepend(url)); - } - }; - return ( , - rightSideItems: [ - , - !!slo && isApmIndicatorType(slo.indicator.type) ? ( - - {i18n.translate('xpack.observability.slos.sloDetails.exploreInApm', { - defaultMessage: 'Explore in APM', - })} - - ) : null, - ], + rightSideItems: [], bottomBorder: false, }} data-test-subj="sloDetailsPage" From da4e7826c4ce16c14da569ec9819eccb26ea081f Mon Sep 17 00:00:00 2001 From: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> Date: Mon, 3 Apr 2023 10:04:18 -0400 Subject: [PATCH 08/10] [DOCS] Add details to xpack.fleet.outputs settings (#154247) This updates the [Fleet settings in Kibana](https://www.elastic.co/guide/en/kibana/current/fleet-settings-kb.html) page with details for the `xpack.fleet.outputs` settings. @nchaulet and @jeanfabrice In addition to what we discussed I updated the `is_default` setting as well, but it probably could use some fixing up (i.e., is the setting for "all non-monitoring data" or something else)? Preview: ![Screenshot 2023-04-03 at 9 25 55 AM](https://user-images.githubusercontent.com/41695641/229524885-f468deff-947f-4ae8-8f87-5044ca3eae7f.png) --- docs/settings/fleet-settings.asciidoc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/settings/fleet-settings.asciidoc b/docs/settings/fleet-settings.asciidoc index 41319a85612ee..28a855c29bace 100644 --- a/docs/settings/fleet-settings.asciidoc +++ b/docs/settings/fleet-settings.asciidoc @@ -142,6 +142,8 @@ If configured in your `kibana.yml`, output settings are grayed out and unavailable in the {fleet} UI. To make these settings editable in the UI, do not configure them in the configuration file. + +NOTE: The `xpack.fleet.outputs` settings are intended for advanced configurations such as having multiple outputs. We recommend not enabling the `xpack.fleet.agents.elasticsearch.host` settings when using `xpack.fleet.outputs`. ++ .Required properties of `xpack.fleet.outputs` [%collapsible%open] ===== @@ -161,7 +163,9 @@ configure them in the configuration file. [%collapsible%open] ===== `is_default`::: - If `true`, this output is the default output. + If `true`, the output specified in `xpack.fleet.outputs` will be the one used to send agent data unless there is another one configured specifically for the agent policy. + `is_default_monitoring`::: + If `true`, the output specified in `xpack.fleet.outputs` will be the one used to send agent monitoring data unless there is another one configured specifically for the agent policy. ===== + Example configuration: From 597f4bb8793ffc7ca3155da4cd9076365902b291 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Mon, 3 Apr 2023 15:06:24 +0100 Subject: [PATCH 09/10] skip flaky suite (#154220,#154221,#154222,#154223,#154224,#154225,#154226) --- .../fleet_api_integration/apis/epm/final_pipeline.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts b/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts index 164df10481be5..dff9c8fd6eeed 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/final_pipeline.ts @@ -33,7 +33,14 @@ export default function (providerContext: FtrProviderContext) { .expect(201); } - describe('fleet_final_pipeline', () => { + // FLAKY: https://github.com/elastic/kibana/issues/154220 + // FLAKY: https://github.com/elastic/kibana/issues/154221 + // FLAKY: https://github.com/elastic/kibana/issues/154222 + // FLAKY: https://github.com/elastic/kibana/issues/154223 + // FLAKY: https://github.com/elastic/kibana/issues/154224 + // FLAKY: https://github.com/elastic/kibana/issues/154225 + // FLAKY: https://github.com/elastic/kibana/issues/154226 + describe.skip('fleet_final_pipeline', () => { skipIfNoDockerRegistry(providerContext); before(async () => { await esArchiver.load('x-pack/test/functional/es_archives/fleet/empty_fleet_server'); From a18627d27a665fc325eeabc709920ead785cd31f Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Mon, 3 Apr 2023 15:10:17 +0100 Subject: [PATCH 10/10] skip flaky suite (#154227) --- .../fleet_api_integration/apis/epm/custom_ingest_pipeline.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/fleet_api_integration/apis/epm/custom_ingest_pipeline.ts b/x-pack/test/fleet_api_integration/apis/epm/custom_ingest_pipeline.ts index 4afcf4759d2a0..5022f47b414bd 100644 --- a/x-pack/test/fleet_api_integration/apis/epm/custom_ingest_pipeline.ts +++ b/x-pack/test/fleet_api_integration/apis/epm/custom_ingest_pipeline.ts @@ -59,7 +59,8 @@ export default function (providerContext: FtrProviderContext) { } }); - describe('Without custom pipeline', () => { + // FLAKY: https://github.com/elastic/kibana/issues/154227 + describe.skip('Without custom pipeline', () => { it('Should write doc correctly', async () => { const res = await es.index({ index: 'logs-log.log-test',