diff --git a/packages/kbn-alerts-as-data-utils/src/field_maps/index.ts b/packages/kbn-alerts-as-data-utils/src/field_maps/index.ts index 9aef7690b343c..04928dbd46191 100644 --- a/packages/kbn-alerts-as-data-utils/src/field_maps/index.ts +++ b/packages/kbn-alerts-as-data-utils/src/field_maps/index.ts @@ -9,4 +9,5 @@ export * from './alert_field_map'; export * from './ecs_field_map'; export * from './legacy_alert_field_map'; +export * from './legacy_experimental_field_map'; export type { FieldMap, MultiField } from './types'; diff --git a/packages/kbn-alerts-as-data-utils/src/field_maps/legacy_experimental_field_map.ts b/packages/kbn-alerts-as-data-utils/src/field_maps/legacy_experimental_field_map.ts new file mode 100644 index 0000000000000..8c8445ad6761a --- /dev/null +++ b/packages/kbn-alerts-as-data-utils/src/field_maps/legacy_experimental_field_map.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ALERT_EVALUATION_THRESHOLD, ALERT_EVALUATION_VALUE } from '@kbn/rule-data-utils'; + +export const legacyExperimentalFieldMap = { + [ALERT_EVALUATION_THRESHOLD]: { + type: 'scaled_float', + scaling_factor: 100, + required: false, + }, + [ALERT_EVALUATION_VALUE]: { type: 'scaled_float', scaling_factor: 100, required: false }, +} as const; + +export type ExperimentalRuleFieldMap = typeof legacyExperimentalFieldMap; diff --git a/x-pack/plugins/alerting/common/alert_schema/field_maps/component_template_from_field_map.test.ts b/x-pack/plugins/alerting/common/alert_schema/field_maps/component_template_from_field_map.test.ts new file mode 100644 index 0000000000000..17bde15bd01a5 --- /dev/null +++ b/x-pack/plugins/alerting/common/alert_schema/field_maps/component_template_from_field_map.test.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { getComponentTemplateFromFieldMap } from './component_template_from_field_map'; +import { testFieldMap, expectedTestMapping } from './mapping_from_field_map.test'; + +describe('getComponentTemplateFromFieldMap', () => { + it('correctly creates component template from field map', () => { + expect( + getComponentTemplateFromFieldMap({ name: 'test-mappings', fieldMap: testFieldMap }) + ).toEqual({ + name: 'test-mappings', + _meta: { + managed: true, + }, + template: { + settings: {}, + mappings: { + dynamic: 'strict', + ...expectedTestMapping, + }, + }, + }); + }); + + it('correctly creates component template with settings when includeSettings = true', () => { + expect( + getComponentTemplateFromFieldMap({ + name: 'test-mappings', + fieldMap: testFieldMap, + includeSettings: true, + }) + ).toEqual({ + name: 'test-mappings', + _meta: { + managed: true, + }, + template: { + settings: { + number_of_shards: 1, + 'index.mapping.total_fields.limit': 1500, + }, + mappings: { + dynamic: 'strict', + ...expectedTestMapping, + }, + }, + }); + }); + + it('correctly creates component template with dynamic setting when defined', () => { + expect( + getComponentTemplateFromFieldMap({ + name: 'test-mappings', + fieldMap: testFieldMap, + includeSettings: true, + dynamic: false, + }) + ).toEqual({ + name: 'test-mappings', + _meta: { + managed: true, + }, + template: { + settings: { + number_of_shards: 1, + 'index.mapping.total_fields.limit': 1500, + }, + mappings: { + dynamic: false, + ...expectedTestMapping, + }, + }, + }); + }); +}); diff --git a/x-pack/plugins/alerting/common/alert_schema/field_maps/component_template_from_field_map.ts b/x-pack/plugins/alerting/common/alert_schema/field_maps/component_template_from_field_map.ts index 4fc36193a15d9..9d5c651e92cda 100644 --- a/x-pack/plugins/alerting/common/alert_schema/field_maps/component_template_from_field_map.ts +++ b/x-pack/plugins/alerting/common/alert_schema/field_maps/component_template_from_field_map.ts @@ -11,13 +11,15 @@ import { mappingFromFieldMap } from './mapping_from_field_map'; export interface GetComponentTemplateFromFieldMapOpts { name: string; - fieldLimit?: number; fieldMap: FieldMap; + includeSettings?: boolean; + dynamic?: 'strict' | false; } export const getComponentTemplateFromFieldMap = ({ name, fieldMap, - fieldLimit, + dynamic, + includeSettings, }: GetComponentTemplateFromFieldMapOpts): ClusterPutComponentTemplateRequest => { return { name, @@ -26,10 +28,16 @@ export const getComponentTemplateFromFieldMap = ({ }, template: { settings: { - number_of_shards: 1, - 'index.mapping.total_fields.limit': fieldLimit ?? 1000, + ...(includeSettings + ? { + number_of_shards: 1, + 'index.mapping.total_fields.limit': + Math.ceil(Object.keys(fieldMap).length / 1000) * 1000 + 500, + } + : {}), }, - mappings: mappingFromFieldMap(fieldMap, 'strict'), + + mappings: mappingFromFieldMap(fieldMap, dynamic ?? 'strict'), }, }; }; diff --git a/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts b/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts index f5eeeb8ba6c35..ff3d9aeecd79f 100644 --- a/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts +++ b/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts @@ -7,179 +7,183 @@ import { alertFieldMap, legacyAlertFieldMap, type FieldMap } from '@kbn/alerts-as-data-utils'; import { mappingFromFieldMap } from './mapping_from_field_map'; -describe('mappingFromFieldMap', () => { - const fieldMap: FieldMap = { - date_field: { - type: 'date', - array: false, - required: true, - }, - keyword_field: { - type: 'keyword', - array: false, - required: false, +export const testFieldMap: FieldMap = { + date_field: { + type: 'date', + array: false, + required: true, + }, + keyword_field: { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + long_field: { + type: 'long', + array: false, + required: false, + }, + multifield_field: { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + multi_fields: [ + { + flat_name: 'multifield_field.text', + name: 'text', + type: 'match_only_text', + }, + ], + }, + geopoint_field: { + type: 'geo_point', + array: false, + required: false, + }, + ip_field: { + type: 'ip', + array: false, + required: false, + }, + array_field: { + type: 'keyword', + array: true, + required: false, + ignore_above: 1024, + }, + nested_array_field: { + type: 'nested', + array: false, + required: false, + }, + 'nested_array_field.field1': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + 'nested_array_field.field2': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + scaled_float_field: { + type: 'scaled_float', + array: false, + required: false, + scaling_factor: 1000, + }, + constant_keyword_field: { + type: 'constant_keyword', + array: false, + required: false, + }, + 'parent_field.child1': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + 'parent_field.child2': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + unmapped_object: { + type: 'object', + required: false, + enabled: false, + }, + formatted_field: { + type: 'date_range', + required: false, + format: 'epoch_millis||strict_date_optional_time', + }, +}; +export const expectedTestMapping = { + properties: { + array_field: { ignore_above: 1024, + type: 'keyword', }, - long_field: { - type: 'long', - array: false, - required: false, + constant_keyword_field: { + type: 'constant_keyword', + }, + date_field: { + type: 'date', }, multifield_field: { - type: 'keyword', - array: false, - required: false, - ignore_above: 1024, - multi_fields: [ - { - flat_name: 'multifield_field.text', - name: 'text', + fields: { + text: { type: 'match_only_text', }, - ], + }, + ignore_above: 1024, + type: 'keyword', }, geopoint_field: { type: 'geo_point', - array: false, - required: false, }, ip_field: { type: 'ip', - array: false, - required: false, }, - array_field: { - type: 'keyword', - array: true, - required: false, + keyword_field: { ignore_above: 1024, + type: 'keyword', + }, + long_field: { + type: 'long', }, nested_array_field: { + properties: { + field1: { + ignore_above: 1024, + type: 'keyword', + }, + field2: { + ignore_above: 1024, + type: 'keyword', + }, + }, type: 'nested', - array: false, - required: false, }, - 'nested_array_field.field1': { - type: 'keyword', - array: false, - required: false, - ignore_above: 1024, - }, - 'nested_array_field.field2': { - type: 'keyword', - array: false, - required: false, - ignore_above: 1024, + parent_field: { + properties: { + child1: { + ignore_above: 1024, + type: 'keyword', + }, + child2: { + ignore_above: 1024, + type: 'keyword', + }, + }, }, scaled_float_field: { - type: 'scaled_float', - array: false, - required: false, scaling_factor: 1000, - }, - constant_keyword_field: { - type: 'constant_keyword', - array: false, - required: false, - }, - 'parent_field.child1': { - type: 'keyword', - array: false, - required: false, - ignore_above: 1024, - }, - 'parent_field.child2': { - type: 'keyword', - array: false, - required: false, - ignore_above: 1024, + type: 'scaled_float', }, unmapped_object: { - type: 'object', - required: false, enabled: false, + type: 'object', }, formatted_field: { type: 'date_range', - required: false, format: 'epoch_millis||strict_date_optional_time', }, - }; - const expectedMapping = { - properties: { - array_field: { - ignore_above: 1024, - type: 'keyword', - }, - constant_keyword_field: { - type: 'constant_keyword', - }, - date_field: { - type: 'date', - }, - multifield_field: { - fields: { - text: { - type: 'match_only_text', - }, - }, - ignore_above: 1024, - type: 'keyword', - }, - geopoint_field: { - type: 'geo_point', - }, - ip_field: { - type: 'ip', - }, - keyword_field: { - ignore_above: 1024, - type: 'keyword', - }, - long_field: { - type: 'long', - }, - nested_array_field: { - properties: { - field1: { - ignore_above: 1024, - type: 'keyword', - }, - field2: { - ignore_above: 1024, - type: 'keyword', - }, - }, - type: 'nested', - }, - parent_field: { - properties: { - child1: { - ignore_above: 1024, - type: 'keyword', - }, - child2: { - ignore_above: 1024, - type: 'keyword', - }, - }, - }, - scaled_float_field: { - scaling_factor: 1000, - type: 'scaled_float', - }, - unmapped_object: { - enabled: false, - type: 'object', - }, - formatted_field: { - type: 'date_range', - format: 'epoch_millis||strict_date_optional_time', - }, - }, - }; + }, +}; + +describe('mappingFromFieldMap', () => { it('correctly creates mapping from field map', () => { - expect(mappingFromFieldMap(fieldMap)).toEqual({ dynamic: 'strict', ...expectedMapping }); + expect(mappingFromFieldMap(testFieldMap)).toEqual({ + dynamic: 'strict', + ...expectedTestMapping, + }); expect(mappingFromFieldMap(alertFieldMap)).toEqual({ dynamic: 'strict', properties: { @@ -344,6 +348,9 @@ describe('mappingFromFieldMap', () => { }); it('uses dynamic setting if specified', () => { - expect(mappingFromFieldMap(fieldMap, true)).toEqual({ dynamic: true, ...expectedMapping }); + expect(mappingFromFieldMap(testFieldMap, true)).toEqual({ + dynamic: true, + ...expectedTestMapping, + }); }); }); diff --git a/x-pack/plugins/alerting/server/alerts_service/alerts_service.mock.ts b/x-pack/plugins/alerting/server/alerts_service/alerts_service.mock.ts index d11e95f909c19..a3754d66e1cad 100644 --- a/x-pack/plugins/alerting/server/alerts_service/alerts_service.mock.ts +++ b/x-pack/plugins/alerting/server/alerts_service/alerts_service.mock.ts @@ -8,10 +8,9 @@ const creatAlertsServiceMock = () => { return jest.fn().mockImplementation(() => { return { - initialize: jest.fn(), register: jest.fn(), isInitialized: jest.fn(), - isContextInitialized: jest.fn(), + getContextInitializationPromise: jest.fn(), }; }); }; diff --git a/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts b/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts index ca64faa7c51ea..1360e5f7924a5 100644 --- a/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts +++ b/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts @@ -11,6 +11,7 @@ import { errors as EsErrors } from '@elastic/elasticsearch'; import { ReplaySubject, Subject } from 'rxjs'; import { AlertsService } from './alerts_service'; import { IRuleTypeAlerts } from '../types'; +import { retryUntil } from './test_utils'; let logger: ReturnType; const clusterClient = elasticsearchServiceMock.createClusterClient().asInternalUser; @@ -81,19 +82,21 @@ interface GetIndexTemplatePutBodyOpts { context?: string; useLegacyAlerts?: boolean; useEcs?: boolean; + secondaryAlias?: string; } const getIndexTemplatePutBody = (opts?: GetIndexTemplatePutBodyOpts) => { const context = opts ? opts.context : undefined; const useLegacyAlerts = opts ? opts.useLegacyAlerts : undefined; const useEcs = opts ? opts.useEcs : undefined; + const secondaryAlias = opts ? opts.secondaryAlias : undefined; return { - name: `.alerts-${context ? context : 'test'}-default-template`, + name: `.alerts-${context ? context : 'test'}.alerts-default-index-template`, body: { - index_patterns: [`.alerts-${context ? context : 'test'}-default-*`], + index_patterns: [`.internal.alerts-${context ? context : 'test'}.alerts-default-*`], composed_of: [ - `.alerts-${context ? context : 'test'}-mappings`, - ...(useLegacyAlerts ? ['.alerts-legacy-alert-mappings'] : []), ...(useEcs ? ['.alerts-ecs-mappings'] : []), + `.alerts-${context ? `${context}.alerts` : 'test.alerts'}-mappings`, + ...(useLegacyAlerts ? ['.alerts-legacy-alert-mappings'] : []), '.alerts-framework-mappings', ], template: { @@ -102,16 +105,32 @@ const getIndexTemplatePutBody = (opts?: GetIndexTemplatePutBodyOpts) => { hidden: true, 'index.lifecycle': { name: '.alerts-ilm-policy', - rollover_alias: `.alerts-${context ? context : 'test'}-default`, + rollover_alias: `.alerts-${context ? context : 'test'}.alerts-default`, }, 'index.mapping.total_fields.limit': 2500, }, mappings: { dynamic: false, + _meta: { + kibana: { version: '8.8.0' }, + managed: true, + namespace: 'default', + }, }, + ...(secondaryAlias + ? { + aliases: { + [`${secondaryAlias}-default`]: { + is_write_index: false, + }, + }, + } + : {}), }, _meta: { + kibana: { version: '8.8.0' }, managed: true, + namespace: 'default', }, }, }; @@ -119,12 +138,15 @@ const getIndexTemplatePutBody = (opts?: GetIndexTemplatePutBodyOpts) => { const TestRegistrationContext: IRuleTypeAlerts = { context: 'test', - fieldMap: { field: { type: 'keyword', required: false } }, + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, }; -const AnotherRegistrationContext: IRuleTypeAlerts = { - context: 'another', - fieldMap: { field: { type: 'keyword', required: false } }, +const getContextInitialized = async ( + alertsService: AlertsService, + context: string = TestRegistrationContext.context +) => { + const { result } = await alertsService.getContextInitializationPromise(context); + return result; }; describe('Alerts Service', () => { @@ -146,16 +168,19 @@ describe('Alerts Service', () => { pluginStop$.next(); pluginStop$.complete(); }); - describe('initialize()', () => { + describe('AlertsService()', () => { test('should correctly initialize common resources', async () => { const alertsService = new AlertsService({ logger, elasticsearchClientPromise: Promise.resolve(clusterClient), pluginStop$, + kibanaVersion: '8.8.0', }); - alertsService.initialize(); - await new Promise((r) => setTimeout(r, 50)); + await retryUntil( + 'alert service initialized', + async () => alertsService.isInitialized() === true + ); expect(alertsService.isInitialized()).toEqual(true); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); @@ -175,10 +200,10 @@ describe('Alerts Service', () => { logger, elasticsearchClientPromise: Promise.resolve(clusterClient), pluginStop$, + kibanaVersion: '8.8.0', }); - alertsService.initialize(); - await new Promise((r) => setTimeout(r, 50)); + await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); expect(alertsService.isInitialized()).toEqual(false); @@ -196,10 +221,10 @@ describe('Alerts Service', () => { logger, elasticsearchClientPromise: Promise.resolve(clusterClient), pluginStop$, + kibanaVersion: '8.8.0', }); - alertsService.initialize(); - await new Promise((r) => setTimeout(r, 50)); + await retryUntil('error log called', async () => logger.error.mock.calls.length > 0); expect(alertsService.isInitialized()).toEqual(false); expect(logger.error).toHaveBeenCalledWith( @@ -274,12 +299,14 @@ describe('Alerts Service', () => { logger, elasticsearchClientPromise: Promise.resolve(clusterClient), pluginStop$, + kibanaVersion: '8.8.0', }); - alertsService.initialize(); - await new Promise((r) => setTimeout(r, 50)); + await retryUntil( + 'alert service initialized', + async () => alertsService.isInitialized() === true + ); - expect(alertsService.isInitialized()).toEqual(true); expect(clusterClient.indices.getIndexTemplate).toHaveBeenCalledTimes(1); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(1); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({ @@ -301,32 +328,34 @@ describe('Alerts Service', () => { // after updating index template field limit expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); }); + }); - test('should install resources for contexts awaiting initialization when common resources are initialized', async () => { - const alertsService = new AlertsService({ + describe('register()', () => { + let alertsService: AlertsService; + beforeEach(async () => { + alertsService = new AlertsService({ logger, elasticsearchClientPromise: Promise.resolve(clusterClient), pluginStop$, + kibanaVersion: '8.8.0', }); - // pre-register contexts so they get installed right after initialization - alertsService.register(TestRegistrationContext); - alertsService.register(AnotherRegistrationContext); - alertsService.initialize(); - await new Promise((r) => setTimeout(r, 50)); - - expect(alertsService.isInitialized()).toEqual(true); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - true + await retryUntil( + 'alert service initialized', + async () => alertsService.isInitialized() === true ); - expect(await alertsService.isContextInitialized(AnotherRegistrationContext.context)).toEqual( - true + }); + + test('should correctly install resources for context when common initialization is complete', async () => { + alertsService.register(TestRegistrationContext); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(alertsService)) === true ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); - // 1x for framework component template, 1x for legacy alert, 1x for ecs, 2x for context specific - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(5); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; expect(componentTemplate1.name).toEqual('.alerts-framework-mappings'); const componentTemplate2 = clusterClient.cluster.putComponentTemplate.mock.calls[1][0]; @@ -334,73 +363,35 @@ describe('Alerts Service', () => { const componentTemplate3 = clusterClient.cluster.putComponentTemplate.mock.calls[2][0]; expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); const componentTemplate4 = clusterClient.cluster.putComponentTemplate.mock.calls[3][0]; - expect(componentTemplate4.name).toEqual('.alerts-another-mappings'); - const componentTemplate5 = clusterClient.cluster.putComponentTemplate.mock.calls[4][0]; - expect(componentTemplate5.name).toEqual('.alerts-test-mappings'); - - expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(2); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenNthCalledWith( - 1, - getIndexTemplatePutBody({ context: 'another' }) - ); - expect(clusterClient.indices.putIndexTemplate).toHaveBeenNthCalledWith( - 2, + expect(componentTemplate4.name).toEqual('.alerts-test.alerts-mappings'); + + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith( getIndexTemplatePutBody() ); - - expect(clusterClient.indices.getAlias).toHaveBeenCalledTimes(2); - expect(clusterClient.indices.getAlias).toHaveBeenNthCalledWith(1, { - index: '.alerts-another-default-*', - }); - expect(clusterClient.indices.getAlias).toHaveBeenNthCalledWith(2, { - index: '.alerts-test-default-*', - }); - expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(4); - expect(clusterClient.indices.create).toHaveBeenCalledTimes(2); - expect(clusterClient.indices.create).toHaveBeenNthCalledWith(1, { - index: '.alerts-another-default-000001', - body: { - aliases: { - '.alerts-another-default': { - is_write_index: true, - }, - }, - }, + expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({ + index: '.internal.alerts-test.alerts-default-*', + name: '.alerts-test.alerts-*', }); - expect(clusterClient.indices.create).toHaveBeenNthCalledWith(2, { - index: '.alerts-test-default-000001', + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.create).toHaveBeenCalledWith({ + index: '.internal.alerts-test.alerts-default-000001', body: { aliases: { - '.alerts-test-default': { + '.alerts-test.alerts-default': { is_write_index: true, }, }, }, }); }); - }); - - describe('register()', () => { - let alertsService: AlertsService; - beforeEach(async () => { - alertsService = new AlertsService({ - logger, - elasticsearchClientPromise: Promise.resolve(clusterClient), - pluginStop$, - }); - alertsService.initialize(); - await new Promise((r) => setTimeout(r, 50)); - expect(alertsService.isInitialized()).toEqual(true); - }); - - test('should correctly install resources for context when common initialization is complete', async () => { - alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - true + test('should correctly install resources for context when useLegacyAlerts is true', async () => { + alertsService.register({ ...TestRegistrationContext, useLegacyAlerts: true }); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(alertsService)) === true ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); @@ -413,22 +404,23 @@ describe('Alerts Service', () => { const componentTemplate3 = clusterClient.cluster.putComponentTemplate.mock.calls[2][0]; expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); const componentTemplate4 = clusterClient.cluster.putComponentTemplate.mock.calls[3][0]; - expect(componentTemplate4.name).toEqual('.alerts-test-mappings'); + expect(componentTemplate4.name).toEqual('.alerts-test.alerts-mappings'); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith( - getIndexTemplatePutBody() + getIndexTemplatePutBody({ useLegacyAlerts: true }) ); expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({ - index: '.alerts-test-default-*', + index: '.internal.alerts-test.alerts-default-*', + name: '.alerts-test.alerts-*', }); expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(2); expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(2); expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(2); expect(clusterClient.indices.create).toHaveBeenCalledWith({ - index: '.alerts-test-default-000001', + index: '.internal.alerts-test.alerts-default-000001', body: { aliases: { - '.alerts-test-default': { + '.alerts-test.alerts-default': { is_write_index: true, }, }, @@ -436,11 +428,11 @@ describe('Alerts Service', () => { }); }); - test('should correctly install resources for context when useLegacyAlerts is true', async () => { - alertsService.register({ ...TestRegistrationContext, useLegacyAlerts: true }); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - true + test('should correctly install resources for context when useEcs is true', async () => { + alertsService.register({ ...TestRegistrationContext, useEcs: true }); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(alertsService)) === true ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); @@ -453,22 +445,23 @@ describe('Alerts Service', () => { const componentTemplate3 = clusterClient.cluster.putComponentTemplate.mock.calls[2][0]; expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); const componentTemplate4 = clusterClient.cluster.putComponentTemplate.mock.calls[3][0]; - expect(componentTemplate4.name).toEqual('.alerts-test-mappings'); + expect(componentTemplate4.name).toEqual('.alerts-test.alerts-mappings'); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith( - getIndexTemplatePutBody({ useLegacyAlerts: true }) + getIndexTemplatePutBody({ useEcs: true }) ); expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({ - index: '.alerts-test-default-*', + index: '.internal.alerts-test.alerts-default-*', + name: '.alerts-test.alerts-*', }); expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(2); expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(2); expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(2); expect(clusterClient.indices.create).toHaveBeenCalledWith({ - index: '.alerts-test-default-000001', + index: '.internal.alerts-test.alerts-default-000001', body: { aliases: { - '.alerts-test-default': { + '.alerts-test.alerts-default': { is_write_index: true, }, }, @@ -476,11 +469,11 @@ describe('Alerts Service', () => { }); }); - test('should correctly install resources for context when useEcs is true', async () => { - alertsService.register({ ...TestRegistrationContext, useEcs: true }); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - true + test('should correctly install resources for context when secondaryAlias is defined', async () => { + alertsService.register({ ...TestRegistrationContext, secondaryAlias: 'another.alias' }); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(alertsService)) === true ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); @@ -493,22 +486,23 @@ describe('Alerts Service', () => { const componentTemplate3 = clusterClient.cluster.putComponentTemplate.mock.calls[2][0]; expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); const componentTemplate4 = clusterClient.cluster.putComponentTemplate.mock.calls[3][0]; - expect(componentTemplate4.name).toEqual('.alerts-test-mappings'); + expect(componentTemplate4.name).toEqual('.alerts-test.alerts-mappings'); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith( - getIndexTemplatePutBody({ useEcs: true }) + getIndexTemplatePutBody({ secondaryAlias: 'another.alias' }) ); expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({ - index: '.alerts-test-default-*', + index: '.internal.alerts-test.alerts-default-*', + name: '.alerts-test.alerts-*', }); expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(2); expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(2); expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(2); expect(clusterClient.indices.create).toHaveBeenCalledWith({ - index: '.alerts-test-default-000001', + index: '.internal.alerts-test.alerts-default-000001', body: { aliases: { - '.alerts-test-default': { + '.alerts-test.alerts-default': { is_write_index: true, }, }, @@ -519,10 +513,12 @@ describe('Alerts Service', () => { test('should not install component template for context if fieldMap is empty', async () => { alertsService.register({ context: 'empty', - fieldMap: {}, + mappings: { fieldMap: {} }, }); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized('empty')).toEqual(true); + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(alertsService, 'empty')) === true + ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); @@ -535,9 +531,9 @@ describe('Alerts Service', () => { expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({ - name: `.alerts-empty-default-template`, + name: `.alerts-empty.alerts-default-index-template`, body: { - index_patterns: [`.alerts-empty-default-*`], + index_patterns: [`.internal.alerts-empty.alerts-default-*`], composed_of: ['.alerts-framework-mappings'], template: { settings: { @@ -545,30 +541,38 @@ describe('Alerts Service', () => { hidden: true, 'index.lifecycle': { name: '.alerts-ilm-policy', - rollover_alias: `.alerts-empty-default`, + rollover_alias: `.alerts-empty.alerts-default`, }, 'index.mapping.total_fields.limit': 2500, }, mappings: { + _meta: { + kibana: { version: '8.8.0' }, + managed: true, + namespace: 'default', + }, dynamic: false, }, }, _meta: { + kibana: { version: '8.8.0' }, managed: true, + namespace: 'default', }, }, }); expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({ - index: '.alerts-empty-default-*', + index: '.internal.alerts-empty.alerts-default-*', + name: '.alerts-empty.alerts-*', }); expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(2); expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(2); expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(2); expect(clusterClient.indices.create).toHaveBeenCalledWith({ - index: '.alerts-empty-default-000001', + index: '.internal.alerts-empty.alerts-default-000001', body: { aliases: { - '.alerts-empty-default': { + '.alerts-empty.alerts-default': { is_write_index: true, }, }, @@ -587,14 +591,25 @@ describe('Alerts Service', () => { test('should throw error if context already exists and has been registered with a different field map', async () => { alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 50)); expect(() => { alertsService.register({ ...TestRegistrationContext, - fieldMap: { anotherField: { type: 'keyword', required: false } }, + mappings: { fieldMap: { anotherField: { type: 'keyword', required: false } } }, + }); + }).toThrowErrorMatchingInlineSnapshot( + `"test has already been registered with different options"` + ); + }); + + test('should throw error if context already exists and has been registered with a different options', async () => { + alertsService.register(TestRegistrationContext); + expect(() => { + alertsService.register({ + ...TestRegistrationContext, + useEcs: true, }); }).toThrowErrorMatchingInlineSnapshot( - `"test has already been registered with a different mapping"` + `"test has already been registered with different options"` ); }); @@ -602,13 +617,13 @@ describe('Alerts Service', () => { clusterClient.indices.simulateTemplate.mockRejectedValueOnce(new Error('fail')); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - true + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(alertsService)) === true ); expect(logger.error).toHaveBeenCalledWith( - `Failed to simulate index template mappings for .alerts-test-default-template; not applying mappings - fail` + `Failed to simulate index template mappings for .alerts-test.alerts-default-index-template; not applying mappings - fail` ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); @@ -633,14 +648,18 @@ describe('Alerts Service', () => { })); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - false - ); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); + expect( + await alertsService.getContextInitializationPromise(TestRegistrationContext.context) + ).toEqual({ + result: false, + error: + 'Failure during installation. No mappings would be generated for .alerts-test.alerts-default-index-template, possibly due to failed/misconfigured bootstrapping', + }); expect(logger.error).toHaveBeenCalledWith( new Error( - `No mappings would be generated for .alerts-test-default-template, possibly due to failed/misconfigured bootstrapping` + `No mappings would be generated for .alerts-test.alerts-default-index-template, possibly due to failed/misconfigured bootstrapping` ) ); @@ -659,13 +678,14 @@ describe('Alerts Service', () => { clusterClient.indices.putIndexTemplate.mockRejectedValueOnce(new Error('fail')); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - false - ); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); + + expect( + await alertsService.getContextInitializationPromise(TestRegistrationContext.context) + ).toEqual({ error: 'Failure during installation. fail', result: false }); expect(logger.error).toHaveBeenCalledWith( - `Error installing index template .alerts-test-default-template - fail` + `Error installing index template .alerts-test.alerts-default-index-template - fail` ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); @@ -683,13 +703,13 @@ describe('Alerts Service', () => { clusterClient.indices.getAlias.mockRejectedValueOnce(new Error('fail')); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - false - ); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); + expect( + await alertsService.getContextInitializationPromise(TestRegistrationContext.context) + ).toEqual({ error: 'Failure during installation. fail', result: false }); expect(logger.error).toHaveBeenCalledWith( - `Error fetching concrete indices for .alerts-test-default-* pattern - fail` + `Error fetching concrete indices for .internal.alerts-test.alerts-default-* pattern - fail` ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); @@ -708,9 +728,9 @@ describe('Alerts Service', () => { clusterClient.indices.getAlias.mockRejectedValueOnce(error); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - true + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(alertsService)) === true ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); @@ -727,10 +747,11 @@ describe('Alerts Service', () => { clusterClient.indices.putSettings.mockRejectedValueOnce(new Error('fail')); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - false - ); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); + + expect( + await alertsService.getContextInitializationPromise(TestRegistrationContext.context) + ).toEqual({ error: 'Failure during installation. fail', result: false }); expect(logger.error).toHaveBeenCalledWith( `Failed to PUT index.mapping.total_fields.limit settings for alias alias_1: fail` @@ -751,9 +772,9 @@ describe('Alerts Service', () => { clusterClient.indices.simulateIndexTemplate.mockRejectedValueOnce(new Error('fail')); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - true + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(alertsService)) === true ); expect(logger.error).toHaveBeenCalledWith( @@ -775,10 +796,10 @@ describe('Alerts Service', () => { clusterClient.indices.putMapping.mockRejectedValueOnce(new Error('fail')); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - false - ); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); + expect( + await alertsService.getContextInitializationPromise(TestRegistrationContext.context) + ).toEqual({ error: 'Failure during installation. fail', result: false }); expect(logger.error).toHaveBeenCalledWith(`Failed to PUT mapping for alias alias_1: fail`); @@ -797,9 +818,9 @@ describe('Alerts Service', () => { clusterClient.indices.getAlias.mockImplementationOnce(async () => ({})); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - true + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(alertsService)) === true ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); @@ -815,9 +836,9 @@ describe('Alerts Service', () => { test('should log error and set initialized to false if concrete indices exist but none are write index', async () => { clusterClient.indices.getAlias.mockImplementationOnce(async () => ({ - '.alerts-test-default-0001': { + '.internal.alerts-test.alerts-default-0001': { aliases: { - '.alerts-test-default': { + '.alerts-test.alerts-default': { is_write_index: false, is_hidden: true, }, @@ -830,14 +851,18 @@ describe('Alerts Service', () => { })); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - false - ); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); + expect( + await alertsService.getContextInitializationPromise(TestRegistrationContext.context) + ).toEqual({ + error: + 'Failure during installation. Indices matching pattern .internal.alerts-test.alerts-default-* exist but none are set as the write index for alias .alerts-test.alerts-default', + result: false, + }); expect(logger.error).toHaveBeenCalledWith( new Error( - `Indices matching pattern .alerts-test-default-* exist but none are set as the write index for alias .alerts-test-default` + `Indices matching pattern .internal.alerts-test.alerts-default-* exist but none are set as the write index for alias .alerts-test.alerts-default` ) ); @@ -854,9 +879,9 @@ describe('Alerts Service', () => { test('does not create new index if concrete write index exists', async () => { clusterClient.indices.getAlias.mockImplementationOnce(async () => ({ - '.alerts-test-default-0001': { + '.internal.alerts-test.alerts-default-0001': { aliases: { - '.alerts-test-default': { + '.alerts-test.alerts-default': { is_write_index: true, is_hidden: true, }, @@ -869,9 +894,9 @@ describe('Alerts Service', () => { })); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - true + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(alertsService)) === true ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); @@ -889,10 +914,11 @@ describe('Alerts Service', () => { clusterClient.indices.create.mockRejectedValueOnce(new Error('fail')); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - false - ); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); + + expect( + await alertsService.getContextInitializationPromise(TestRegistrationContext.context) + ).toEqual({ error: 'Failure during installation. fail', result: false }); expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); @@ -918,15 +944,15 @@ describe('Alerts Service', () => { }; clusterClient.indices.create.mockRejectedValueOnce(error); clusterClient.indices.get.mockImplementationOnce(async () => ({ - '.alerts-test-default-000001': { - aliases: { '.alerts-test-default': { is_write_index: true } }, + '.internal.alerts-test.alerts-default-000001': { + aliases: { '.alerts-test.alerts-default': { is_write_index: true } }, }, })); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - true + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(alertsService)) === true ); expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); @@ -954,16 +980,20 @@ describe('Alerts Service', () => { }; clusterClient.indices.create.mockRejectedValueOnce(error); clusterClient.indices.get.mockImplementationOnce(async () => ({ - '.alerts-test-default-000001': { - aliases: { '.alerts-test-default': { is_write_index: false } }, + '.internal.alerts-test.alerts-default-000001': { + aliases: { '.alerts-test.alerts-default': { is_write_index: false } }, }, })); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 50)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - false - ); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); + expect( + await alertsService.getContextInitializationPromise(TestRegistrationContext.context) + ).toEqual({ + error: + 'Failure during installation. Attempted to create index: .internal.alerts-test.alerts-default-000001 as the write index for alias: .alerts-test.alerts-default, but the index already exists and is not the write index for the alias', + result: false, + }); expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); @@ -990,11 +1020,13 @@ describe('Alerts Service', () => { logger, elasticsearchClientPromise: Promise.resolve(clusterClient), pluginStop$, + kibanaVersion: '8.8.0', }); - alertsService.initialize(); - await new Promise((r) => setTimeout(r, 150)); - expect(alertsService.isInitialized()).toEqual(true); + await retryUntil( + 'alert service initialized', + async () => alertsService.isInitialized() === true + ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledTimes(3); }); @@ -1007,11 +1039,13 @@ describe('Alerts Service', () => { logger, elasticsearchClientPromise: Promise.resolve(clusterClient), pluginStop$, + kibanaVersion: '8.8.0', }); - alertsService.initialize(); - await new Promise((r) => setTimeout(r, 150)); - expect(alertsService.isInitialized()).toEqual(true); + await retryUntil( + 'alert service initialized', + async () => alertsService.isInitialized() === true + ); expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(5); }); @@ -1024,17 +1058,21 @@ describe('Alerts Service', () => { logger, elasticsearchClientPromise: Promise.resolve(clusterClient), pluginStop$, + kibanaVersion: '8.8.0', }); - alertsService.initialize(); - await new Promise((r) => setTimeout(r, 150)); + await retryUntil( + 'alert service initialized', + async () => alertsService.isInitialized() === true + ); expect(alertsService.isInitialized()).toEqual(true); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 150)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - true + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(alertsService)) === true ); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(3); }); @@ -1047,17 +1085,20 @@ describe('Alerts Service', () => { logger, elasticsearchClientPromise: Promise.resolve(clusterClient), pluginStop$, + kibanaVersion: '8.8.0', }); - alertsService.initialize(); - await new Promise((r) => setTimeout(r, 150)); - expect(alertsService.isInitialized()).toEqual(true); + await retryUntil( + 'alert service initialized', + async () => alertsService.isInitialized() === true + ); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 150)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - true + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(alertsService)) === true ); + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(4); }); @@ -1070,17 +1111,20 @@ describe('Alerts Service', () => { logger, elasticsearchClientPromise: Promise.resolve(clusterClient), pluginStop$, + kibanaVersion: '8.8.0', }); - alertsService.initialize(); - await new Promise((r) => setTimeout(r, 150)); - expect(alertsService.isInitialized()).toEqual(true); + await retryUntil( + 'alert service initialized', + async () => alertsService.isInitialized() === true + ); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 150)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - true + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(alertsService)) === true ); + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(4); }); @@ -1093,17 +1137,20 @@ describe('Alerts Service', () => { logger, elasticsearchClientPromise: Promise.resolve(clusterClient), pluginStop$, + kibanaVersion: '8.8.0', }); - alertsService.initialize(); - await new Promise((r) => setTimeout(r, 150)); - expect(alertsService.isInitialized()).toEqual(true); + await retryUntil( + 'alert service initialized', + async () => alertsService.isInitialized() === true + ); alertsService.register(TestRegistrationContext); - await new Promise((r) => setTimeout(r, 150)); - expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( - true + await retryUntil( + 'context initialized', + async () => (await getContextInitialized(alertsService)) === true ); + expect(clusterClient.indices.create).toHaveBeenCalledTimes(3); }); }); @@ -1114,29 +1161,30 @@ describe('Alerts Service', () => { await new Promise((resolve) => setTimeout(resolve, 20)); return { acknowledged: true }; }); - const alertsService = new AlertsService({ + new AlertsService({ logger, elasticsearchClientPromise: Promise.resolve(clusterClient), pluginStop$, + kibanaVersion: '8.8.0', + timeoutMs: 10, }); - alertsService.initialize(10); - await new Promise((r) => setTimeout(r, 150)); - expect(alertsService.isInitialized()).toEqual(false); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); expect(logger.error).toHaveBeenCalledWith(new Error(`Timeout: it took more than 10ms`)); }); test('should short circuit initialization if pluginStop$ signal received but not throw error', async () => { pluginStop$.next(); - const alertsService = new AlertsService({ + new AlertsService({ logger, elasticsearchClientPromise: Promise.resolve(clusterClient), pluginStop$, + kibanaVersion: '8.8.0', + timeoutMs: 10, }); - alertsService.initialize(); - await new Promise((r) => setTimeout(r, 50)); + await retryUntil('error logger called', async () => logger.error.mock.calls.length > 0); expect(logger.error).toHaveBeenCalledWith( new Error(`Server is stopping; must stop all async operations`) diff --git a/x-pack/plugins/alerting/server/alerts_service/alerts_service.ts b/x-pack/plugins/alerting/server/alerts_service/alerts_service.ts index 22cb9f4df1884..5e32a599de744 100644 --- a/x-pack/plugins/alerting/server/alerts_service/alerts_service.ts +++ b/x-pack/plugins/alerting/server/alerts_service/alerts_service.ts @@ -13,13 +13,11 @@ import { import { get, isEmpty, isEqual } from 'lodash'; import { Logger, ElasticsearchClient } from '@kbn/core/server'; import { firstValueFrom, Observable } from 'rxjs'; +import { alertFieldMap, ecsFieldMap, legacyAlertFieldMap } from '@kbn/alerts-as-data-utils'; import { - alertFieldMap, - ecsFieldMap, - legacyAlertFieldMap, - type FieldMap, -} from '@kbn/alerts-as-data-utils'; -import { IndicesGetIndexTemplateIndexTemplateItem } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + IndicesGetIndexTemplateIndexTemplateItem, + Metadata, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { asyncForEach } from '@kbn/std'; import { DEFAULT_ALERTS_ILM_POLICY_NAME, @@ -30,23 +28,28 @@ import { getComponentTemplateName, getIndexTemplateAndPattern, IIndexPatternString, -} from './types'; +} from './resource_installer_utils'; import { retryTransientEsErrors } from './retry_transient_es_errors'; import { IRuleTypeAlerts } from '../types'; import { createResourceInstallationHelper, + errorResult, + InitializationPromise, ResourceInstallationHelper, + successResult, } from './create_resource_installation_helper'; const TOTAL_FIELDS_LIMIT = 2500; const INSTALLATION_TIMEOUT = 20 * 60 * 1000; // 20 minutes const LEGACY_ALERT_CONTEXT = 'legacy-alert'; export const ECS_CONTEXT = `ecs`; -export const ECS_COMPONENT_TEMPLATE_NAME = getComponentTemplateName(ECS_CONTEXT); +export const ECS_COMPONENT_TEMPLATE_NAME = getComponentTemplateName({ name: ECS_CONTEXT }); interface AlertsServiceParams { logger: Logger; pluginStop$: Observable; + kibanaVersion: string; elasticsearchClientPromise: Promise; + timeoutMs?: number; } interface ConcreteIndexInfo { @@ -54,17 +57,8 @@ interface ConcreteIndexInfo { alias: string; isWriteIndex: boolean; } -interface IAlertsService { - /** - * Initializes the common ES resources needed for framework alerts as data - * - ILM policy - common policy shared by all AAD indices - * - Component template - common mappings for fields populated and used by the framework - * - * Once common resource initialization is complete, look for any solution-specific - * resources that have been registered and are awaiting initialization. - */ - initialize(timeoutMs?: number): void; +interface IAlertsService { /** * Register solution specific resources. If common resource initialization is * complete, go ahead and install those resources, otherwise add to queue to @@ -78,16 +72,37 @@ interface IAlertsService { register(opts: IRuleTypeAlerts, timeoutMs?: number): void; isInitialized(): boolean; + + /** + * Returns promise that resolves when the resources for the given + * context are installed. These include the context specific component template, + * the index template for the default namespace and the concrete write index + * for the default namespace. + */ + getContextInitializationPromise(context: string): Promise; } +export type PublicAlertsService = Pick; +export type PublicFrameworkAlertsService = PublicAlertsService & { + enabled: () => boolean; +}; + export class AlertsService implements IAlertsService { private initialized: boolean; private resourceInitializationHelper: ResourceInstallationHelper; - private registeredContexts: Map = new Map(); + private registeredContexts: Map = new Map(); + private commonInitPromise: Promise; constructor(private readonly options: AlertsServiceParams) { this.initialized = false; + + // Kick off initialization of common assets and save the promise + this.commonInitPromise = this.initializeCommon(this.options.timeoutMs); + + // Create helper for initializing context-specific resources this.resourceInitializationHelper = createResourceInstallationHelper( + this.options.logger, + this.commonInitPromise, this.initializeContext.bind(this) ); } @@ -96,88 +111,118 @@ export class AlertsService implements IAlertsService { return this.initialized; } - public async isContextInitialized(context: string) { - return (await this.resourceInitializationHelper.getInitializedContexts().get(context)) ?? false; - } - - public initialize(timeoutMs?: number) { - // Only initialize once - if (this.initialized) return; - - this.options.logger.debug(`Initializing resources for AlertsService`); - - // Use setImmediate to execute async fns as soon as possible - setImmediate(async () => { - try { - const esClient = await this.options.elasticsearchClientPromise; - - // Common initialization installs ILM policy and shared component template - const initFns = [ - () => this.createOrUpdateIlmPolicy(esClient), - () => this.createOrUpdateComponentTemplate(esClient, getComponentTemplate(alertFieldMap)), - () => - this.createOrUpdateComponentTemplate( - esClient, - getComponentTemplate(legacyAlertFieldMap, LEGACY_ALERT_CONTEXT) - ), - () => - this.createOrUpdateComponentTemplate( - esClient, - getComponentTemplate(ecsFieldMap, ECS_CONTEXT) - ), - ]; - - for (const fn of initFns) { - await this.installWithTimeout(async () => await fn(), timeoutMs); - } - - this.initialized = true; - } catch (err) { - this.options.logger.error( - `Error installing common resources for AlertsService. No additional resources will be installed and rule execution may be impacted.` - ); - this.initialized = false; - } - - if (this.initialized) { - this.resourceInitializationHelper.setReadyToInitialize(timeoutMs); - } - }); + public async getContextInitializationPromise( + context: string, + timeoutMs?: number + ): Promise { + if (!this.registeredContexts.has(context)) { + const errMsg = `Error getting initialized status for context ${context} - context has not been registered.`; + this.options.logger.error(errMsg); + return Promise.resolve(errorResult(errMsg)); + } + return this.resourceInitializationHelper.getInitializedContext(context, timeoutMs); } public register(opts: IRuleTypeAlerts, timeoutMs?: number) { - const { context, fieldMap } = opts; + const { context } = opts; // check whether this context has been registered before if (this.registeredContexts.has(context)) { - const registeredFieldMap = this.registeredContexts.get(context); - if (!isEqual(fieldMap, registeredFieldMap)) { - throw new Error(`${context} has already been registered with a different mapping`); + const registeredOptions = this.registeredContexts.get(context); + if (!isEqual(opts, registeredOptions)) { + throw new Error(`${context} has already been registered with different options`); } this.options.logger.debug(`Resources for context "${context}" have already been registered.`); return; } this.options.logger.info(`Registering resources for context "${context}".`); - this.registeredContexts.set(context, fieldMap); + this.registeredContexts.set(context, opts); this.resourceInitializationHelper.add(opts, timeoutMs); } + /** + * Initializes the common ES resources needed for framework alerts as data + * - ILM policy - common policy shared by all AAD indices + * - Component template - common mappings for fields populated and used by the framework + */ + private async initializeCommon(timeoutMs?: number): Promise { + try { + this.options.logger.debug(`Initializing resources for AlertsService`); + const esClient = await this.options.elasticsearchClientPromise; + + // Common initialization installs ILM policy and shared component template + const initFns = [ + () => this.createOrUpdateIlmPolicy(esClient), + () => + this.createOrUpdateComponentTemplate( + esClient, + getComponentTemplate({ fieldMap: alertFieldMap, includeSettings: true }) + ), + () => + this.createOrUpdateComponentTemplate( + esClient, + getComponentTemplate({ + fieldMap: legacyAlertFieldMap, + name: LEGACY_ALERT_CONTEXT, + includeSettings: true, + }) + ), + () => + this.createOrUpdateComponentTemplate( + esClient, + getComponentTemplate({ + fieldMap: ecsFieldMap, + name: ECS_CONTEXT, + includeSettings: true, + }) + ), + ]; + + for (const fn of initFns) { + await this.installWithTimeout(async () => await fn(), timeoutMs); + } + + this.initialized = true; + return successResult(); + } catch (err) { + this.options.logger.error( + `Error installing common resources for AlertsService. No additional resources will be installed and rule execution may be impacted. - ${err.message}` + ); + this.initialized = false; + return errorResult(err.message); + } + } + private async initializeContext( - { context, fieldMap, useEcs, useLegacyAlerts }: IRuleTypeAlerts, + { context, mappings, useEcs, useLegacyAlerts, secondaryAlias }: IRuleTypeAlerts, timeoutMs?: number ) { const esClient = await this.options.elasticsearchClientPromise; - const indexTemplateAndPattern = getIndexTemplateAndPattern(context); + const indexTemplateAndPattern = getIndexTemplateAndPattern({ context, secondaryAlias }); let initFns: Array<() => Promise> = []; // List of component templates to reference + // Order matters in this list - templates specified last take precedence over those specified first + // 1. ECS component template, if using + // 2. Context specific component template, if defined during registration + // 3. Legacy alert component template, if using + // 4. Framework common component template, always included const componentTemplateRefs: string[] = []; - // If fieldMap is not empty, create a context specific component template - if (!isEmpty(fieldMap)) { - const componentTemplate = getComponentTemplate(fieldMap, context); + // If useEcs is set to true, add the ECS component template to the references + if (useEcs) { + componentTemplateRefs.push(getComponentTemplateName({ name: ECS_CONTEXT })); + } + + // If fieldMap is not empty, create a context specific component template and add to the references + if (!isEmpty(mappings.fieldMap)) { + const componentTemplate = getComponentTemplate({ + fieldMap: mappings.fieldMap, + dynamic: mappings.dynamic, + context, + }); initFns.push( async () => await this.createOrUpdateComponentTemplate(esClient, componentTemplate) ); @@ -186,12 +231,7 @@ export class AlertsService implements IAlertsService { // If useLegacy is set to true, add the legacy alert component template to the references if (useLegacyAlerts) { - componentTemplateRefs.push(getComponentTemplateName(LEGACY_ALERT_CONTEXT)); - } - - // If useEcs is set to true, add the ECS component template to the references - if (useEcs) { - componentTemplateRefs.push(getComponentTemplateName(ECS_CONTEXT)); + componentTemplateRefs.push(getComponentTemplateName({ name: LEGACY_ALERT_CONTEXT })); } // Add framework component template to the references @@ -326,6 +366,14 @@ export class AlertsService implements IAlertsService { ) { this.options.logger.info(`Installing index template ${indexPatterns.template}`); + const indexMetadata: Metadata = { + kibana: { + version: this.options.kibanaVersion, + }, + managed: true, + namespace: 'default', // hard-coded to default here until we start supporting space IDs + }; + const indexTemplate = { name: indexPatterns.template, body: { @@ -343,12 +391,21 @@ export class AlertsService implements IAlertsService { }, mappings: { dynamic: false, + _meta: indexMetadata, }, + ...(indexPatterns.secondaryAlias + ? { + aliases: { + [indexPatterns.secondaryAlias]: { + is_write_index: false, + }, + }, + } + : {}), }, - _meta: { - managed: true, - }, - // do we need metadata? like kibana version? doesn't that get updated every version? or just the first version its installed + _meta: indexMetadata, + + // TODO - set priority of this template when we start supporting spaces }, }; @@ -484,8 +541,11 @@ export class AlertsService implements IAlertsService { // check if a concrete write index already exists let concreteIndices: ConcreteIndexInfo[] = []; try { + // Specify both the index pattern for the backing indices and their aliases + // The alias prevents the request from finding other namespaces that could match the -* pattern const response = await esClient.indices.getAlias({ index: indexPatterns.pattern, + name: indexPatterns.basePattern, }); concreteIndices = Object.entries(response).flatMap(([index, { aliases }]) => diff --git a/x-pack/plugins/alerting/server/alerts_service/create_resource_installation_helper.test.ts b/x-pack/plugins/alerting/server/alerts_service/create_resource_installation_helper.test.ts index f9ce460d04093..ba128d46a2887 100644 --- a/x-pack/plugins/alerting/server/alerts_service/create_resource_installation_helper.test.ts +++ b/x-pack/plugins/alerting/server/alerts_service/create_resource_installation_helper.test.ts @@ -7,7 +7,14 @@ import { loggingSystemMock } from '@kbn/core/server/mocks'; import { IRuleTypeAlerts } from '../types'; -import { createResourceInstallationHelper } from './create_resource_installation_helper'; +import { + createResourceInstallationHelper, + errorResult, + InitializationPromise, + ResourceInstallationHelper, + successResult, +} from './create_resource_installation_helper'; +import { retryUntil } from './test_utils'; const logger: ReturnType = loggingSystemMock.createLogger(); @@ -16,13 +23,29 @@ const initFn = async (context: IRuleTypeAlerts, timeoutMs?: number) => { logger.info(context.context); }; -const initFnWithDelay = async (context: IRuleTypeAlerts, timeoutMs?: number) => { - logger.info(context.context); - await new Promise((r) => setTimeout(r, 50)); +const initFnWithError = async (context: IRuleTypeAlerts, timeoutMs?: number) => { + throw new Error('no go'); }; -const initFnWithError = async (context: IRuleTypeAlerts, timeoutMs?: number) => { - throw new Error('fail'); +const getCommonInitPromise = async ( + resolution: boolean, + timeoutMs: number = 1 +): Promise => { + if (timeoutMs < 0) { + throw new Error('fail'); + } + // delay resolution of promise by timeout value + await new Promise((r) => setTimeout(r, timeoutMs)); + logger.info(`commonInitPromise resolved`); + return Promise.resolve(resolution ? successResult() : errorResult(`error initializing`)); +}; + +const getContextInitialized = async ( + helper: ResourceInstallationHelper, + context: string = 'test1' +) => { + const { result } = await helper.getInitializedContext(context); + return result; }; describe('createResourceInstallationHelper', () => { @@ -30,108 +53,117 @@ describe('createResourceInstallationHelper', () => { jest.clearAllMocks(); }); - test(`should not call init function if readyToInitialize is false`, () => { - const helper = createResourceInstallationHelper(initFn); - - // Add two contexts that need to be initialized but don't call helper.setReadyToInitialize() - helper.add({ context: 'test1', fieldMap: { field: { type: 'keyword', required: false } } }); - helper.add({ context: 'test2', fieldMap: { field: { type: 'keyword', required: false } } }); + test(`should wait for commonInitFunction to resolve before calling initFns for registered contexts`, async () => { + const helper = createResourceInstallationHelper( + logger, + getCommonInitPromise(true, 100), + initFn + ); - expect(logger.info).not.toHaveBeenCalled(); - const initializedContexts = helper.getInitializedContexts(); - expect([...initializedContexts.keys()].length).toEqual(0); + // Add two contexts that need to be initialized + helper.add({ + context: 'test1', + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, + }); + helper.add({ + context: 'test2', + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, + }); + + await retryUntil('init fns run', async () => logger.info.mock.calls.length === 3); + + expect(logger.info).toHaveBeenNthCalledWith(1, `commonInitPromise resolved`); + expect(logger.info).toHaveBeenNthCalledWith(2, 'test1'); + expect(logger.info).toHaveBeenNthCalledWith(3, 'test2'); + expect(await helper.getInitializedContext('test1')).toEqual({ result: true }); + expect(await helper.getInitializedContext('test2')).toEqual({ result: true }); }); - test(`should call init function if readyToInitialize is set to true`, async () => { - const helper = createResourceInstallationHelper(initFn); - - // Add two contexts that need to be initialized and then call helper.setReadyToInitialize() - helper.add({ context: 'test1', fieldMap: { field: { type: 'keyword', required: false } } }); - helper.add({ context: 'test2', fieldMap: { field: { type: 'keyword', required: false } } }); - - helper.setReadyToInitialize(); - - // for the setImmediate - await new Promise((r) => setTimeout(r, 10)); - - expect(logger.info).toHaveBeenCalledTimes(2); - const initializedContexts = helper.getInitializedContexts(); - expect([...initializedContexts.keys()].length).toEqual(2); - - expect(await initializedContexts.get('test1')).toEqual(true); - expect(await initializedContexts.get('test2')).toEqual(true); + test(`should return false if context is unrecognized`, async () => { + const helper = createResourceInstallationHelper( + logger, + getCommonInitPromise(true, 100), + initFn + ); + + helper.add({ + context: 'test1', + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, + }); + + await retryUntil('init fns run', async () => logger.info.mock.calls.length === 2); + + expect(await helper.getInitializedContext('test1')).toEqual({ result: true }); + expect(await helper.getInitializedContext('test2')).toEqual({ + result: false, + error: `Unrecognized context test2`, + }); }); - test(`should install resources for contexts added after readyToInitialize is called`, async () => { - const helper = createResourceInstallationHelper(initFnWithDelay); - - // Add two contexts that need to be initialized - helper.add({ context: 'test1', fieldMap: { field: { type: 'keyword', required: false } } }); - helper.add({ context: 'test2', fieldMap: { field: { type: 'keyword', required: false } } }); - - // Start processing the queued contexts - helper.setReadyToInitialize(); - - // for the setImmediate - await new Promise((r) => setTimeout(r, 10)); - - // Add another context to process - helper.add({ context: 'test3', fieldMap: { field: { type: 'keyword', required: false } } }); - - // 3 contexts with delay will take 150 - await new Promise((r) => setTimeout(r, 10)); - - expect(logger.info).toHaveBeenCalledTimes(3); - const initializedContexts = helper.getInitializedContexts(); - expect([...initializedContexts.keys()].length).toEqual(3); - - expect(await initializedContexts.get('test1')).toEqual(true); - expect(await initializedContexts.get('test2')).toEqual(true); - expect(await initializedContexts.get('test3')).toEqual(true); + test(`should log and return false if common init function returns false`, async () => { + const helper = createResourceInstallationHelper( + logger, + getCommonInitPromise(false, 100), + initFn + ); + + helper.add({ + context: 'test1', + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, + }); + + await retryUntil('common init fns run', async () => logger.info.mock.calls.length === 1); + + expect(logger.warn).toHaveBeenCalledWith( + `Common resources were not initialized, cannot initialize context for test1` + ); + expect(await helper.getInitializedContext('test1')).toEqual({ + result: false, + error: `error initializing`, + }); }); - test(`should install resources for contexts added after initial processing loop has run`, async () => { - const helper = createResourceInstallationHelper(initFn); - - // No contexts queued so this should finish quickly - helper.setReadyToInitialize(); - - // for the setImmediate - await new Promise((r) => setTimeout(r, 10)); + test(`should log and return false if common init function throws error`, async () => { + const helper = createResourceInstallationHelper(logger, getCommonInitPromise(true, -1), initFn); - expect(logger.info).not.toHaveBeenCalled(); - let initializedContexts = helper.getInitializedContexts(); - expect([...initializedContexts.keys()].length).toEqual(0); + helper.add({ + context: 'test1', + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, + }); - // Add a context to process - helper.add({ context: 'test1', fieldMap: { field: { type: 'keyword', required: false } } }); + await retryUntil( + 'common init fns run', + async () => (await getContextInitialized(helper)) === false + ); - // for the setImmediate - await new Promise((r) => setTimeout(r, 10)); - - expect(logger.info).toHaveBeenCalledTimes(1); - initializedContexts = helper.getInitializedContexts(); - expect([...initializedContexts.keys()].length).toEqual(1); - - expect(await initializedContexts.get('test1')).toEqual(true); + expect(logger.error).toHaveBeenCalledWith(`Error initializing context test1 - fail`); + expect(await helper.getInitializedContext('test1')).toEqual({ + result: false, + error: `fail`, + }); }); - test(`should gracefully handle errors during initialization and set initialized flag to false`, async () => { - const helper = createResourceInstallationHelper(initFnWithError); - - helper.setReadyToInitialize(); - - // for the setImmediate - await new Promise((r) => setTimeout(r, 10)); - - // Add a context to process - helper.add({ context: 'test1', fieldMap: { field: { type: 'keyword', required: false } } }); - - // for the setImmediate - await new Promise((r) => setTimeout(r, 10)); - - const initializedContexts = helper.getInitializedContexts(); - expect([...initializedContexts.keys()].length).toEqual(1); - expect(await initializedContexts.get('test1')).toEqual(false); + test(`should log and return false if context init function throws error`, async () => { + const helper = createResourceInstallationHelper( + logger, + getCommonInitPromise(true, 100), + initFnWithError + ); + + helper.add({ + context: 'test1', + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, + }); + + await retryUntil( + 'context init fns run', + async () => (await getContextInitialized(helper)) === false + ); + + expect(logger.error).toHaveBeenCalledWith(`Error initializing context test1 - no go`); + expect(await helper.getInitializedContext('test1')).toEqual({ + result: false, + error: `no go`, + }); }); }); diff --git a/x-pack/plugins/alerting/server/alerts_service/create_resource_installation_helper.ts b/x-pack/plugins/alerting/server/alerts_service/create_resource_installation_helper.ts index 0e3cbe0f87a9a..36dddcbf214d6 100644 --- a/x-pack/plugins/alerting/server/alerts_service/create_resource_installation_helper.ts +++ b/x-pack/plugins/alerting/server/alerts_service/create_resource_installation_helper.ts @@ -5,12 +5,16 @@ * 2.0. */ +import { Logger } from '@kbn/core/server'; import { IRuleTypeAlerts } from '../types'; +export interface InitializationPromise { + result: boolean; + error?: string; +} export interface ResourceInstallationHelper { add: (context: IRuleTypeAlerts, timeoutMs?: number) => void; - setReadyToInitialize: (timeoutMs?: number) => void; - getInitializedContexts: () => Map>; + getInitializedContext: (context: string, delayMs?: number) => Promise; } /** @@ -25,57 +29,49 @@ export interface ResourceInstallationHelper { * running, kick off the processing loop */ export function createResourceInstallationHelper( - initFn: (context: IRuleTypeAlerts, timeoutMs?: number) => Promise + logger: Logger, + commonResourcesInitPromise: Promise, + installFn: (context: IRuleTypeAlerts, timeoutMs?: number) => Promise ): ResourceInstallationHelper { - let readyToInitialize = false; - let isInitializing: boolean = false; - const contextsToInitialize: IRuleTypeAlerts[] = []; - const initializedContexts: Map> = new Map(); + const initializedContexts: Map> = new Map(); const waitUntilContextResourcesInstalled = async ( context: IRuleTypeAlerts, timeoutMs?: number - ): Promise => { + ): Promise => { try { - await initFn(context, timeoutMs); - return true; + const { result: commonInitResult, error: commonInitError } = await commonResourcesInitPromise; + if (commonInitResult) { + await installFn(context, timeoutMs); + return successResult(); + } else { + logger.warn( + `Common resources were not initialized, cannot initialize context for ${context.context}` + ); + return errorResult(commonInitError); + } } catch (err) { - return false; + logger.error(`Error initializing context ${context.context} - ${err.message}`); + return errorResult(err.message); } }; - const startInitialization = (timeoutMs?: number) => { - if (!readyToInitialize) { - return; - } - - setImmediate(async () => { - isInitializing = true; - while (contextsToInitialize.length > 0) { - const context = contextsToInitialize.pop()!; - initializedContexts.set( - context.context, - - // Return a promise than can be checked when needed - waitUntilContextResourcesInstalled(context, timeoutMs) - ); - } - isInitializing = false; - }); - }; return { add: (context: IRuleTypeAlerts, timeoutMs?: number) => { - contextsToInitialize.push(context); - if (!isInitializing) { - startInitialization(timeoutMs); - } - }, - setReadyToInitialize: (timeoutMs?: number) => { - readyToInitialize = true; - startInitialization(timeoutMs); + initializedContexts.set( + context.context, + + // Return a promise than can be checked when needed + waitUntilContextResourcesInstalled(context, timeoutMs) + ); }, - getInitializedContexts: () => { - return initializedContexts; + getInitializedContext: async (context: string): Promise => { + return initializedContexts.has(context) + ? initializedContexts.get(context)! + : errorResult(`Unrecognized context ${context}`); }, }; } + +export const successResult = () => ({ result: true }); +export const errorResult = (error?: string) => ({ result: false, error }); diff --git a/x-pack/plugins/alerting/server/alerts_service/index.ts b/x-pack/plugins/alerting/server/alerts_service/index.ts index 49247f3baa243..5c3dd2a17d086 100644 --- a/x-pack/plugins/alerting/server/alerts_service/index.ts +++ b/x-pack/plugins/alerting/server/alerts_service/index.ts @@ -10,4 +10,10 @@ export { DEFAULT_ALERTS_ILM_POLICY_NAME, } from './default_lifecycle_policy'; export { ECS_COMPONENT_TEMPLATE_NAME, ECS_CONTEXT } from './alerts_service'; -export { getComponentTemplate } from './types'; +export { getComponentTemplate } from './resource_installer_utils'; +export { + type InitializationPromise, + successResult, + errorResult, +} from './create_resource_installation_helper'; +export { AlertsService, type PublicFrameworkAlertsService } from './alerts_service'; diff --git a/x-pack/plugins/alerting/server/alerts_service/resource_installer_utils.test.ts b/x-pack/plugins/alerting/server/alerts_service/resource_installer_utils.test.ts new file mode 100644 index 0000000000000..0f5f482cf3c84 --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_service/resource_installer_utils.test.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getComponentTemplateName, getIndexTemplateAndPattern } from './resource_installer_utils'; + +describe('getComponentTemplateName', () => { + test('should use default when name and context are undefined', () => { + expect(getComponentTemplateName()).toEqual(`.alerts-framework-mappings`); + }); + + test('should use name as is when name is defined', () => { + expect(getComponentTemplateName({ name: 'test-my-mappings' })).toEqual( + `.alerts-test-my-mappings-mappings` + ); + }); + + test('should use append .alerts to context when context is defined', () => { + expect(getComponentTemplateName({ context: 'test-my-mappings' })).toEqual( + `.alerts-test-my-mappings.alerts-mappings` + ); + }); + + test('should prioritize context when context and name are both defined', () => { + expect( + getComponentTemplateName({ context: 'test-my-mappings', name: 'dont-name-me-this' }) + ).toEqual(`.alerts-test-my-mappings.alerts-mappings`); + }); +}); + +describe('getIndexTemplateAndPattern', () => { + test('should use default namespace when namespace is undefined', () => { + expect(getIndexTemplateAndPattern({ context: 'test' })).toEqual({ + template: '.alerts-test.alerts-default-index-template', + pattern: '.internal.alerts-test.alerts-default-*', + basePattern: '.alerts-test.alerts-*', + alias: '.alerts-test.alerts-default', + name: '.internal.alerts-test.alerts-default-000001', + }); + }); + + test('should use namespace when namespace is defined', () => { + expect(getIndexTemplateAndPattern({ context: 'test', namespace: 'special' })).toEqual({ + template: '.alerts-test.alerts-special-index-template', + pattern: '.internal.alerts-test.alerts-special-*', + basePattern: '.alerts-test.alerts-*', + alias: '.alerts-test.alerts-special', + name: '.internal.alerts-test.alerts-special-000001', + }); + }); + + test('should return secondaryAlias when secondaryAlias is defined', () => { + expect( + getIndexTemplateAndPattern({ + context: 'test', + namespace: 'special', + secondaryAlias: 'siem.signals', + }) + ).toEqual({ + template: '.alerts-test.alerts-special-index-template', + pattern: '.internal.alerts-test.alerts-special-*', + basePattern: '.alerts-test.alerts-*', + alias: '.alerts-test.alerts-special', + name: '.internal.alerts-test.alerts-special-000001', + secondaryAlias: `siem.signals-special`, + }); + }); +}); diff --git a/x-pack/plugins/alerting/server/alerts_service/resource_installer_utils.ts b/x-pack/plugins/alerting/server/alerts_service/resource_installer_utils.ts new file mode 100644 index 0000000000000..eba614e655617 --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_service/resource_installer_utils.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import type { FieldMap } from '@kbn/alerts-as-data-utils'; +import { getComponentTemplateFromFieldMap } from '../../common'; + +interface GetComponentTemplateNameOpts { + context?: string; + name?: string; +} +export const getComponentTemplateName = ({ context, name }: GetComponentTemplateNameOpts = {}) => + `.alerts-${context ? `${context}.alerts` : name ? name : 'framework'}-mappings`; + +export interface IIndexPatternString { + template: string; + pattern: string; + alias: string; + name: string; + basePattern: string; + secondaryAlias?: string; +} + +interface GetIndexTemplateAndPatternOpts { + context: string; + secondaryAlias?: string; + namespace?: string; +} + +export const getIndexTemplateAndPattern = ({ + context, + namespace, + secondaryAlias, +}: GetIndexTemplateAndPatternOpts): IIndexPatternString => { + const concreteNamespace = namespace ? namespace : 'default'; + const pattern = `${context}.alerts`; + const patternWithNamespace = `${pattern}-${concreteNamespace}`; + return { + template: `.alerts-${patternWithNamespace}-index-template`, + pattern: `.internal.alerts-${patternWithNamespace}-*`, + basePattern: `.alerts-${pattern}-*`, + name: `.internal.alerts-${patternWithNamespace}-000001`, + alias: `.alerts-${patternWithNamespace}`, + ...(secondaryAlias ? { secondaryAlias: `${secondaryAlias}-${concreteNamespace}` } : {}), + }; +}; + +type GetComponentTemplateOpts = GetComponentTemplateNameOpts & { + fieldMap: FieldMap; + dynamic?: 'strict' | false; + includeSettings?: boolean; +}; + +export const getComponentTemplate = ({ + fieldMap, + context, + name, + dynamic, + includeSettings, +}: GetComponentTemplateOpts): ClusterPutComponentTemplateRequest => + getComponentTemplateFromFieldMap({ + name: getComponentTemplateName({ context, name }), + fieldMap, + dynamic, + includeSettings, + }); diff --git a/x-pack/plugins/alerting/server/alerts_service/test_utils.ts b/x-pack/plugins/alerting/server/alerts_service/test_utils.ts new file mode 100644 index 0000000000000..1d710caf57670 --- /dev/null +++ b/x-pack/plugins/alerting/server/alerts_service/test_utils.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +const RETRY_UNTIL_DEFAULT_COUNT = 20; +const RETRY_UNTIL_DEFAULT_WAIT = 100; // milliseconds +type RetryableFunction = () => Promise; + +export const retryUntil = async ( + label: string, + fn: RetryableFunction, + count: number = RETRY_UNTIL_DEFAULT_COUNT, + wait: number = RETRY_UNTIL_DEFAULT_WAIT +): Promise => { + await delay(wait); + while (count > 0) { + count--; + + if (await fn()) return true; + + // eslint-disable-next-line no-console + console.log(`attempt failed waiting for "${label}", attempts left: ${count}`); + + if (count === 0) return false; + await delay(wait); + } + + return false; +}; + +const delay = async (millis: number) => await new Promise((resolve) => setTimeout(resolve, millis)); diff --git a/x-pack/plugins/alerting/server/alerts_service/types.ts b/x-pack/plugins/alerting/server/alerts_service/types.ts deleted file mode 100644 index aeb73cab6ffd2..0000000000000 --- a/x-pack/plugins/alerting/server/alerts_service/types.ts +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; -import type { FieldMap } from '@kbn/alerts-as-data-utils'; -import { getComponentTemplateFromFieldMap } from '../../common'; - -export const getComponentTemplateName = (context?: string) => - `.alerts-${context || 'framework'}-mappings`; - -export interface IIndexPatternString { - template: string; - pattern: string; - alias: string; - name: string; -} - -export const getIndexTemplateAndPattern = ( - context: string, - namespace?: string -): IIndexPatternString => { - const pattern = `${context}-${namespace ? namespace : 'default'}`; - return { - template: `.alerts-${pattern}-template`, - pattern: `.alerts-${pattern}-*`, - alias: `.alerts-${pattern}`, - name: `.alerts-${pattern}-000001`, - }; -}; - -export const getComponentTemplate = ( - fieldMap: FieldMap, - context?: string -): ClusterPutComponentTemplateRequest => - getComponentTemplateFromFieldMap({ - name: getComponentTemplateName(context), - fieldMap, - // set field limit slightly higher than actual number of fields - fieldLimit: Math.ceil(Object.keys(fieldMap).length / 1000) * 1000 + 500, - }); diff --git a/x-pack/plugins/alerting/server/index.ts b/x-pack/plugins/alerting/server/index.ts index 6b5ad3012d8a9..d7546d8fd73a6 100644 --- a/x-pack/plugins/alerting/server/index.ts +++ b/x-pack/plugins/alerting/server/index.ts @@ -30,6 +30,7 @@ export type { RuleParamsAndRefs, GetSummarizedAlertsFnOpts, ExecutorType, + IRuleTypeAlerts, } from './types'; export { RuleNotifyWhen } from '../common'; export { DEFAULT_MAX_EPHEMERAL_ACTIONS_PER_ALERT } from './config'; @@ -59,6 +60,7 @@ export { ECS_COMPONENT_TEMPLATE_NAME, ECS_CONTEXT, getComponentTemplate, + type PublicFrameworkAlertsService, } from './alerts_service'; export const plugin = (initContext: PluginInitializerContext) => new AlertingPlugin(initContext); diff --git a/x-pack/plugins/alerting/server/mocks.ts b/x-pack/plugins/alerting/server/mocks.ts index 787ab3e9c856f..a5cadcd9bafa1 100644 --- a/x-pack/plugins/alerting/server/mocks.ts +++ b/x-pack/plugins/alerting/server/mocks.ts @@ -30,7 +30,10 @@ const createSetupMock = () => { registerType: jest.fn(), getSecurityHealth: jest.fn(), getConfig: jest.fn(), - getFrameworkAlertsEnabled: jest.fn(), + frameworkAlerts: { + enabled: jest.fn(), + getContextInitializationPromise: jest.fn(), + }, }; return mock; }; diff --git a/x-pack/plugins/alerting/server/plugin.test.ts b/x-pack/plugins/alerting/server/plugin.test.ts index f0dd5ce64315f..ed4b8575b9810 100644 --- a/x-pack/plugins/alerting/server/plugin.test.ts +++ b/x-pack/plugins/alerting/server/plugin.test.ts @@ -135,9 +135,8 @@ describe('Alerting Plugin', () => { const setupContract = await plugin.setup(setupMocks, mockPlugins); expect(AlertsService).toHaveBeenCalled(); - expect(mockAlertService.initialize).toHaveBeenCalled(); - expect(setupContract.getFrameworkAlertsEnabled()).toEqual(true); + expect(setupContract.frameworkAlerts.enabled()).toEqual(true); }); it(`exposes configured minimumScheduleInterval()`, async () => { @@ -153,7 +152,7 @@ describe('Alerting Plugin', () => { minimumScheduleInterval: { value: '1m', enforce: false }, }); - expect(setupContract.getFrameworkAlertsEnabled()).toEqual(false); + expect(setupContract.frameworkAlerts.enabled()).toEqual(false); }); describe('registerType()', () => { diff --git a/x-pack/plugins/alerting/server/plugin.ts b/x-pack/plugins/alerting/server/plugin.ts index 414d4f4e01b91..8840a541c10c7 100644 --- a/x-pack/plugins/alerting/server/plugin.ts +++ b/x-pack/plugins/alerting/server/plugin.ts @@ -87,7 +87,12 @@ import { getSecurityHealth, SecurityHealth } from './lib/get_security_health'; import { registerNodeCollector, registerClusterCollector, InMemoryMetrics } from './monitoring'; import { getRuleTaskTimeout } from './lib/get_rule_task_timeout'; import { getActionsConfigMap } from './lib/get_actions_config_map'; -import { AlertsService } from './alerts_service/alerts_service'; +import { + AlertsService, + type PublicFrameworkAlertsService, + type InitializationPromise, + errorResult, +} from './alerts_service'; import { rulesSettingsFeature } from './rules_settings_feature'; export const EVENT_LOG_PROVIDER = 'alerting'; @@ -126,7 +131,7 @@ export interface PluginSetupContract { ): void; getSecurityHealth: () => Promise; getConfig: () => AlertingRulesConfig; - getFrameworkAlertsEnabled: () => boolean; + frameworkAlerts: PublicFrameworkAlertsService; } export interface PluginStartContract { @@ -245,11 +250,11 @@ export class AlertingPlugin { this.alertsService = new AlertsService({ logger: this.logger, pluginStop$: this.pluginStop$, + kibanaVersion: this.kibanaVersion, elasticsearchClientPromise: core .getStartServices() .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser), }); - this.alertsService!.initialize(); } const ruleTypeRegistry = new RuleTypeRegistry({ @@ -386,7 +391,16 @@ export class AlertingPlugin { isUsingSecurity: this.licenseState ? !!this.licenseState.getIsSecurityEnabled() : false, }; }, - getFrameworkAlertsEnabled: () => this.config.enableFrameworkAlerts, + frameworkAlerts: { + enabled: () => this.config.enableFrameworkAlerts, + getContextInitializationPromise: (context: string): Promise => { + if (this.alertsService) { + return this.alertsService.getContextInitializationPromise(context); + } + + return Promise.resolve(errorResult(`Framework alerts service not available`)); + }, + }, }; } diff --git a/x-pack/plugins/alerting/server/rule_type_registry.test.ts b/x-pack/plugins/alerting/server/rule_type_registry.test.ts index 7ce33e2649d71..b6f6d6237fd63 100644 --- a/x-pack/plugins/alerting/server/rule_type_registry.test.ts +++ b/x-pack/plugins/alerting/server/rule_type_registry.test.ts @@ -472,13 +472,13 @@ describe('Create Lifecycle', () => { producer: 'alerts', alerts: { context: 'test', - fieldMap: { field: { type: 'keyword', required: false } }, + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, }, }); expect(alertsService.register).toHaveBeenCalledWith({ context: 'test', - fieldMap: { field: { type: 'keyword', required: false } }, + mappings: { fieldMap: { field: { type: 'keyword', required: false } } }, }); }); diff --git a/x-pack/plugins/alerting/server/types.ts b/x-pack/plugins/alerting/server/types.ts index 4997d9563d59e..8a463e7ec1aa1 100644 --- a/x-pack/plugins/alerting/server/types.ts +++ b/x-pack/plugins/alerting/server/types.ts @@ -168,12 +168,45 @@ export interface GetViewInAppRelativeUrlFnOpts { export type GetViewInAppRelativeUrlFn = ( opts: GetViewInAppRelativeUrlFnOpts ) => string; + +interface ComponentTemplateSpec { + dynamic?: 'strict' | false; // defaults to 'strict' + fieldMap: FieldMap; +} + export interface IRuleTypeAlerts { + /** + * Specifies the target alerts-as-data resource + * for this rule type. All alerts created with the same + * context are written to the same alerts-as-data index. + * + * All custom mappings defined for a context must be the same! + */ context: string; - namespace?: string; - fieldMap: FieldMap; + + /** + * Specifies custom mappings for the target alerts-as-data + * index. These mappings will be translated into a component template + * and used in the index template for the index. + */ + mappings: ComponentTemplateSpec; + + /** + * Optional flag to include a reference to the ECS component template. + */ useEcs?: boolean; + + /** + * Optional flag to include a reference to the legacy alert component template. + * Any rule type that is migrating from the rule registry should set this + * flag to true to ensure their alerts-as-data indices are backwards compatible. + */ useLegacyAlerts?: boolean; + + /** + * Optional secondary alias to use. This alias should not include the namespace. + */ + secondaryAlias?: string; } export interface RuleType< diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index 5b98c8e437782..43bbe13d9662c 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -15,13 +15,16 @@ import { PluginInitializerContext, } from '@kbn/core/server'; import { isEmpty, mapValues } from 'lodash'; -import { experimentalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/experimental_rule_field_map'; import { Dataset } from '@kbn/rule-registry-plugin/server'; import { UI_SETTINGS } from '@kbn/data-plugin/common'; import { mappingFromFieldMap } from '@kbn/alerting-plugin/common'; import { APMConfig, APM_SERVER_FEATURE_ID } from '.'; import { APM_FEATURE, registerFeaturesUsage } from './feature'; -import { registerApmRuleTypes } from './routes/alerts/register_apm_rule_types'; +import { + registerApmRuleTypes, + apmRuleTypeAlertFieldMap, + APM_RULE_TYPE_ALERT_CONTEXT, +} from './routes/alerts/register_apm_rule_types'; import { registerFleetPolicyCallbacks } from './routes/fleet/register_fleet_policy_callbacks'; import { createApmTelemetry } from './lib/apm_telemetry'; import { APMEventClient } from './lib/helpers/create_es_client/create_apm_event_client'; @@ -46,14 +49,6 @@ import { } from './types'; import { registerRoutes } from './routes/apm_routes/register_apm_server_routes'; import { getGlobalApmServerRouteRepository } from './routes/apm_routes/get_global_apm_server_route_repository'; -import { - PROCESSOR_EVENT, - SERVICE_ENVIRONMENT, - SERVICE_NAME, - TRANSACTION_TYPE, - AGENT_NAME, - SERVICE_LANGUAGE_NAME, -} from '../common/es_fields/apm'; import { tutorialProvider } from './tutorial'; import { migrateLegacyAPMIndicesToSpaceAware } from './saved_objects/migrations/migrate_legacy_apm_indices_to_space_aware'; import { scheduleSourceMapMigration } from './routes/source_maps/schedule_source_map_migration'; @@ -119,47 +114,13 @@ export class APMPlugin const { ruleDataService } = plugins.ruleRegistry; const ruleDataClient = ruleDataService.initializeIndex({ feature: APM_SERVER_FEATURE_ID, - registrationContext: 'observability.apm', + registrationContext: APM_RULE_TYPE_ALERT_CONTEXT, dataset: Dataset.alerts, componentTemplateRefs: [], componentTemplates: [ { name: 'mappings', - mappings: mappingFromFieldMap( - { - ...experimentalRuleFieldMap, - [SERVICE_NAME]: { - type: 'keyword', - required: false, - }, - [SERVICE_ENVIRONMENT]: { - type: 'keyword', - required: false, - }, - [TRANSACTION_TYPE]: { - type: 'keyword', - required: false, - }, - [PROCESSOR_EVENT]: { - type: 'keyword', - required: false, - }, - [AGENT_NAME]: { - type: 'keyword', - required: false, - }, - [SERVICE_LANGUAGE_NAME]: { - type: 'keyword', - required: false, - }, - labels: { - type: 'object', - dynamic: true, - required: false, - }, - }, - 'strict' - ), + mappings: mappingFromFieldMap(apmRuleTypeAlertFieldMap, 'strict'), }, ], }); diff --git a/x-pack/plugins/apm/server/routes/alerts/register_apm_rule_types.ts b/x-pack/plugins/apm/server/routes/alerts/register_apm_rule_types.ts index b2abf6b7ed126..951d733836528 100644 --- a/x-pack/plugins/apm/server/routes/alerts/register_apm_rule_types.ts +++ b/x-pack/plugins/apm/server/routes/alerts/register_apm_rule_types.ts @@ -7,16 +7,70 @@ import { Observable } from 'rxjs'; import { IBasePath, Logger } from '@kbn/core/server'; -import { PluginSetupContract as AlertingPluginSetupContract } from '@kbn/alerting-plugin/server'; +import { + PluginSetupContract as AlertingPluginSetupContract, + type IRuleTypeAlerts, +} from '@kbn/alerting-plugin/server'; import { ObservabilityPluginSetup } from '@kbn/observability-plugin/server'; import { IRuleDataClient } from '@kbn/rule-registry-plugin/server'; import { MlPluginSetup } from '@kbn/ml-plugin/server'; +import { legacyExperimentalFieldMap } from '@kbn/alerts-as-data-utils'; +import { + AGENT_NAME, + PROCESSOR_EVENT, + SERVICE_ENVIRONMENT, + SERVICE_LANGUAGE_NAME, + SERVICE_NAME, + TRANSACTION_TYPE, +} from '../../../common/es_fields/apm'; import { registerTransactionDurationRuleType } from './rule_types/transaction_duration/register_transaction_duration_rule_type'; import { registerAnomalyRuleType } from './rule_types/anomaly/register_anomaly_rule_type'; import { registerErrorCountRuleType } from './rule_types/error_count/register_error_count_rule_type'; import { APMConfig } from '../..'; import { registerTransactionErrorRateRuleType } from './rule_types/transaction_error_rate/register_transaction_error_rate_rule_type'; +export const APM_RULE_TYPE_ALERT_CONTEXT = 'observability.apm'; + +export const apmRuleTypeAlertFieldMap = { + ...legacyExperimentalFieldMap, + [SERVICE_NAME]: { + type: 'keyword', + required: false, + }, + [SERVICE_ENVIRONMENT]: { + type: 'keyword', + required: false, + }, + [TRANSACTION_TYPE]: { + type: 'keyword', + required: false, + }, + [PROCESSOR_EVENT]: { + type: 'keyword', + required: false, + }, + [AGENT_NAME]: { + type: 'keyword', + required: false, + }, + [SERVICE_LANGUAGE_NAME]: { + type: 'keyword', + required: false, + }, + labels: { + type: 'object', + dynamic: true, + required: false, + }, +}; + +// Defines which alerts-as-data index alerts will use +export const ApmRuleTypeAlertDefinition: IRuleTypeAlerts = { + context: APM_RULE_TYPE_ALERT_CONTEXT, + mappings: { fieldMap: apmRuleTypeAlertFieldMap }, + useLegacyAlerts: true, +}; + export interface RegisterRuleDependencies { alerting: AlertingPluginSetupContract; basePath: IBasePath; diff --git a/x-pack/plugins/apm/server/routes/alerts/rule_types/anomaly/register_anomaly_rule_type.ts b/x-pack/plugins/apm/server/routes/alerts/rule_types/anomaly/register_anomaly_rule_type.ts index a10d803ef6d86..1870ee8778356 100644 --- a/x-pack/plugins/apm/server/routes/alerts/rule_types/anomaly/register_anomaly_rule_type.ts +++ b/x-pack/plugins/apm/server/routes/alerts/rule_types/anomaly/register_anomaly_rule_type.ts @@ -45,7 +45,10 @@ import { asMutableArray } from '../../../../../common/utils/as_mutable_array'; import { getAlertUrlTransaction } from '../../../../../common/utils/formatters'; import { getMLJobs } from '../../../service_map/get_service_anomalies'; import { apmActionVariables } from '../../action_variables'; -import { RegisterRuleDependencies } from '../../register_apm_rule_types'; +import { + ApmRuleTypeAlertDefinition, + RegisterRuleDependencies, +} from '../../register_apm_rule_types'; import { getServiceGroupFieldsForAnomaly } from './get_service_group_fields_for_anomaly'; import { anomalyParamsSchema } from '../../../../../common/rules/schema'; @@ -322,6 +325,7 @@ export function registerAnomalyRuleType({ return { state: {} }; }, + alerts: ApmRuleTypeAlertDefinition, }) ); } diff --git a/x-pack/plugins/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.ts b/x-pack/plugins/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.ts index c811e71fe1f17..dee4c0e3cb950 100644 --- a/x-pack/plugins/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.ts +++ b/x-pack/plugins/apm/server/routes/alerts/rule_types/error_count/register_error_count_rule_type.ts @@ -42,7 +42,10 @@ import { getAlertUrlErrorCount } from '../../../../../common/utils/formatters'; import { getApmIndices } from '../../../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from '../../action_variables'; import { alertingEsClient } from '../../alerting_es_client'; -import { RegisterRuleDependencies } from '../../register_apm_rule_types'; +import { + ApmRuleTypeAlertDefinition, + RegisterRuleDependencies, +} from '../../register_apm_rule_types'; import { getServiceGroupFields, getServiceGroupFieldsAgg, @@ -221,6 +224,7 @@ export function registerErrorCountRuleType({ return { state: {} }; }, + alerts: ApmRuleTypeAlertDefinition, }) ); } diff --git a/x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts b/x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts index bc11cee03a506..3df41732c6e11 100644 --- a/x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts +++ b/x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_duration/register_transaction_duration_rule_type.ts @@ -53,7 +53,10 @@ import { import { getApmIndices } from '../../../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from '../../action_variables'; import { alertingEsClient } from '../../alerting_es_client'; -import { RegisterRuleDependencies } from '../../register_apm_rule_types'; +import { + ApmRuleTypeAlertDefinition, + RegisterRuleDependencies, +} from '../../register_apm_rule_types'; import { getServiceGroupFields, getServiceGroupFieldsAgg, @@ -293,6 +296,7 @@ export function registerTransactionDurationRuleType({ return { state: {} }; }, + alerts: ApmRuleTypeAlertDefinition, }); alerting.registerType(ruleType); diff --git a/x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.ts b/x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.ts index df94a3e8a3c75..305419e9ea362 100644 --- a/x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.ts +++ b/x-pack/plugins/apm/server/routes/alerts/rule_types/transaction_error_rate/register_transaction_error_rate_rule_type.ts @@ -51,7 +51,10 @@ import { getDocumentTypeFilterForTransactions } from '../../../../lib/helpers/tr import { getApmIndices } from '../../../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from '../../action_variables'; import { alertingEsClient } from '../../alerting_es_client'; -import { RegisterRuleDependencies } from '../../register_apm_rule_types'; +import { + ApmRuleTypeAlertDefinition, + RegisterRuleDependencies, +} from '../../register_apm_rule_types'; import { getServiceGroupFields, getServiceGroupFieldsAgg, @@ -295,6 +298,7 @@ export function registerTransactionErrorRateRuleType({ return { state: {} }; }, + alerts: ApmRuleTypeAlertDefinition, }) ); } diff --git a/x-pack/plugins/apm/tsconfig.json b/x-pack/plugins/apm/tsconfig.json index 4fc45adafd44d..149b43ebc6483 100644 --- a/x-pack/plugins/apm/tsconfig.json +++ b/x-pack/plugins/apm/tsconfig.json @@ -80,6 +80,7 @@ "@kbn/core-saved-objects-api-server", "@kbn/safer-lodash-set", "@kbn/shared-ux-router", + "@kbn/alerts-as-data-utils", ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/register_inventory_metric_threshold_rule_type.ts b/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/register_inventory_metric_threshold_rule_type.ts index 77fa814a622a4..ba7a0794757b0 100644 --- a/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/register_inventory_metric_threshold_rule_type.ts +++ b/x-pack/plugins/infra/server/lib/alerting/inventory_metric_threshold/register_inventory_metric_threshold_rule_type.ts @@ -52,6 +52,7 @@ import { FIRED_ACTIONS_ID, WARNING_ACTIONS, } from './inventory_metric_threshold_executor'; +import { MetricsRulesTypeAlertDefinition } from '../register_rule_types'; const condition = schema.object({ threshold: schema.arrayOf(schema.number()), @@ -154,5 +155,6 @@ export async function registerMetricInventoryThresholdRuleType( ], }, getSummarizedAlerts: libs.metricsRules.createGetSummarizedAlerts(), + alerts: MetricsRulesTypeAlertDefinition, }); } diff --git a/x-pack/plugins/infra/server/lib/alerting/log_threshold/register_log_threshold_rule_type.ts b/x-pack/plugins/infra/server/lib/alerting/log_threshold/register_log_threshold_rule_type.ts index 297416d0d6632..70cf8bfb59f76 100644 --- a/x-pack/plugins/infra/server/lib/alerting/log_threshold/register_log_threshold_rule_type.ts +++ b/x-pack/plugins/infra/server/lib/alerting/log_threshold/register_log_threshold_rule_type.ts @@ -26,6 +26,7 @@ import { orchestratorActionVariableDescription, tagsActionVariableDescription, } from '../common/messages'; +import { LogsRulesTypeAlertDefinition } from '../register_rule_types'; const timestampActionVariableDescription = i18n.translate( 'xpack.infra.logs.alerting.threshold.timestampActionVariableDescription', @@ -171,5 +172,6 @@ export async function registerLogThresholdRuleType( extractReferences, injectReferences, }, + alerts: LogsRulesTypeAlertDefinition, }); } diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/register_metric_threshold_rule_type.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/register_metric_threshold_rule_type.ts index d2d0d36f42ac5..0c18288236af7 100644 --- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/register_metric_threshold_rule_type.ts +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/register_metric_threshold_rule_type.ts @@ -42,6 +42,7 @@ import { WARNING_ACTIONS, NO_DATA_ACTIONS, } from './metric_threshold_executor'; +import { MetricsRulesTypeAlertDefinition } from '../register_rule_types'; type MetricThresholdAllowedActionGroups = ActionGroupIdsOf< typeof FIRED_ACTIONS | typeof WARNING_ACTIONS | typeof NO_DATA_ACTIONS @@ -192,5 +193,6 @@ export async function registerMetricThresholdRuleType( }, producer: 'infrastructure', getSummarizedAlerts: libs.metricsRules.createGetSummarizedAlerts(), + alerts: MetricsRulesTypeAlertDefinition, }); } diff --git a/x-pack/plugins/infra/server/lib/alerting/register_rule_types.ts b/x-pack/plugins/infra/server/lib/alerting/register_rule_types.ts index e27c5b7fd4809..ee05dc38cc1f5 100644 --- a/x-pack/plugins/infra/server/lib/alerting/register_rule_types.ts +++ b/x-pack/plugins/infra/server/lib/alerting/register_rule_types.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { PluginSetupContract } from '@kbn/alerting-plugin/server'; +import { legacyExperimentalFieldMap } from '@kbn/alerts-as-data-utils'; +import { type IRuleTypeAlerts, PluginSetupContract } from '@kbn/alerting-plugin/server'; import { MlPluginSetup } from '@kbn/ml-plugin/server'; import { registerMetricThresholdRuleType } from './metric_threshold/register_metric_threshold_rule_type'; import { registerMetricInventoryThresholdRuleType } from './inventory_metric_threshold/register_inventory_metric_threshold_rule_type'; @@ -13,6 +14,24 @@ import { registerMetricAnomalyRuleType } from './metric_anomaly/register_metric_ import { registerLogThresholdRuleType } from './log_threshold/register_log_threshold_rule_type'; import { InfraBackendLibs } from '../infra_types'; +export const LOGS_RULES_ALERT_CONTEXT = 'observability.logs'; +// Defines which alerts-as-data index logs rules will use +export const LogsRulesTypeAlertDefinition: IRuleTypeAlerts = { + context: LOGS_RULES_ALERT_CONTEXT, + mappings: { fieldMap: legacyExperimentalFieldMap }, + useEcs: true, + useLegacyAlerts: true, +}; + +export const METRICS_RULES_ALERT_CONTEXT = 'observability.metrics'; +// Defines which alerts-as-data index metrics rules will use +export const MetricsRulesTypeAlertDefinition: IRuleTypeAlerts = { + context: METRICS_RULES_ALERT_CONTEXT, + mappings: { fieldMap: legacyExperimentalFieldMap }, + useEcs: true, + useLegacyAlerts: true, +}; + const registerRuleTypes = ( alertingPlugin: PluginSetupContract, libs: InfraBackendLibs, diff --git a/x-pack/plugins/infra/server/plugin.ts b/x-pack/plugins/infra/server/plugin.ts index 67563f857aedb..0fb0aad42d0cc 100644 --- a/x-pack/plugins/infra/server/plugin.ts +++ b/x-pack/plugins/infra/server/plugin.ts @@ -32,6 +32,10 @@ import { InfraKibanaLogEntriesAdapter } from './lib/adapters/log_entries/kibana_ import { KibanaMetricsAdapter } from './lib/adapters/metrics/kibana_metrics_adapter'; import { InfraElasticsearchSourceStatusAdapter } from './lib/adapters/source_status'; import { registerRuleTypes } from './lib/alerting'; +import { + LOGS_RULES_ALERT_CONTEXT, + METRICS_RULES_ALERT_CONTEXT, +} from './lib/alerting/register_rule_types'; import { InfraFieldsDomain } from './lib/domains/fields_domain'; import { InfraLogEntriesDomain } from './lib/domains/log_entries_domain'; import { InfraMetricsDomain } from './lib/domains/metrics_domain'; @@ -116,12 +120,12 @@ export class InfraServerPlugin this.logsRules = new RulesService( LOGS_FEATURE_ID, - 'observability.logs', + LOGS_RULES_ALERT_CONTEXT, this.logger.get('logsRules') ); this.metricsRules = new RulesService( METRICS_FEATURE_ID, - 'observability.metrics', + METRICS_RULES_ALERT_CONTEXT, this.logger.get('metricsRules') ); diff --git a/x-pack/plugins/infra/server/services/rules/rule_data_client.ts b/x-pack/plugins/infra/server/services/rules/rule_data_client.ts index 3a81f957e9314..ebbe3139167c3 100644 --- a/x-pack/plugins/infra/server/services/rules/rule_data_client.ts +++ b/x-pack/plugins/infra/server/services/rules/rule_data_client.ts @@ -6,7 +6,7 @@ */ import { CoreSetup, Logger } from '@kbn/core/server'; -import { experimentalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/experimental_rule_field_map'; +import { legacyExperimentalFieldMap } from '@kbn/alerts-as-data-utils'; import { Dataset, RuleRegistryPluginSetupContract } from '@kbn/rule-registry-plugin/server'; import { mappingFromFieldMap } from '@kbn/alerting-plugin/common'; @@ -35,7 +35,7 @@ export const createRuleDataClient = ({ componentTemplates: [ { name: 'mappings', - mappings: mappingFromFieldMap(experimentalRuleFieldMap, 'strict'), + mappings: mappingFromFieldMap(legacyExperimentalFieldMap, 'strict'), }, ], }); diff --git a/x-pack/plugins/infra/tsconfig.json b/x-pack/plugins/infra/tsconfig.json index 07586b6a25e1c..2d9c6911435fd 100644 --- a/x-pack/plugins/infra/tsconfig.json +++ b/x-pack/plugins/infra/tsconfig.json @@ -53,8 +53,9 @@ "@kbn/core-saved-objects-common", "@kbn/core-analytics-server", "@kbn/analytics-client", + "@kbn/shared-ux-router", + "@kbn/alerts-as-data-utils", "@kbn/cases-plugin", - "@kbn/shared-ux-router" ], "exclude": ["target/**/*"] } diff --git a/x-pack/plugins/observability/public/pages/alerts/components/parse_alert.ts b/x-pack/plugins/observability/public/pages/alerts/components/parse_alert.ts index 6c34d6157fd0b..29f4e94f70f3e 100644 --- a/x-pack/plugins/observability/public/pages/alerts/components/parse_alert.ts +++ b/x-pack/plugins/observability/public/pages/alerts/components/parse_alert.ts @@ -14,7 +14,7 @@ import { ALERT_RULE_NAME, ALERT_REASON, } from '@kbn/rule-data-utils'; -import { experimentalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/experimental_rule_field_map'; +import { legacyExperimentalFieldMap } from '@kbn/alerts-as-data-utils'; import { parseTechnicalFields } from '@kbn/rule-registry-plugin/common/parse_technical_fields'; import { parseExperimentalFields } from '@kbn/rule-registry-plugin/common/parse_experimental_fields'; import type { TopAlert } from '..'; @@ -24,7 +24,7 @@ import { ObservabilityRuleTypeRegistry } from '../../../rules/create_observabili export const parseAlert = (observabilityRuleTypeRegistry: ObservabilityRuleTypeRegistry) => (alert: Record): TopAlert => { - const experimentalFields = Object.keys(experimentalRuleFieldMap); + const experimentalFields = Object.keys(legacyExperimentalFieldMap); const alertWithExperimentalFields = experimentalFields.reduce((acc, key) => { if (alert[key]) { return { ...acc, [key]: alert[key] }; diff --git a/x-pack/plugins/observability/server/lib/rules/slo_burn_rate/register.ts b/x-pack/plugins/observability/server/lib/rules/slo_burn_rate/register.ts index 4f01d59f70174..c847aa5fc484e 100644 --- a/x-pack/plugins/observability/server/lib/rules/slo_burn_rate/register.ts +++ b/x-pack/plugins/observability/server/lib/rules/slo_burn_rate/register.ts @@ -9,6 +9,9 @@ import { schema } from '@kbn/config-schema'; import { i18n } from '@kbn/i18n'; import { LicenseType } from '@kbn/licensing-plugin/server'; import { createLifecycleExecutor } from '@kbn/rule-registry-plugin/server'; +import { legacyExperimentalFieldMap } from '@kbn/alerts-as-data-utils'; +import { sloFeatureId } from '../../../../common'; +import { SLO_RULE_REGISTRATION_CONTEXT } from '../../../common/constants'; import { SLO_BURN_RATE_RULE_ID } from '../../../../common/constants'; import { FIRED_ACTION, getRuleExecutor } from './executor'; @@ -37,7 +40,7 @@ export function sloBurnRateRuleType(createLifecycleRuleExecutor: CreateLifecycle }, defaultActionGroupId: FIRED_ACTION.id, actionGroups: [FIRED_ACTION], - producer: 'slo', + producer: sloFeatureId, minimumLicenseRequired: 'platinum' as LicenseType, isExportable: true, executor: createLifecycleRuleExecutor(getRuleExecutor()), @@ -51,6 +54,12 @@ export function sloBurnRateRuleType(createLifecycleRuleExecutor: CreateLifecycle { name: 'shortWindow', description: windowActionVariableDescription }, ], }, + alerts: { + context: SLO_RULE_REGISTRATION_CONTEXT, + mappings: { fieldMap: legacyExperimentalFieldMap }, + useEcs: true, + useLegacyAlerts: true, + }, }; } diff --git a/x-pack/plugins/observability/server/plugin.ts b/x-pack/plugins/observability/server/plugin.ts index 4b2e2c722230b..b59421f8f0054 100644 --- a/x-pack/plugins/observability/server/plugin.ts +++ b/x-pack/plugins/observability/server/plugin.ts @@ -17,12 +17,12 @@ import { hiddenTypes as filesSavedObjectTypes } from '@kbn/files-plugin/server/s import { PluginSetupContract } from '@kbn/alerting-plugin/server'; import { Dataset, RuleRegistryPluginSetupContract } from '@kbn/rule-registry-plugin/server'; import { PluginSetupContract as FeaturesSetup } from '@kbn/features-plugin/server'; +import { legacyExperimentalFieldMap } from '@kbn/alerts-as-data-utils'; import { createUICapabilities as createCasesUICapabilities, getApiTags as getCasesApiTags, } from '@kbn/cases-plugin/common'; import { SpacesPluginSetup } from '@kbn/spaces-plugin/server'; -import { experimentalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/experimental_rule_field_map'; import { ECS_COMPONENT_TEMPLATE_NAME } from '@kbn/alerting-plugin/server'; import type { GuidedOnboardingPluginSetup } from '@kbn/guided-onboarding-plugin/server'; @@ -228,7 +228,7 @@ export class ObservabilityPlugin implements Plugin { componentTemplates: [ { name: 'mappings', - mappings: mappingFromFieldMap(experimentalRuleFieldMap, 'strict'), + mappings: mappingFromFieldMap(legacyExperimentalFieldMap, 'strict'), }, ], }); diff --git a/x-pack/plugins/observability/tsconfig.json b/x-pack/plugins/observability/tsconfig.json index eadf340054f52..37471473183d7 100644 --- a/x-pack/plugins/observability/tsconfig.json +++ b/x-pack/plugins/observability/tsconfig.json @@ -70,6 +70,7 @@ "@kbn/securitysolution-ecs", "@kbn/shared-ux-router", "@kbn/alerts-ui-shared", + "@kbn/alerts-as-data-utils", "@kbn/core-application-browser", "@kbn/files-plugin", ], diff --git a/x-pack/plugins/rule_registry/common/assets/field_maps/experimental_rule_field_map.test.ts b/x-pack/plugins/rule_registry/common/assets/field_maps/experimental_rule_field_map.test.ts deleted file mode 100644 index 3a6dbc4f20982..0000000000000 --- a/x-pack/plugins/rule_registry/common/assets/field_maps/experimental_rule_field_map.test.ts +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { experimentalRuleFieldMap } from './experimental_rule_field_map'; - -// This test purely exists to see what the resultant mappings are and -// make it obvious when some dependency results in the mappings changing -it('matches snapshot', () => { - expect(experimentalRuleFieldMap).toMatchInlineSnapshot(` - Object { - "kibana.alert.evaluation.threshold": Object { - "required": false, - "scaling_factor": 100, - "type": "scaled_float", - }, - "kibana.alert.evaluation.value": Object { - "required": false, - "scaling_factor": 100, - "type": "scaled_float", - }, - } - `); -}); diff --git a/x-pack/plugins/rule_registry/common/assets/field_maps/experimental_rule_field_map.ts b/x-pack/plugins/rule_registry/common/assets/field_maps/experimental_rule_field_map.ts deleted file mode 100644 index 3859ebe6df9b6..0000000000000 --- a/x-pack/plugins/rule_registry/common/assets/field_maps/experimental_rule_field_map.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import * as Fields from '../../technical_rule_data_field_names'; - -export const experimentalRuleFieldMap = { - [Fields.ALERT_EVALUATION_THRESHOLD]: { - type: 'scaled_float', - scaling_factor: 100, - required: false, - }, - [Fields.ALERT_EVALUATION_VALUE]: { type: 'scaled_float', scaling_factor: 100, required: false }, -} as const; - -export type ExperimentalRuleFieldMap = typeof experimentalRuleFieldMap; diff --git a/x-pack/plugins/rule_registry/common/parse_experimental_fields.ts b/x-pack/plugins/rule_registry/common/parse_experimental_fields.ts index 4347a7e5b8d48..77db88f2f9766 100644 --- a/x-pack/plugins/rule_registry/common/parse_experimental_fields.ts +++ b/x-pack/plugins/rule_registry/common/parse_experimental_fields.ts @@ -7,19 +7,17 @@ import { isLeft } from 'fp-ts/lib/Either'; import { PathReporter } from 'io-ts/lib/PathReporter'; import { pick } from 'lodash'; -import { - experimentalRuleFieldMap, - ExperimentalRuleFieldMap, -} from './assets/field_maps/experimental_rule_field_map'; +import { legacyExperimentalFieldMap, ExperimentalRuleFieldMap } from '@kbn/alerts-as-data-utils'; import { runtimeTypeFromFieldMap } from './field_map'; -const experimentalFieldRuntimeType = - runtimeTypeFromFieldMap(experimentalRuleFieldMap); +const experimentalFieldRuntimeType = runtimeTypeFromFieldMap( + legacyExperimentalFieldMap +); export const parseExperimentalFields = (input: unknown, partial = false) => { const decodePartial = (alert: unknown) => { - const limitedFields = pick(experimentalRuleFieldMap, Object.keys(alert as object)); + const limitedFields = pick(legacyExperimentalFieldMap, Object.keys(alert as object)); const partialTechnicalFieldRuntimeType = runtimeTypeFromFieldMap( limitedFields as unknown as ExperimentalRuleFieldMap ); diff --git a/x-pack/plugins/rule_registry/server/plugin.ts b/x-pack/plugins/rule_registry/server/plugin.ts index 8b5c754e5b908..12fec80f53bb6 100644 --- a/x-pack/plugins/rule_registry/server/plugin.ts +++ b/x-pack/plugins/rule_registry/server/plugin.ts @@ -110,7 +110,7 @@ export class RuleRegistryPlugin const deps = await startDependencies; return deps.core.elasticsearch.client.asInternalUser; }, - areFrameworkAlertsEnabled: plugins.alerting.getFrameworkAlertsEnabled(), + frameworkAlerts: plugins.alerting.frameworkAlerts, pluginStop$: this.pluginStop$, }); diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.test.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.test.ts index 083c4d08d4253..1b4668bb2fe80 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.test.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.test.ts @@ -16,6 +16,11 @@ import { ECS_COMPONENT_TEMPLATE_NAME } from '@kbn/alerting-plugin/server'; import { elasticsearchServiceMock, ElasticsearchClientMock } from '@kbn/core/server/mocks'; import { TECHNICAL_COMPONENT_TEMPLATE_NAME } from '../../common/assets'; +const frameworkAlertsService = { + enabled: () => false, + getContextInitializationPromise: async () => ({ result: false, error: `failed` }), +}; + describe('resourceInstaller', () => { let pluginStop$: Subject; @@ -38,7 +43,7 @@ describe('resourceInstaller', () => { disabledRegistrationContexts: [], getResourceName: jest.fn(), getClusterClient, - areFrameworkAlertsEnabled: false, + frameworkAlerts: frameworkAlertsService, pluginStop$, }); installer.installCommonResources(); @@ -55,7 +60,7 @@ describe('resourceInstaller', () => { disabledRegistrationContexts: [], getResourceName: jest.fn(), getClusterClient, - areFrameworkAlertsEnabled: false, + frameworkAlerts: frameworkAlertsService, pluginStop$, }); const indexOptions = { @@ -86,7 +91,7 @@ describe('resourceInstaller', () => { disabledRegistrationContexts: [], getResourceName: jest.fn(), getClusterClient, - areFrameworkAlertsEnabled: false, + frameworkAlerts: frameworkAlertsService, pluginStop$, }); @@ -113,7 +118,10 @@ describe('resourceInstaller', () => { disabledRegistrationContexts: [], getResourceName: jest.fn(), getClusterClient, - areFrameworkAlertsEnabled: true, + frameworkAlerts: { + ...frameworkAlertsService, + enabled: () => true, + }, pluginStop$, }); @@ -128,6 +136,7 @@ describe('resourceInstaller', () => { expect.objectContaining({ name: TECHNICAL_COMPONENT_TEMPLATE_NAME }) ); }); + it('should install index level resources', async () => { const mockClusterClient = elasticsearchServiceMock.createElasticsearchClient(); const getClusterClient = jest.fn(() => Promise.resolve(mockClusterClient)); @@ -137,7 +146,7 @@ describe('resourceInstaller', () => { disabledRegistrationContexts: [], getResourceName: jest.fn(), getClusterClient, - areFrameworkAlertsEnabled: false, + frameworkAlerts: frameworkAlertsService, pluginStop$, }); @@ -159,6 +168,265 @@ describe('resourceInstaller', () => { expect.objectContaining({ name: '.alerts-observability.logs.alerts-mappings' }) ); }); + + it('should not install index level component template when framework alerts are enabled', async () => { + const mockClusterClient = elasticsearchServiceMock.createElasticsearchClient(); + const getClusterClient = jest.fn(() => Promise.resolve(mockClusterClient)); + const installer = new ResourceInstaller({ + logger: loggerMock.create(), + isWriteEnabled: true, + disabledRegistrationContexts: [], + getResourceName: jest.fn(), + getClusterClient, + frameworkAlerts: { + ...frameworkAlertsService, + enabled: () => true, + }, + pluginStop$, + }); + + const indexOptions = { + feature: AlertConsumers.LOGS, + registrationContext: 'observability.logs', + dataset: Dataset.alerts, + componentTemplateRefs: [], + componentTemplates: [ + { + name: 'mappings', + }, + ], + }; + const indexInfo = new IndexInfo({ indexOptions, kibanaVersion: '8.1.0' }); + + await installer.installIndexLevelResources(indexInfo); + expect(mockClusterClient.cluster.putComponentTemplate).not.toHaveBeenCalled(); + }); + + it('should install namespace level resources for the default space', async () => { + const mockClusterClient = elasticsearchServiceMock.createElasticsearchClient(); + mockClusterClient.indices.simulateTemplate.mockImplementation(async () => ({ + template: { + aliases: { + alias_name_1: { + is_hidden: true, + }, + alias_name_2: { + is_hidden: true, + }, + }, + mappings: { enabled: false }, + settings: {}, + }, + })); + mockClusterClient.indices.getAlias.mockImplementation(async () => ({ + real_index: { + aliases: { + alias_1: { + is_hidden: true, + }, + alias_2: { + is_hidden: true, + }, + }, + }, + })); + const getClusterClient = jest.fn(() => Promise.resolve(mockClusterClient)); + const installer = new ResourceInstaller({ + logger: loggerMock.create(), + isWriteEnabled: true, + disabledRegistrationContexts: [], + getResourceName: jest.fn(), + getClusterClient, + frameworkAlerts: frameworkAlertsService, + pluginStop$, + }); + + const indexOptions = { + feature: AlertConsumers.LOGS, + registrationContext: 'observability.logs', + dataset: Dataset.alerts, + componentTemplateRefs: [], + componentTemplates: [ + { + name: 'mappings', + }, + ], + }; + const indexInfo = new IndexInfo({ indexOptions, kibanaVersion: '8.1.0' }); + + await installer.installAndUpdateNamespaceLevelResources(indexInfo, 'default'); + expect(mockClusterClient.indices.simulateTemplate).toHaveBeenCalledWith( + expect.objectContaining({ + name: '.alerts-observability.logs.alerts-default-index-template', + }) + ); + expect(mockClusterClient.indices.putIndexTemplate).toHaveBeenCalledWith( + expect.objectContaining({ + name: '.alerts-observability.logs.alerts-default-index-template', + }) + ); + expect(mockClusterClient.indices.getAlias).toHaveBeenCalledWith( + expect.objectContaining({ name: '.alerts-observability.logs.alerts-*' }) + ); + expect(mockClusterClient.indices.create).toHaveBeenCalledWith( + expect.objectContaining({ + index: '.internal.alerts-observability.logs.alerts-default-000001', + }) + ); + }); + + it('should not install namespace level resources for the default space when framework alerts are available', async () => { + const mockClusterClient = elasticsearchServiceMock.createElasticsearchClient(); + const getClusterClient = jest.fn(() => Promise.resolve(mockClusterClient)); + const installer = new ResourceInstaller({ + logger: loggerMock.create(), + isWriteEnabled: true, + disabledRegistrationContexts: [], + getResourceName: jest.fn(), + getClusterClient, + frameworkAlerts: { + ...frameworkAlertsService, + enabled: () => true, + getContextInitializationPromise: async () => ({ result: true }), + }, + pluginStop$, + }); + + const indexOptions = { + feature: AlertConsumers.LOGS, + registrationContext: 'observability.logs', + dataset: Dataset.alerts, + componentTemplateRefs: [], + componentTemplates: [ + { + name: 'mappings', + }, + ], + }; + const indexInfo = new IndexInfo({ indexOptions, kibanaVersion: '8.1.0' }); + + await installer.installAndUpdateNamespaceLevelResources(indexInfo, 'default'); + expect(mockClusterClient.indices.simulateTemplate).not.toHaveBeenCalled(); + expect(mockClusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(mockClusterClient.indices.getAlias).not.toHaveBeenCalled(); + expect(mockClusterClient.indices.create).not.toHaveBeenCalled(); + }); + + it('should throw error if framework was unable to install namespace level resources', async () => { + const mockClusterClient = elasticsearchServiceMock.createElasticsearchClient(); + const getClusterClient = jest.fn(() => Promise.resolve(mockClusterClient)); + const installer = new ResourceInstaller({ + logger: loggerMock.create(), + isWriteEnabled: true, + disabledRegistrationContexts: [], + getResourceName: jest.fn(), + getClusterClient, + frameworkAlerts: { + ...frameworkAlertsService, + enabled: () => true, + }, + pluginStop$, + }); + + const indexOptions = { + feature: AlertConsumers.LOGS, + registrationContext: 'observability.logs', + dataset: Dataset.alerts, + componentTemplateRefs: [], + componentTemplates: [ + { + name: 'mappings', + }, + ], + }; + const indexInfo = new IndexInfo({ indexOptions, kibanaVersion: '8.1.0' }); + + await expect( + installer.installAndUpdateNamespaceLevelResources(indexInfo, 'default') + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"There was an error in the framework installing namespace-level resources and creating concrete indices for .alerts-observability.logs.alerts-default - failed"` + ); + expect(mockClusterClient.indices.simulateTemplate).not.toHaveBeenCalled(); + expect(mockClusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + expect(mockClusterClient.indices.getAlias).not.toHaveBeenCalled(); + expect(mockClusterClient.indices.create).not.toHaveBeenCalled(); + }); + + it('should install namespace level resources for non-default space even when framework alerts are available', async () => { + const mockClusterClient = elasticsearchServiceMock.createElasticsearchClient(); + mockClusterClient.indices.simulateTemplate.mockImplementation(async () => ({ + template: { + aliases: { + alias_name_1: { + is_hidden: true, + }, + alias_name_2: { + is_hidden: true, + }, + }, + mappings: { enabled: false }, + settings: {}, + }, + })); + mockClusterClient.indices.getAlias.mockImplementation(async () => ({ + real_index: { + aliases: { + alias_1: { + is_hidden: true, + }, + alias_2: { + is_hidden: true, + }, + }, + }, + })); + const getClusterClient = jest.fn(() => Promise.resolve(mockClusterClient)); + const installer = new ResourceInstaller({ + logger: loggerMock.create(), + isWriteEnabled: true, + disabledRegistrationContexts: [], + getResourceName: jest.fn(), + getClusterClient, + frameworkAlerts: { + ...frameworkAlertsService, + enabled: () => true, + }, + pluginStop$, + }); + + const indexOptions = { + feature: AlertConsumers.LOGS, + registrationContext: 'observability.logs', + dataset: Dataset.alerts, + componentTemplateRefs: [], + componentTemplates: [ + { + name: 'mappings', + }, + ], + }; + const indexInfo = new IndexInfo({ indexOptions, kibanaVersion: '8.1.0' }); + + await installer.installAndUpdateNamespaceLevelResources(indexInfo, 'my-staging-space'); + expect(mockClusterClient.indices.simulateTemplate).toHaveBeenCalledWith( + expect.objectContaining({ + name: '.alerts-observability.logs.alerts-my-staging-space-index-template', + }) + ); + expect(mockClusterClient.indices.putIndexTemplate).toHaveBeenCalledWith( + expect.objectContaining({ + name: '.alerts-observability.logs.alerts-my-staging-space-index-template', + }) + ); + expect(mockClusterClient.indices.getAlias).toHaveBeenCalledWith( + expect.objectContaining({ name: '.alerts-observability.logs.alerts-*' }) + ); + expect(mockClusterClient.indices.create).toHaveBeenCalledWith( + expect.objectContaining({ + index: '.internal.alerts-observability.logs.alerts-my-staging-space-000001', + }) + ); + }); }); // These tests only test the updateAliasWriteIndexMapping() @@ -209,7 +477,7 @@ describe('resourceInstaller', () => { disabledRegistrationContexts: [], getResourceName: jest.fn(), getClusterClient: async () => mockClusterClient, - areFrameworkAlertsEnabled: false, + frameworkAlerts: frameworkAlertsService, pluginStop$, }; const indexOptions = { diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts index 59c74b81712d8..266b548e01c06 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts @@ -16,6 +16,7 @@ import { DEFAULT_ALERTS_ILM_POLICY, DEFAULT_ALERTS_ILM_POLICY_NAME, ECS_COMPONENT_TEMPLATE_NAME, + type PublicFrameworkAlertsService, } from '@kbn/alerting-plugin/server'; import { TECHNICAL_COMPONENT_TEMPLATE_NAME } from '../../common/assets'; import { technicalComponentTemplate } from '../../common/assets/component_templates/technical_component_template'; @@ -31,7 +32,7 @@ interface ConstructorOptions { logger: Logger; isWriteEnabled: boolean; disabledRegistrationContexts: string[]; - areFrameworkAlertsEnabled: boolean; + frameworkAlerts: PublicFrameworkAlertsService; pluginStop$: Observable; } @@ -96,14 +97,14 @@ export class ResourceInstaller { */ public async installCommonResources(): Promise { await this.installWithTimeout('common resources shared between all indices', async () => { - const { logger, areFrameworkAlertsEnabled } = this.options; + const { logger, frameworkAlerts } = this.options; try { // We can install them in parallel await Promise.all([ - // Install ILM policy only if framework alerts are not enabled - // If framework alerts are enabled, the alerting framework will install this ILM policy - ...(areFrameworkAlertsEnabled + // Install ILM policy and ECS component template only if framework alerts are not enabled + // If framework alerts are enabled, the alerting framework will install these + ...(frameworkAlerts.enabled() ? [] : [ this.createOrUpdateLifecyclePolicy({ @@ -139,7 +140,8 @@ export class ResourceInstaller { */ public async installIndexLevelResources(indexInfo: IndexInfo): Promise { await this.installWithTimeout(`resources for index ${indexInfo.baseName}`, async () => { - const { componentTemplates, ilmPolicy } = indexInfo.indexOptions; + const { frameworkAlerts } = this.options; + const { componentTemplates, ilmPolicy, additionalPrefix } = indexInfo.indexOptions; if (ilmPolicy != null) { await this.createOrUpdateLifecyclePolicy({ name: indexInfo.getIlmPolicyName(), @@ -147,20 +149,22 @@ export class ResourceInstaller { }); } - await Promise.all( - componentTemplates.map(async (ct) => { - await this.createOrUpdateComponentTemplate({ - name: indexInfo.getComponentTemplateName(ct.name), - body: { - template: { - settings: ct.settings ?? {}, - mappings: ct.mappings, + if (!frameworkAlerts.enabled() || additionalPrefix) { + await Promise.all( + componentTemplates.map(async (ct) => { + await this.createOrUpdateComponentTemplate({ + name: indexInfo.getComponentTemplateName(ct.name), + body: { + template: { + settings: ct.settings ?? {}, + mappings: ct.mappings, + }, + _meta: ct._meta, }, - _meta: ct._meta, - }, - }); - }) - ); + }); + }) + ); + } }); } @@ -252,10 +256,28 @@ export class ResourceInstaller { indexInfo: IndexInfo, namespace: string ): Promise { - const { logger } = this.options; + const { logger, frameworkAlerts } = this.options; const alias = indexInfo.getPrimaryAlias(namespace); + if ( + namespace === 'default' && + !indexInfo.indexOptions.additionalPrefix && + frameworkAlerts.enabled() + ) { + const { result: initialized, error } = await frameworkAlerts.getContextInitializationPromise( + indexInfo.indexOptions.registrationContext + ); + + if (!initialized) { + throw new Error( + `There was an error in the framework installing namespace-level resources and creating concrete indices for ${alias} - ${error}` + ); + } else { + return; + } + } + logger.info(`Installing namespace-level resources and creating concrete index for ${alias}`); // Install / update the index template diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.test.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.test.ts index 1022ea038bc3e..2ba9146f4f2db 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.test.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.test.ts @@ -18,6 +18,11 @@ jest.mock('../rule_data_client/rule_data_client', () => ({ RuleDataClient: jest.fn().mockImplementation(() => mockCreateRuleDataClient()), })); +const frameworkAlertsService = { + enabled: () => false, + getContextInitializationPromise: async () => ({ result: false }), +}; + describe('ruleDataPluginService', () => { let pluginStop$: Subject; @@ -43,7 +48,7 @@ describe('ruleDataPluginService', () => { isWriteEnabled: true, disabledRegistrationContexts: ['observability.logs'], isWriterCacheEnabled: true, - areFrameworkAlertsEnabled: false, + frameworkAlerts: frameworkAlertsService, pluginStop$, }); expect(ruleDataService.isRegistrationContextDisabled('observability.logs')).toBe(true); @@ -60,7 +65,7 @@ describe('ruleDataPluginService', () => { isWriteEnabled: true, disabledRegistrationContexts: ['observability.logs'], isWriterCacheEnabled: true, - areFrameworkAlertsEnabled: false, + frameworkAlerts: frameworkAlertsService, pluginStop$, }); expect(ruleDataService.isRegistrationContextDisabled('observability.apm')).toBe(false); @@ -79,7 +84,7 @@ describe('ruleDataPluginService', () => { isWriteEnabled: true, disabledRegistrationContexts: ['observability.logs'], isWriterCacheEnabled: true, - areFrameworkAlertsEnabled: false, + frameworkAlerts: frameworkAlertsService, pluginStop$, }); @@ -99,7 +104,7 @@ describe('ruleDataPluginService', () => { isWriteEnabled: true, disabledRegistrationContexts: ['observability.logs'], isWriterCacheEnabled: true, - areFrameworkAlertsEnabled: false, + frameworkAlerts: frameworkAlertsService, pluginStop$, }); const indexOptions = { diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.ts index b3f54a1d3794d..62f8cc88ca221 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/rule_data_plugin_service.ts @@ -11,6 +11,7 @@ import type { ValidFeatureId } from '@kbn/rule-data-utils'; import type { ElasticsearchClient, Logger } from '@kbn/core/server'; +import { type PublicFrameworkAlertsService } from '@kbn/alerting-plugin/server'; import { INDEX_PREFIX } from '../config'; import { type IRuleDataClient, RuleDataClient, WaitResult } from '../rule_data_client'; import { IndexInfo } from './index_info'; @@ -91,7 +92,7 @@ interface ConstructorOptions { isWriteEnabled: boolean; isWriterCacheEnabled: boolean; disabledRegistrationContexts: string[]; - areFrameworkAlertsEnabled: boolean; + frameworkAlerts: PublicFrameworkAlertsService; pluginStop$: Observable; } @@ -113,7 +114,7 @@ export class RuleDataService implements IRuleDataService { logger: options.logger, disabledRegistrationContexts: options.disabledRegistrationContexts, isWriteEnabled: options.isWriteEnabled, - areFrameworkAlertsEnabled: options.areFrameworkAlertsEnabled, + frameworkAlerts: options.frameworkAlerts, pluginStop$: options.pluginStop$, }); diff --git a/x-pack/plugins/security_solution/common/utils/field_formatters.ts b/x-pack/plugins/security_solution/common/utils/field_formatters.ts index 65fc3871c7fc3..da41cbd3bee94 100644 --- a/x-pack/plugins/security_solution/common/utils/field_formatters.ts +++ b/x-pack/plugins/security_solution/common/utils/field_formatters.ts @@ -6,7 +6,7 @@ */ import { ecsFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/ecs_field_map'; -import { experimentalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/experimental_rule_field_map'; +import { legacyExperimentalFieldMap } from '@kbn/alerts-as-data-utils'; import { technicalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/technical_rule_field_map'; import { isEmpty } from 'lodash/fp'; import { ENRICHMENT_DESTINATION_PATH } from '../constants'; @@ -76,7 +76,7 @@ export const getDataFromFieldsHits = ( // return simple field value (non-esc object, non-array) if ( !isObjectArray || - Object.keys({ ...ecsFieldMap, ...technicalRuleFieldMap, ...experimentalRuleFieldMap }).find( + Object.keys({ ...ecsFieldMap, ...technicalRuleFieldMap, ...legacyExperimentalFieldMap }).find( (ecsField) => ecsField === field ) === undefined ) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts index 38ce087fb6aae..99339a4e8fb53 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts @@ -14,6 +14,8 @@ import { createPersistenceRuleTypeWrapper } from '@kbn/rule-registry-plugin/serv import { parseScheduleDates } from '@kbn/securitysolution-io-ts-utils'; import { buildExceptionFilter } from '@kbn/lists-plugin/server/services/exception_lists'; +import { technicalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/technical_rule_field_map'; +import type { FieldMap } from '@kbn/alerts-as-data-utils'; import { checkPrivilegesFromEsClient, getExceptions, @@ -43,6 +45,23 @@ import { withSecuritySpan } from '../../../utils/with_security_span'; import { getInputIndex, DataViewError } from './utils/get_input_output_index'; import { TIMESTAMP_RUNTIME_FIELD } from './constants'; import { buildTimestampRuntimeMapping } from './utils/build_timestamp_runtime_mapping'; +import { alertsFieldMap, rulesFieldMap } from '../../../../common/field_maps'; + +const aliasesFieldMap: FieldMap = {}; +Object.entries(aadFieldConversion).forEach(([key, value]) => { + aliasesFieldMap[key] = { + type: 'alias', + required: false, + path: value, + }; +}); + +export const securityRuleTypeFieldMap = { + ...technicalRuleFieldMap, + ...alertsFieldMap, + ...rulesFieldMap, + ...aliasesFieldMap, +}; /* eslint-disable complexity */ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = @@ -509,5 +528,15 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = return { state: result.state }; }); }, + alerts: { + context: 'security', + mappings: { + dynamic: false, + fieldMap: securityRuleTypeFieldMap, + }, + useEcs: true, + useLegacyAlerts: true, + secondaryAlias: config.signalsIndex, + }, }); }; diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index cbc595a22bbf0..58bfba0b9bd37 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -23,8 +23,6 @@ import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; import { ECS_COMPONENT_TEMPLATE_NAME } from '@kbn/alerting-plugin/server'; import { mappingFromFieldMap } from '@kbn/alerting-plugin/common'; -import type { FieldMap } from '@kbn/alerts-as-data-utils'; -import { technicalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/technical_rule_field_map'; import type { IRuleDataClient } from '@kbn/rule-registry-plugin/server'; import { Dataset } from '@kbn/rule-registry-plugin/server'; import type { ListPluginSetup } from '@kbn/lists-plugin/server'; @@ -71,7 +69,6 @@ import { TelemetryReceiver } from './lib/telemetry/receiver'; import { licenseService } from './lib/license'; import { PolicyWatcher } from './endpoint/lib/policy/license_watch'; import { migrateArtifactsToFleet } from './endpoint/lib/artifacts/migrate_artifacts_to_fleet'; -import aadFieldConversion from './lib/detection_engine/routes/index/signal_aad_mapping.json'; import previewPolicy from './lib/detection_engine/routes/index/preview_policy.json'; import { createRuleExecutionLogService } from './lib/detection_engine/rule_monitoring'; import { getKibanaPrivilegesFeaturePrivileges, getCasesKibanaFeature } from './features'; @@ -85,7 +82,10 @@ import { legacyRulesNotificationAlertType, legacyIsNotificationAlertExecutor, } from './lib/detection_engine/rule_actions_legacy'; -import { createSecurityRuleTypeWrapper } from './lib/detection_engine/rule_types/create_security_rule_type_wrapper'; +import { + createSecurityRuleTypeWrapper, + securityRuleTypeFieldMap, +} from './lib/detection_engine/rule_types/create_security_rule_type_wrapper'; import { RequestContextFactory } from './request_context_factory'; @@ -99,7 +99,6 @@ import type { SecuritySolutionPluginStart, PluginInitializerContext, } from './plugin_contract'; -import { alertsFieldMap, rulesFieldMap } from '../common/field_maps'; import { EndpointFleetServicesFactory } from './endpoint/services/fleet'; import { featureUsageService } from './endpoint/services/feature_usage'; import { setIsElasticCloudDeployment } from './lib/telemetry/helpers'; @@ -216,15 +215,6 @@ export class Plugin implements ISecuritySolutionPlugin { version: pluginContext.env.packageInfo.version, }; - const aliasesFieldMap: FieldMap = {}; - Object.entries(aadFieldConversion).forEach(([key, value]) => { - aliasesFieldMap[key] = { - type: 'alias', - required: false, - path: value, - }; - }); - const ruleDataServiceOptions = { feature: SERVER_APP_ID, registrationContext: 'security', @@ -233,10 +223,7 @@ export class Plugin implements ISecuritySolutionPlugin { componentTemplates: [ { name: 'mappings', - mappings: mappingFromFieldMap( - { ...technicalRuleFieldMap, ...alertsFieldMap, ...rulesFieldMap, ...aliasesFieldMap }, - false - ), + mappings: mappingFromFieldMap(securityRuleTypeFieldMap, false), }, ], secondaryAlias: config.signalsIndex, diff --git a/x-pack/plugins/synthetics/common/constants/synthetics_alerts.ts b/x-pack/plugins/synthetics/common/constants/synthetics_alerts.ts index 454dd8fd79297..7f6d306f8be88 100644 --- a/x-pack/plugins/synthetics/common/constants/synthetics_alerts.ts +++ b/x-pack/plugins/synthetics/common/constants/synthetics_alerts.ts @@ -31,3 +31,5 @@ export const SYNTHETICS_ALERT_RULE_TYPES = { }; export const SYNTHETICS_RULE_TYPES = [SYNTHETICS_STATUS_RULE]; + +export const SYNTHETICS_RULE_TYPES_ALERT_CONTEXT = 'observability.uptime'; diff --git a/x-pack/plugins/synthetics/server/alert_rules/status_rule/monitor_status_rule.ts b/x-pack/plugins/synthetics/server/alert_rules/status_rule/monitor_status_rule.ts index e1719c1e84b93..6622c68d31004 100644 --- a/x-pack/plugins/synthetics/server/alert_rules/status_rule/monitor_status_rule.ts +++ b/x-pack/plugins/synthetics/server/alert_rules/status_rule/monitor_status_rule.ts @@ -34,6 +34,7 @@ import { import { getInstanceId } from '../../legacy_uptime/lib/alerts/status_check'; import { UMServerLibs } from '../../legacy_uptime/uptime_server'; import { SyntheticsMonitorClient } from '../../synthetics_service/synthetics_monitor/synthetics_monitor_client'; +import { UptimeRuleTypeAlertDefinition } from '../../legacy_uptime/lib/alerts/common'; export type ActionGroupIds = ActionGroupIdsOf; @@ -136,5 +137,6 @@ export const registerSyntheticsStatusCheckRule = ( state: updateState(ruleState, !isEmpty(downConfigs), { downConfigs }), }; }, + alerts: UptimeRuleTypeAlertDefinition, }); }; diff --git a/x-pack/plugins/synthetics/server/legacy_uptime/lib/alerts/common.ts b/x-pack/plugins/synthetics/server/legacy_uptime/lib/alerts/common.ts index 2da05dcd42e23..4ff12c7f1623c 100644 --- a/x-pack/plugins/synthetics/server/legacy_uptime/lib/alerts/common.ts +++ b/x-pack/plugins/synthetics/server/legacy_uptime/lib/alerts/common.ts @@ -7,9 +7,12 @@ import { isRight } from 'fp-ts/lib/Either'; import Mustache from 'mustache'; +import { legacyExperimentalFieldMap } from '@kbn/alerts-as-data-utils'; import { IBasePath } from '@kbn/core/server'; -import { RuleExecutorServices } from '@kbn/alerting-plugin/server'; +import { type IRuleTypeAlerts, RuleExecutorServices } from '@kbn/alerting-plugin/server'; import { addSpaceIdToPath } from '@kbn/spaces-plugin/common'; +import { uptimeRuleFieldMap } from '../../../../common/rules/uptime_rule_field_map'; +import { SYNTHETICS_RULE_TYPES_ALERT_CONTEXT } from '../../../../common/constants/synthetics_alerts'; import { UptimeCommonState, UptimeCommonStateType } from '../../../../common/runtime_types'; import { ALERT_DETAILS_URL } from './action_variables'; @@ -103,3 +106,11 @@ export const setRecoveredAlertsContext = ({ }); } }; + +export const uptimeRuleTypeFieldMap = { ...uptimeRuleFieldMap, ...legacyExperimentalFieldMap }; + +export const UptimeRuleTypeAlertDefinition: IRuleTypeAlerts = { + context: SYNTHETICS_RULE_TYPES_ALERT_CONTEXT, + mappings: { fieldMap: uptimeRuleTypeFieldMap }, + useLegacyAlerts: true, +}; diff --git a/x-pack/plugins/synthetics/server/legacy_uptime/lib/alerts/duration_anomaly.ts b/x-pack/plugins/synthetics/server/legacy_uptime/lib/alerts/duration_anomaly.ts index b959d16d5ade0..f70e3a96d1fb8 100644 --- a/x-pack/plugins/synthetics/server/legacy_uptime/lib/alerts/duration_anomaly.ts +++ b/x-pack/plugins/synthetics/server/legacy_uptime/lib/alerts/duration_anomaly.ts @@ -21,6 +21,7 @@ import { generateAlertMessage, getViewInAppUrl, setRecoveredAlertsContext, + UptimeRuleTypeAlertDefinition, } from './common'; import { CLIENT_ALERT_TYPES, DURATION_ANOMALY } from '../../../../common/constants/uptime_alerts'; import { commonStateTranslations, durationAnomalyTranslations } from './translations'; @@ -189,4 +190,5 @@ export const durationAnomalyAlertFactory: UptimeAlertTypeFactory return { state: updateState(state, foundAnomalies) }; }, + alerts: UptimeRuleTypeAlertDefinition, }); diff --git a/x-pack/plugins/synthetics/server/legacy_uptime/lib/alerts/status_check.ts b/x-pack/plugins/synthetics/server/legacy_uptime/lib/alerts/status_check.ts index 488a3ef2d284c..6da1aabeda9f4 100644 --- a/x-pack/plugins/synthetics/server/legacy_uptime/lib/alerts/status_check.ts +++ b/x-pack/plugins/synthetics/server/legacy_uptime/lib/alerts/status_check.ts @@ -28,6 +28,7 @@ import { getViewInAppUrl, setRecoveredAlertsContext, getAlertDetailsUrl, + UptimeRuleTypeAlertDefinition, } from './common'; import { commonMonitorStateI18, @@ -533,4 +534,5 @@ export const statusCheckAlertFactory: UptimeAlertTypeFactory = ( return { state: updateState(state, downMonitorsByLocation.length > 0) }; }, + alerts: UptimeRuleTypeAlertDefinition, }); diff --git a/x-pack/plugins/synthetics/server/legacy_uptime/lib/alerts/tls.ts b/x-pack/plugins/synthetics/server/legacy_uptime/lib/alerts/tls.ts index 3d39d1d9329e6..6b23d90db0a7e 100644 --- a/x-pack/plugins/synthetics/server/legacy_uptime/lib/alerts/tls.ts +++ b/x-pack/plugins/synthetics/server/legacy_uptime/lib/alerts/tls.ts @@ -16,6 +16,7 @@ import { generateAlertMessage, setRecoveredAlertsContext, getAlertDetailsUrl, + UptimeRuleTypeAlertDefinition, } from './common'; import { CLIENT_ALERT_TYPES, TLS } from '../../../../common/constants/uptime_alerts'; import { DYNAMIC_SETTINGS_DEFAULTS } from '../../../../common/constants'; @@ -227,4 +228,5 @@ export const tlsAlertFactory: UptimeAlertTypeFactory = ( return { state: updateState(state, foundCerts) }; }, + alerts: UptimeRuleTypeAlertDefinition, }); diff --git a/x-pack/plugins/synthetics/server/plugin.ts b/x-pack/plugins/synthetics/server/plugin.ts index c6120c70c3818..298f733f3d3e4 100644 --- a/x-pack/plugins/synthetics/server/plugin.ts +++ b/x-pack/plugins/synthetics/server/plugin.ts @@ -14,13 +14,11 @@ import { SavedObjectsClientContract, } from '@kbn/core/server'; import { mappingFromFieldMap } from '@kbn/alerting-plugin/common'; -import { experimentalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/experimental_rule_field_map'; import { Dataset } from '@kbn/rule-registry-plugin/server'; import { SyntheticsMonitorClient } from './synthetics_service/synthetics_monitor/synthetics_monitor_client'; import { initSyntheticsServer } from './server'; import { initUptimeServer } from './legacy_uptime/uptime_server'; import { uptimeFeature } from './feature'; -import { uptimeRuleFieldMap } from '../common/rules/uptime_rule_field_map'; import { KibanaTelemetryAdapter, UptimeCorePluginsSetup, @@ -35,6 +33,8 @@ import { import { UptimeConfig } from '../common/config'; import { SyntheticsService } from './synthetics_service/synthetics_service'; import { syntheticsServiceApiKey } from './legacy_uptime/lib/saved_objects/service_api_key'; +import { SYNTHETICS_RULE_TYPES_ALERT_CONTEXT } from '../common/constants/synthetics_alerts'; +import { uptimeRuleTypeFieldMap } from './legacy_uptime/lib/alerts/common'; export type UptimeRuleRegistry = ReturnType['ruleRegistry']; @@ -63,16 +63,13 @@ export class Plugin implements PluginType { const ruleDataClient = ruleDataService.initializeIndex({ feature: 'uptime', - registrationContext: 'observability.uptime', + registrationContext: SYNTHETICS_RULE_TYPES_ALERT_CONTEXT, dataset: Dataset.alerts, componentTemplateRefs: [], componentTemplates: [ { name: 'mappings', - mappings: mappingFromFieldMap( - { ...uptimeRuleFieldMap, ...experimentalRuleFieldMap }, - 'strict' - ), + mappings: mappingFromFieldMap(uptimeRuleTypeFieldMap, 'strict'), }, ], }); diff --git a/x-pack/plugins/synthetics/tsconfig.json b/x-pack/plugins/synthetics/tsconfig.json index 730a1f381d142..6cffd71042dee 100644 --- a/x-pack/plugins/synthetics/tsconfig.json +++ b/x-pack/plugins/synthetics/tsconfig.json @@ -74,6 +74,7 @@ "@kbn/shared-ux-prompt-not-found", "@kbn/safer-lodash-set", "@kbn/shared-ux-router", + "@kbn/alerts-as-data-utils", ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/timelines/common/utils/field_formatters.ts b/x-pack/plugins/timelines/common/utils/field_formatters.ts index 49590bfea54c1..e9b8b45bfefca 100644 --- a/x-pack/plugins/timelines/common/utils/field_formatters.ts +++ b/x-pack/plugins/timelines/common/utils/field_formatters.ts @@ -10,7 +10,7 @@ import { isEmpty } from 'lodash/fp'; import { ALERT_RULE_PARAMETERS } from '@kbn/rule-data-utils'; import { ecsFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/ecs_field_map'; import { technicalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/technical_rule_field_map'; -import { experimentalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/experimental_rule_field_map'; +import { legacyExperimentalFieldMap } from '@kbn/alerts-as-data-utils'; import { EventHit, TimelineEventsDetailsItem } from '../search_strategy'; import { toObjectArrayOfStrings, toStringArray } from './to_array'; import { ENRICHMENT_DESTINATION_PATH } from '../constants'; @@ -79,9 +79,11 @@ export const getDataFromFieldsHits = ( // return simple field value (non-ecs object, non-array) if ( !isObjectArray || - (Object.keys({ ...ecsFieldMap, ...technicalRuleFieldMap, ...experimentalRuleFieldMap }).find( - (ecsField) => ecsField === field - ) === undefined && + (Object.keys({ + ...ecsFieldMap, + ...technicalRuleFieldMap, + ...legacyExperimentalFieldMap, + }).find((ecsField) => ecsField === field) === undefined && !isRuleParametersFieldOrSubfield(field, prependField)) ) { return [ diff --git a/x-pack/plugins/timelines/tsconfig.json b/x-pack/plugins/timelines/tsconfig.json index 0612384baf8a7..91fb476e1f983 100644 --- a/x-pack/plugins/timelines/tsconfig.json +++ b/x-pack/plugins/timelines/tsconfig.json @@ -32,6 +32,7 @@ "@kbn/i18n", "@kbn/security-plugin", "@kbn/safer-lodash-set", + "@kbn/alerts-as-data-utils", "@kbn/logging", ], "exclude": [ diff --git a/x-pack/test/alerting_api_integration/common/plugins/alerts/server/alert_types.ts b/x-pack/test/alerting_api_integration/common/plugins/alerts/server/alert_types.ts index 3c2880d69f776..c2f6b5ca17d21 100644 --- a/x-pack/test/alerting_api_integration/common/plugins/alerts/server/alert_types.ts +++ b/x-pack/test/alerting_api_integration/common/plugins/alerts/server/alert_types.ts @@ -94,22 +94,24 @@ function getAlwaysFiringAlertType() { executor: curry(alwaysFiringExecutor)(), alerts: { context: 'test.always-firing', - fieldMap: { - instance_state_value: { - required: false, - type: 'boolean', - }, - instance_params_value: { - required: false, - type: 'boolean', - }, - instance_context_value: { - required: false, - type: 'boolean', - }, - group_in_series_index: { - required: false, - type: 'long', + mappings: { + fieldMap: { + instance_state_value: { + required: false, + type: 'boolean', + }, + instance_params_value: { + required: false, + type: 'boolean', + }, + instance_context_value: { + required: false, + type: 'boolean', + }, + group_in_series_index: { + required: false, + type: 'long', + }, }, }, }, @@ -823,6 +825,13 @@ function getAlwaysFiringAlertAsDataRuleType( return { state: {} }; }, + alerts: { + context: 'observability.test.alerts', + mappings: { + fieldMap: {}, + }, + useLegacyAlerts: true, + }, }); } diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data.ts index ad2c33b079b0a..c65af87d39aa7 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data.ts @@ -111,9 +111,9 @@ export default function createAlertsAsDataTest({ getService }: FtrProviderContex }); it('should install context specific alerts as data resources on startup', async () => { - const componentTemplateName = '.alerts-test.always-firing-mappings'; - const indexTemplateName = '.alerts-test.always-firing-default-template'; - const indexName = '.alerts-test.always-firing-default-000001'; + const componentTemplateName = '.alerts-test.always-firing.alerts-mappings'; + const indexTemplateName = '.alerts-test.always-firing.alerts-default-index-template'; + const indexName = '.internal.alerts-test.always-firing.alerts-default-000001'; const contextSpecificMappings = { instance_params_value: { type: 'boolean', @@ -139,16 +139,7 @@ export default function createAlertsAsDataTest({ getService }: FtrProviderContex dynamic: 'strict', properties: contextSpecificMappings, }); - expect(contextComponentTemplate.component_template.template.settings).to.eql({ - index: { - number_of_shards: 1, - mapping: { - total_fields: { - limit: 1500, - }, - }, - }, - }); + expect(contextComponentTemplate.component_template.template.settings).to.eql({}); const { index_templates: indexTemplates } = await es.indices.getIndexTemplate({ name: indexTemplateName, @@ -157,20 +148,25 @@ export default function createAlertsAsDataTest({ getService }: FtrProviderContex const contextIndexTemplate = indexTemplates[0]; expect(contextIndexTemplate.name).to.eql(indexTemplateName); expect(contextIndexTemplate.index_template.index_patterns).to.eql([ - '.alerts-test.always-firing-default-*', + '.internal.alerts-test.always-firing.alerts-default-*', ]); expect(contextIndexTemplate.index_template.composed_of).to.eql([ - '.alerts-test.always-firing-mappings', + '.alerts-test.always-firing.alerts-mappings', '.alerts-framework-mappings', ]); - expect(contextIndexTemplate.index_template.template!.mappings).to.eql({ - dynamic: false, - }); + expect(contextIndexTemplate.index_template.template!.mappings?.dynamic).to.eql(false); + expect(contextIndexTemplate.index_template.template!.mappings?._meta?.managed).to.eql(true); + expect(contextIndexTemplate.index_template.template!.mappings?._meta?.namespace).to.eql( + 'default' + ); + expect( + contextIndexTemplate.index_template.template!.mappings?._meta?.kibana?.version + ).to.be.a('string'); expect(contextIndexTemplate.index_template.template!.settings).to.eql({ index: { lifecycle: { name: '.alerts-ilm-policy', - rollover_alias: '.alerts-test.always-firing-default', + rollover_alias: '.alerts-test.always-firing.alerts-default', }, mapping: { total_fields: { @@ -187,22 +183,22 @@ export default function createAlertsAsDataTest({ getService }: FtrProviderContex }); expect(contextIndex[indexName].aliases).to.eql({ - '.alerts-test.always-firing-default': { + '.alerts-test.always-firing.alerts-default': { is_write_index: true, }, }); - - expect(contextIndex[indexName].mappings).to.eql({ - dynamic: 'false', - properties: { - ...contextSpecificMappings, - ...frameworkMappings.properties, - }, + expect(contextIndex[indexName].mappings?._meta?.managed).to.eql(true); + expect(contextIndex[indexName].mappings?._meta?.namespace).to.eql('default'); + expect(contextIndex[indexName].mappings?._meta?.kibana?.version).to.be.a('string'); + expect(contextIndex[indexName].mappings?.dynamic).to.eql('false'); + expect(contextIndex[indexName].mappings?.properties).to.eql({ + ...contextSpecificMappings, + ...frameworkMappings.properties, }); expect(contextIndex[indexName].settings?.index?.lifecycle).to.eql({ name: '.alerts-ilm-policy', - rollover_alias: '.alerts-test.always-firing-default', + rollover_alias: '.alerts-test.always-firing.alerts-default', }); expect(contextIndex[indexName].settings?.index?.mapping).to.eql({ @@ -215,7 +211,7 @@ export default function createAlertsAsDataTest({ getService }: FtrProviderContex expect(contextIndex[indexName].settings?.index?.number_of_shards).to.eql(1); expect(contextIndex[indexName].settings?.index?.auto_expand_replicas).to.eql('0-1'); expect(contextIndex[indexName].settings?.index?.provided_name).to.eql( - '.alerts-test.always-firing-default-000001' + '.internal.alerts-test.always-firing.alerts-default-000001' ); }); }); diff --git a/x-pack/test/rule_registry/spaces_only/tests/trial/get_summarized_alerts.ts b/x-pack/test/rule_registry/spaces_only/tests/trial/get_summarized_alerts.ts index b62cf7b39965c..3adff40258ca5 100644 --- a/x-pack/test/rule_registry/spaces_only/tests/trial/get_summarized_alerts.ts +++ b/x-pack/test/rule_registry/spaces_only/tests/trial/get_summarized_alerts.ts @@ -80,7 +80,10 @@ export default function createGetSummarizedAlertsTest({ getService }: FtrProvide isWriteEnabled: true, isWriterCacheEnabled: false, disabledRegistrationContexts: [] as string[], - areFrameworkAlertsEnabled: false, + frameworkAlerts: { + enabled: () => false, + getContextInitializationPromise: async () => ({ result: false }), + }, pluginStop$, }); diff --git a/x-pack/test/rule_registry/spaces_only/tests/trial/lifecycle_executor.ts b/x-pack/test/rule_registry/spaces_only/tests/trial/lifecycle_executor.ts index 10351fc6cf2ef..22810df3dd95d 100644 --- a/x-pack/test/rule_registry/spaces_only/tests/trial/lifecycle_executor.ts +++ b/x-pack/test/rule_registry/spaces_only/tests/trial/lifecycle_executor.ts @@ -74,7 +74,10 @@ export default function createLifecycleExecutorApiTest({ getService }: FtrProvid isWriteEnabled: true, isWriterCacheEnabled: false, disabledRegistrationContexts: [] as string[], - areFrameworkAlertsEnabled: false, + frameworkAlerts: { + enabled: () => false, + getContextInitializationPromise: async () => ({ result: false }), + }, pluginStop$, });